Update KeystoneMiddleware to the stable/liberty version.

Change-Id: I225ed685dad129dc7c1d5d6a00e54c0facde0c07

7 files changed, 175 insertions, 26 deletions

diff --git a/keystonemiddleware-moon/keystonemiddleware/auth_token/__init__.py b/keystonemiddleware-moon/keystonemiddleware/auth_token/__init__.py
index 8987e0ea..be268da3 100644
--- a/keystonemiddleware-moon/keystonemiddleware/auth_token/__init__.py
+++ b/keystonemiddleware-moon/keystonemiddleware/auth_token/__init__.py
@@ -206,6 +206,7 @@ object is stored.
 
 """
 
+import binascii
 import datetime
 import logging
 
@@ -511,7 +512,7 @@ class _BaseAuthProtocol(object):
 
         :raises exc.InvalidToken: if token is rejected
         """
-        # 0 seconds of validity means it is invalid right now
+        # 0 seconds of validity means is it valid right now.
         if auth_ref.will_expire_soon(stale_duration=0):
             raise exc.InvalidToken(_('Token authorization failed'))
 
@@ -838,8 +839,9 @@ class AuthProtocol(_BaseAuthProtocol):
                 data = cached
 
                 if self._check_revocations_for_cached:
-                    # A token might have been revoked, regardless of initial
-                    # mechanism used to validate it, and needs to be checked.
+                    # A token stored in Memcached might have been revoked
+                    # regardless of initial mechanism used to validate it,
+                    # and needs to be checked.
                     self._revocations.check(token_hashes)
             else:
                 data = self._validate_offline(token, token_hashes)
@@ -848,19 +850,19 @@ class AuthProtocol(_BaseAuthProtocol):
 
                 self._token_cache.store(token_hashes[0], data)
 
-        except (exceptions.ConnectionRefused, exceptions.RequestTimeout):
-            self.log.debug('Token validation failure.', exc_info=True)
-            self.log.warning(_LW('Authorization failed for token'))
-            raise exc.InvalidToken(_('Token authorization failed'))
-        except exc.ServiceError as e:
-            self.log.critical(_LC('Unable to obtain admin token: %s'), e)
+        except (exceptions.ConnectionRefused, exceptions.RequestTimeout,
+                exc.RevocationListError, exc.ServiceError) as e:
+            self.log.critical(_LC('Unable to validate token: %s'), e)
             raise webob.exc.HTTPServiceUnavailable()
-        except Exception:
+        except exc.InvalidToken:
             self.log.debug('Token validation failure.', exc_info=True)
             if token_hashes:
                 self._token_cache.store_invalid(token_hashes[0])
             self.log.warning(_LW('Authorization failed for token'))
-            raise exc.InvalidToken(_('Token authorization failed'))
+            raise
+        except Exception:
+            self.log.critical(_LC('Unable to validate token'), exc_info=True)
+            raise webob.exc.HTTPInternalServerError()
 
         return data
 
@@ -881,6 +883,18 @@ class AuthProtocol(_BaseAuthProtocol):
                                  'fallback to online validation.'))
         else:
             data = jsonutils.loads(verified)
+
+            audit_ids = None
+            if 'access' in data:
+                # It's a v2 token.
+                audit_ids = data['access']['token'].get('audit_ids')
+            else:
+                # It's a v3 token
+                audit_ids = data['token'].get('audit_ids')
+
+            if audit_ids:
+                self._revocations.check_by_audit_id(audit_ids)
+
             return data
 
     def _validate_token(self, auth_ref):
@@ -905,9 +919,10 @@ class AuthProtocol(_BaseAuthProtocol):
                 return cms.cms_verify(data, signing_cert_path,
                                       signing_ca_path,
                                       inform=inform).decode('utf-8')
-            except cms.subprocess.CalledProcessError as err:
+            except (exceptions.CMSError,
+                    cms.subprocess.CalledProcessError) as err:
                 self.log.warning(_LW('Verify error: %s'), err)
-                raise
+                raise exc.InvalidToken(_('Token authorization failed'))
 
         try:
             return verify()
@@ -939,7 +954,8 @@ class AuthProtocol(_BaseAuthProtocol):
             verified = self._cms_verify(uncompressed, inform=cms.PKIZ_CMS_FORM)
             return verified
         # TypeError If the signed_text is not zlib compressed
-        except TypeError:
+        # binascii.Error if signed_text has incorrect base64 padding (py34)
+        except (TypeError, binascii.Error):
             raise exc.InvalidToken(signed_text)
 
     def _fetch_signing_cert(self):
diff --git a/keystonemiddleware-moon/keystonemiddleware/auth_token/_identity.py b/keystonemiddleware-moon/keystonemiddleware/auth_token/_identity.py
index 98be3b2e..6fbeac27 100644
--- a/keystonemiddleware-moon/keystonemiddleware/auth_token/_identity.py
+++ b/keystonemiddleware-moon/keystonemiddleware/auth_token/_identity.py
@@ -212,25 +212,28 @@ class IdentityServer(object):
         try:
             auth_ref = self._request_strategy.verify_token(user_token)
         except exceptions.NotFound as e:
-            self._LOG.warn(_LW('Authorization failed for token'))
-            self._LOG.warn(_LW('Identity response: %s'), e.response.text)
+            self._LOG.warning(_LW('Authorization failed for token'))
+            self._LOG.warning(_LW('Identity response: %s'), e.response.text)
+            raise exc.InvalidToken(_('Token authorization failed'))
         except exceptions.Unauthorized as e:
             self._LOG.info(_LI('Identity server rejected authorization'))
-            self._LOG.warn(_LW('Identity response: %s'), e.response.text)
+            self._LOG.warning(_LW('Identity response: %s'), e.response.text)
             if retry:
                 self._LOG.info(_LI('Retrying validation'))
                 return self.verify_token(user_token, False)
+            msg = _('Identity server rejected authorization necessary to '
+                    'fetch token data')
+            raise exc.ServiceError(msg)
         except exceptions.HttpError as e:
             self._LOG.error(
                 _LE('Bad response code while validating token: %s'),
                 e.http_status)
-            self._LOG.warn(_LW('Identity response: %s'), e.response.text)
+            self._LOG.warning(_LW('Identity response: %s'), e.response.text)
+            msg = _('Failed to fetch token data from identity server')
+            raise exc.ServiceError(msg)
         else:
             return auth_ref
 
-        msg = _('Failed to fetch token data from identity server')
-        raise exc.InvalidToken(msg)
-
     def fetch_revocation_list(self):
         try:
             data = self._request_strategy.fetch_revocation_list()
diff --git a/keystonemiddleware-moon/keystonemiddleware/auth_token/_revocations.py b/keystonemiddleware-moon/keystonemiddleware/auth_token/_revocations.py
index 8cc449ad..a68356a8 100644
--- a/keystonemiddleware-moon/keystonemiddleware/auth_token/_revocations.py
+++ b/keystonemiddleware-moon/keystonemiddleware/auth_token/_revocations.py
@@ -104,3 +104,25 @@ class Revocations(object):
         if self._any_revoked(token_ids):
             self._log.debug('Token is marked as having been revoked')
             raise exc.InvalidToken(_('Token has been revoked'))
+
+    def check_by_audit_id(self, audit_ids):
+        """Check whether the audit_id appears in the revocation list.
+
+        :raises keystonemiddleware.auth_token._exceptions.InvalidToken:
+            if the audit ID(s) appear in the revocation list.
+
+        """
+        revoked_tokens = self._list.get('revoked', None)
+        if not revoked_tokens:
+            # There's no revoked tokens, so nothing to do.
+            return
+
+        # The audit_id may not be present in the revocation events because
+        # earlier versions of the identity server didn't provide them.
+        revoked_ids = set(
+            x['audit_id'] for x in revoked_tokens if 'audit_id' in x)
+        for audit_id in audit_ids:
+            if audit_id in revoked_ids:
+                self._log.debug(
+                    'Token is marked as having been revoked by audit id')
+                raise exc.InvalidToken(_('Token has been revoked'))
diff --git a/keystonemiddleware-moon/keystonemiddleware/tests/unit/auth_token/test_auth_token_middleware.py b/keystonemiddleware-moon/keystonemiddleware/tests/unit/auth_token/test_auth_token_middleware.py
index bb572aa3..e6a495f4 100644
--- a/keystonemiddleware-moon/keystonemiddleware/tests/unit/auth_token/test_auth_token_middleware.py
+++ b/keystonemiddleware-moon/keystonemiddleware/tests/unit/auth_token/test_auth_token_middleware.py
@@ -21,6 +21,7 @@ import stat
 import tempfile
 import time
 import uuid
+import warnings
 
 import fixtures
 from keystoneclient import auth
@@ -313,6 +314,11 @@ class BaseAuthTokenMiddlewareTest(base.BaseAuthTokenTestCase):
         self.response_status = None
         self.response_headers = None
 
+        # NOTE(gyee): For this test suite and for the stable liberty branch
+        # only, we will ignore deprecated calls that keystonemiddleware makes.
+        warnings.filterwarnings('ignore', category=DeprecationWarning,
+                                module='^keystonemiddleware\\.')
+
     def call_middleware(self, **kwargs):
         return self.call(self.middleware, **kwargs)
 
@@ -773,6 +779,33 @@ class CommonAuthTokenMiddlewareTest(object):
         resp = self.call_middleware(headers={'X-Auth-Token': token})
         self.assertEqual(401, resp.status_int)
 
+    def test_cached_revoked_error(self):
+        # When the token is cached and revocation list retrieval fails,
+        # 503 is returned
+        token = self.token_dict['uuid_token_default']
+        self.middleware._check_revocations_for_cached = True
+
+        # Token should be cached as ok after this.
+        resp = self.call_middleware(headers={'X-Auth-Token': token})
+        self.assertEqual(200, resp.status_int)
+
+        # Cause the revocation list to be fetched again next time so we can
+        # test the case where that retrieval fails
+        self.middleware._revocations._fetched_time = datetime.datetime.min
+        with mock.patch.object(self.middleware._revocations, '_fetch',
+                               side_effect=exc.RevocationListError):
+            resp = self.call_middleware(headers={'X-Auth-Token': token})
+            self.assertEqual(503, resp.status_int)
+
+    def test_unexpected_exception_in_validate_offline(self):
+        # When an unexpected exception is hit during _validate_offline,
+        # 500 is returned
+        token = self.token_dict['uuid_token_default']
+        with mock.patch.object(self.middleware, '_validate_offline',
+                               side_effect=Exception):
+            resp = self.call_middleware(headers={'X-Auth-Token': token})
+            self.assertEqual(500, resp.status_int)
+
     def test_cached_revoked_uuid(self):
         # When the UUID token is cached and revoked, 401 is returned.
         self._test_cache_revoked(self.token_dict['uuid_token_default'])
@@ -869,6 +902,30 @@ class CommonAuthTokenMiddlewareTest(object):
     def test_revoked_hashed_pkiz_token(self):
         self._test_revoked_hashed_token('signed_token_scoped_pkiz')
 
+    def test_revoked_pki_token_by_audit_id(self):
+        # When the audit ID is in the revocation list, the token is invalid.
+        self.set_middleware()
+        token = self.token_dict['signed_token_scoped']
+
+        # Put the token audit ID in the revocation list,
+        # the entry will have a false token ID so the token ID doesn't match.
+        fake_token_id = uuid.uuid4().hex
+        # The audit_id value is in examples/pki/cms/auth_*_token_scoped.json.
+        audit_id = 'SLIXlXQUQZWUi9VJrqdXqA'
+        revocation_list_data = {
+            'revoked': [
+                {
+                    'id': fake_token_id,
+                    'audit_id': audit_id
+                },
+            ]
+        }
+        self.middleware._revocations._list = jsonutils.dumps(
+            revocation_list_data)
+
+        resp = self.call_middleware(headers={'X-Auth-Token': token})
+        self.assertEqual(401, resp.status_int)
+
     def get_revocation_list_json(self, token_ids=None, mode=None):
         if token_ids is None:
             key = 'revoked_token_hash' + (('_' + mode) if mode else '')
@@ -2085,7 +2142,7 @@ class CommonCompositeAuthTests(object):
         }
         self.update_expected_env(expected_env)
 
-        token = 'invalid-user-token'
+        token = 'invalid-token'
         service_token = 'invalid-service-token'
         resp = self.call_middleware(headers={'X-Auth-Token': token,
                                              'X-Service-Token': service_token})
diff --git a/keystonemiddleware-moon/keystonemiddleware/tests/unit/auth_token/test_revocations.py b/keystonemiddleware-moon/keystonemiddleware/tests/unit/auth_token/test_revocations.py
index cef65b8e..258e195a 100644
--- a/keystonemiddleware-moon/keystonemiddleware/tests/unit/auth_token/test_revocations.py
+++ b/keystonemiddleware-moon/keystonemiddleware/tests/unit/auth_token/test_revocations.py
@@ -27,22 +27,24 @@ from keystonemiddleware.tests.unit import utils
 
 class RevocationsTests(utils.BaseTestCase):
 
-    def _check_with_list(self, revoked_list, token_ids):
+    def _setup_revocations(self, revoked_list):
         directory_name = '/tmp/%s' % uuid.uuid4().hex
         signing_directory = _signing_dir.SigningDirectory(directory_name)
         self.addCleanup(shutil.rmtree, directory_name)
 
         identity_server = mock.Mock()
 
-        verify_result_obj = {
-            'revoked': list({'id': r} for r in revoked_list)
-        }
+        verify_result_obj = {'revoked': revoked_list}
         cms_verify = mock.Mock(return_value=json.dumps(verify_result_obj))
 
         revocations = _revocations.Revocations(
             timeout=datetime.timedelta(1), signing_directory=signing_directory,
             identity_server=identity_server, cms_verify=cms_verify)
+        return revocations
 
+    def _check_with_list(self, revoked_list, token_ids):
+        revoked_list = list({'id': r} for r in revoked_list)
+        revocations = self._setup_revocations(revoked_list)
         revocations.check(token_ids)
 
     def test_check_empty_list(self):
@@ -63,3 +65,40 @@ class RevocationsTests(utils.BaseTestCase):
         token_ids = [token_id]
         self.assertRaises(exc.InvalidToken,
                           self._check_with_list, revoked_tokens, token_ids)
+
+    def test_check_by_audit_id_revoked(self):
+        # When the audit ID is in the revocation list, InvalidToken is raised.
+        audit_id = uuid.uuid4().hex
+        revoked_list = [{'id': uuid.uuid4().hex, 'audit_id': audit_id}]
+        revocations = self._setup_revocations(revoked_list)
+        self.assertRaises(exc.InvalidToken,
+                          revocations.check_by_audit_id, [audit_id])
+
+    def test_check_by_audit_id_chain_revoked(self):
+        # When the token's audit chain ID is in the revocation list,
+        # InvalidToken is raised.
+        revoked_audit_id = uuid.uuid4().hex
+        revoked_list = [{'id': uuid.uuid4().hex, 'audit_id': revoked_audit_id}]
+        revocations = self._setup_revocations(revoked_list)
+
+        token_audit_ids = [uuid.uuid4().hex, revoked_audit_id]
+        self.assertRaises(exc.InvalidToken,
+                          revocations.check_by_audit_id, token_audit_ids)
+
+    def test_check_by_audit_id_not_revoked(self):
+        # When the audit ID is not in the revocation list no exception.
+        revoked_list = [{'id': uuid.uuid4().hex, 'audit_id': uuid.uuid4().hex}]
+        revocations = self._setup_revocations(revoked_list)
+
+        audit_id = uuid.uuid4().hex
+        revocations.check_by_audit_id([audit_id])
+
+    def test_check_by_audit_id_no_audit_ids(self):
+        # Older identity servers don't send audit_ids in the revocation list.
+        # When this happens, check_by_audit_id still works, just doesn't
+        # verify anything.
+        revoked_list = [{'id': uuid.uuid4().hex}]
+        revocations = self._setup_revocations(revoked_list)
+
+        audit_id = uuid.uuid4().hex
+        revocations.check_by_audit_id([audit_id])
diff --git a/keystonemiddleware-moon/keystonemiddleware/tests/unit/auth_token/test_user_auth_plugin.py b/keystonemiddleware-moon/keystonemiddleware/tests/unit/auth_token/test_user_auth_plugin.py
index 52d29737..19d3d7a9 100644
--- a/keystonemiddleware-moon/keystonemiddleware/tests/unit/auth_token/test_user_auth_plugin.py
+++ b/keystonemiddleware-moon/keystonemiddleware/tests/unit/auth_token/test_user_auth_plugin.py
@@ -11,6 +11,7 @@
 # under the License.
 
 import uuid
+import warnings
 
 from keystoneclient import auth
 from keystoneclient import fixture
@@ -29,6 +30,11 @@ class BaseUserPluginTests(object):
                              auth_plugin,
                              group='keystone_authtoken',
                              **kwargs):
+        # NOTE(gyee): For this test suite and for the stable liberty branch
+        # only, we will ignore deprecated calls that keystonemiddleware makes.
+        warnings.filterwarnings('ignore', category=DeprecationWarning,
+                                module='^keystonemiddleware\\.')
+
         opts = auth.get_plugin_class(auth_plugin).get_options()
         self.cfg.register_opts(opts, group=group)
 
diff --git a/keystonemiddleware-moon/keystonemiddleware/tests/unit/test_audit_middleware.py b/keystonemiddleware-moon/keystonemiddleware/tests/unit/test_audit_middleware.py
index 48ff9a4f..fc761c0f 100644
--- a/keystonemiddleware-moon/keystonemiddleware/tests/unit/test_audit_middleware.py
+++ b/keystonemiddleware-moon/keystonemiddleware/tests/unit/test_audit_middleware.py
@@ -14,6 +14,7 @@
 import os
 import tempfile
 import uuid
+import warnings
 
 import mock
 from oslo_config import cfg
@@ -64,6 +65,11 @@ class BaseAuditMiddlewareTest(utils.BaseTestCase):
             FakeApp(), audit_map_file=self.audit_map,
             service_name='pycadf')
 
+        # NOTE(stevemar): For this test suite and for the stable liberty branch
+        # only, we will ignore deprecated calls that keystonemiddleware makes.
+        warnings.filterwarnings('ignore', category=DeprecationWarning,
+                                module='^keystonemiddleware\\.')
+
         self.addCleanup(lambda: os.close(self.fd))
         self.addCleanup(cfg.CONF.reset)