Update Keystone core to Mitaka.

Change-Id: Ia10d6add16f4a9d25d1f42d420661c46332e69db

10 files changed, 204 insertions, 58 deletions

diff --git a/keystone-moon/keystone/auth/__init__.py b/keystone-moon/keystone/auth/__init__.py
index b1e4203e..bcbf69fd 100644
--- a/keystone-moon/keystone/auth/__init__.py
+++ b/keystone-moon/keystone/auth/__init__.py
@@ -14,4 +14,3 @@
 
 from keystone.auth import controllers  # noqa
 from keystone.auth.core import *  # noqa
-from keystone.auth import routers  # noqa
diff --git a/keystone-moon/keystone/auth/controllers.py b/keystone-moon/keystone/auth/controllers.py
index 133230d6..3e6af80f 100644
--- a/keystone-moon/keystone/auth/controllers.py
+++ b/keystone-moon/keystone/auth/controllers.py
@@ -23,13 +23,13 @@ from oslo_utils import importutils
 import six
 import stevedore
 
+from keystone.common import config
 from keystone.common import controller
 from keystone.common import dependency
 from keystone.common import utils
 from keystone.common import wsgi
-from keystone import config
-from keystone.contrib.federation import constants as federation_constants
 from keystone import exception
+from keystone.federation import constants
 from keystone.i18n import _, _LI, _LW
 from keystone.resource import controllers as resource_controllers
 
@@ -45,8 +45,8 @@ AUTH_PLUGINS_LOADED = False
 
 def load_auth_method(method):
     plugin_name = CONF.auth.get(method) or 'default'
+    namespace = 'keystone.auth.%s' % method
     try:
-        namespace = 'keystone.auth.%s' % method
         driver_manager = stevedore.DriverManager(namespace, plugin_name,
                                                  invoke_on_load=True)
         return driver_manager.driver
@@ -55,13 +55,16 @@ def load_auth_method(method):
                   'attempt to load using import_object instead.',
                   method, plugin_name)
 
-    @versionutils.deprecated(as_of=versionutils.deprecated.LIBERTY,
-                             in_favor_of='entrypoints',
-                             what='direct import of driver')
-    def _load_using_import(plugin_name):
-        return importutils.import_object(plugin_name)
+    driver = importutils.import_object(plugin_name)
 
-    return _load_using_import(plugin_name)
+    msg = (_(
+        'Direct import of auth plugin %(name)r is deprecated as of Liberty in '
+        'favor of its entrypoint from %(namespace)r and may be removed in '
+        'N.') %
+        {'name': plugin_name, 'namespace': namespace})
+    versionutils.report_deprecated_feature(LOG, msg)
+
+    return driver
 
 
 def load_auth_methods():
@@ -174,6 +177,10 @@ class AuthInfo(object):
                                             target='domain')
         try:
             if domain_name:
+                if (CONF.resource.domain_name_url_safe == 'strict' and
+                        utils.is_not_url_safe(domain_name)):
+                    msg = _('Domain name cannot contain reserved characters.')
+                    raise exception.Unauthorized(message=msg)
                 domain_ref = self.resource_api.get_domain_by_name(
                     domain_name)
             else:
@@ -193,6 +200,10 @@ class AuthInfo(object):
                                             target='project')
         try:
             if project_name:
+                if (CONF.resource.project_name_url_safe == 'strict' and
+                        utils.is_not_url_safe(project_name)):
+                    msg = _('Project name cannot contain reserved characters.')
+                    raise exception.Unauthorized(message=msg)
                 if 'domain' not in project_info:
                     raise exception.ValidationError(attribute='domain',
                                                     target='project')
@@ -423,7 +434,7 @@ class Auth(controller.V3Controller):
             return
 
         # Skip scoping when unscoped federated token is being issued
-        if federation_constants.IDENTITY_PROVIDER in auth_context:
+        if constants.IDENTITY_PROVIDER in auth_context:
             return
 
         # Do not scope if request is for explicitly unscoped token
@@ -479,7 +490,6 @@ class Auth(controller.V3Controller):
 
     def authenticate(self, context, auth_info, auth_context):
         """Authenticate user."""
-
         # The 'external' method allows any 'REMOTE_USER' based authentication
         # In some cases the server can set REMOTE_USER as '' instead of
         # dropping it, so this must be filtered out
@@ -549,13 +559,23 @@ class Auth(controller.V3Controller):
     def revocation_list(self, context, auth=None):
         if not CONF.token.revoke_by_id:
             raise exception.Gone()
+
+        audit_id_only = ('audit_id_only' in context['query_string'])
+
         tokens = self.token_provider_api.list_revoked_tokens()
 
         for t in tokens:
             expires = t['expires']
             if not (expires and isinstance(expires, six.text_type)):
                 t['expires'] = utils.isotime(expires)
+            if audit_id_only:
+                t.pop('id', None)
         data = {'revoked': tokens}
+
+        if audit_id_only:
+            # No need to obfuscate if no token IDs.
+            return data
+
         json_data = jsonutils.dumps(data)
         signed_text = cms.cms_sign_text(json_data,
                                         CONF.signing.certfile,
@@ -580,7 +600,7 @@ class Auth(controller.V3Controller):
         if user_id:
             try:
                 user_refs = self.assignment_api.list_projects_for_user(user_id)
-            except exception.UserNotFound:
+            except exception.UserNotFound:  # nosec
                 # federated users have an id but they don't link to anything
                 pass
 
@@ -601,7 +621,7 @@ class Auth(controller.V3Controller):
         if user_id:
             try:
                 user_refs = self.assignment_api.list_domains_for_user(user_id)
-            except exception.UserNotFound:
+            except exception.UserNotFound:  # nosec
                 # federated users have an id but they don't link to anything
                 pass
 
diff --git a/keystone-moon/keystone/auth/core.py b/keystone-moon/keystone/auth/core.py
index 9da2c123..b865d82b 100644
--- a/keystone-moon/keystone/auth/core.py
+++ b/keystone-moon/keystone/auth/core.py
@@ -89,6 +89,6 @@ class AuthMethodHandler(object):
                   Authentication payload in the form of a dictionary for the
                   next authentication step if this is a multi step
                   authentication.
-        :raises: exception.Unauthorized for authentication failure
+        :raises keystone.exception.Unauthorized: for authentication failure
         """
         raise exception.Unauthorized()
diff --git a/keystone-moon/keystone/auth/plugins/core.py b/keystone-moon/keystone/auth/plugins/core.py
index bcad27e5..c513f815 100644
--- a/keystone-moon/keystone/auth/plugins/core.py
+++ b/keystone-moon/keystone/auth/plugins/core.py
@@ -99,18 +99,17 @@ def convert_integer_to_method_list(method_int):
 
 
 @dependency.requires('identity_api', 'resource_api')
-class UserAuthInfo(object):
+class BaseUserInfo(object):
 
-    @staticmethod
-    def create(auth_payload, method_name):
-        user_auth_info = UserAuthInfo()
+    @classmethod
+    def create(cls, auth_payload, method_name):
+        user_auth_info = cls()
         user_auth_info._validate_and_normalize_auth_data(auth_payload)
         user_auth_info.METHOD_NAME = method_name
         return user_auth_info
 
     def __init__(self):
         self.user_id = None
-        self.password = None
         self.user_ref = None
         self.METHOD_NAME = None
 
@@ -164,7 +163,6 @@ class UserAuthInfo(object):
         if not user_id and not user_name:
             raise exception.ValidationError(attribute='id or name',
                                             target='user')
-        self.password = user_info.get('password')
         try:
             if user_name:
                 if 'domain' not in user_info:
@@ -185,3 +183,29 @@ class UserAuthInfo(object):
         self.user_ref = user_ref
         self.user_id = user_ref['id']
         self.domain_id = domain_ref['id']
+
+
+class UserAuthInfo(BaseUserInfo):
+
+    def __init__(self):
+        super(UserAuthInfo, self).__init__()
+        self.password = None
+
+    def _validate_and_normalize_auth_data(self, auth_payload):
+        super(UserAuthInfo, self)._validate_and_normalize_auth_data(
+            auth_payload)
+        user_info = auth_payload['user']
+        self.password = user_info.get('password')
+
+
+class TOTPUserInfo(BaseUserInfo):
+
+    def __init__(self):
+        super(TOTPUserInfo, self).__init__()
+        self.passcode = None
+
+    def _validate_and_normalize_auth_data(self, auth_payload):
+        super(TOTPUserInfo, self)._validate_and_normalize_auth_data(
+            auth_payload)
+        user_info = auth_payload['user']
+        self.passcode = user_info.get('passcode')
diff --git a/keystone-moon/keystone/auth/plugins/external.py b/keystone-moon/keystone/auth/plugins/external.py
index cabe6282..b00b808a 100644
--- a/keystone-moon/keystone/auth/plugins/external.py
+++ b/keystone-moon/keystone/auth/plugins/external.py
@@ -78,7 +78,6 @@ class Domain(Base):
         The domain will be extracted from the REMOTE_DOMAIN environment
         variable if present. If not, the default domain will be used.
         """
-
         username = remote_user
         try:
             domain_name = context['environment']['REMOTE_DOMAIN']
@@ -94,6 +93,7 @@ class Domain(Base):
 
 class KerberosDomain(Domain):
     """Allows `kerberos` as a method."""
+
     def _authenticate(self, remote_user, context):
         auth_type = context['environment'].get('AUTH_TYPE')
         if auth_type != 'Negotiate':
diff --git a/keystone-moon/keystone/auth/plugins/mapped.py b/keystone-moon/keystone/auth/plugins/mapped.py
index 220ff013..e9716201 100644
--- a/keystone-moon/keystone/auth/plugins/mapped.py
+++ b/keystone-moon/keystone/auth/plugins/mapped.py
@@ -12,23 +12,20 @@
 
 import functools
 
-from oslo_log import log
 from pycadf import cadftaxonomy as taxonomy
 from six.moves.urllib import parse
 
 from keystone import auth
 from keystone.auth import plugins as auth_plugins
 from keystone.common import dependency
-from keystone.contrib.federation import constants as federation_constants
-from keystone.contrib.federation import utils
 from keystone import exception
+from keystone.federation import constants as federation_constants
+from keystone.federation import utils
 from keystone.i18n import _
 from keystone.models import token_model
 from keystone import notifications
 
 
-LOG = log.getLogger(__name__)
-
 METHOD_NAME = 'mapped'
 
 
@@ -56,7 +53,6 @@ class Mapped(auth.AuthMethodHandler):
         ``OS-FEDERATION:protocol``
 
         """
-
         if 'id' in auth_payload:
             token_ref = self._get_token_ref(auth_payload)
             handle_scoped_token(context, auth_payload, auth_context, token_ref,
@@ -139,12 +135,22 @@ def handle_unscoped_token(context, auth_payload, auth_context,
     user_id = None
 
     try:
-        mapped_properties, mapping_id = apply_mapping_filter(
-            identity_provider, protocol, assertion, resource_api,
-            federation_api, identity_api)
+        try:
+            mapped_properties, mapping_id = apply_mapping_filter(
+                identity_provider, protocol, assertion, resource_api,
+                federation_api, identity_api)
+        except exception.ValidationError as e:
+            # if mapping is either invalid or yield no valid identity,
+            # it is considered a failed authentication
+            raise exception.Unauthorized(e)
 
         if is_ephemeral_user(mapped_properties):
-            user = setup_username(context, mapped_properties)
+            unique_id, display_name = (
+                get_user_unique_id_and_display_name(context, mapped_properties)
+            )
+            user = identity_api.shadow_federated_user(identity_provider,
+                                                      protocol, unique_id,
+                                                      display_name)
             user_id = user['id']
             group_ids = mapped_properties['group_ids']
             utils.validate_groups_cardinality(group_ids, mapping_id)
@@ -205,7 +211,7 @@ def apply_mapping_filter(identity_provider, protocol, assertion,
     return mapped_properties, mapping_id
 
 
-def setup_username(context, mapped_properties):
+def get_user_unique_id_and_display_name(context, mapped_properties):
     """Setup federated username.
 
     Function covers all the cases for properly setting user id, a primary
@@ -225,9 +231,10 @@ def setup_username(context, mapped_properties):
     :param mapped_properties: Properties issued by a RuleProcessor.
     :type: dictionary
 
-    :raises: exception.Unauthorized
-    :returns: dictionary with user identification
-    :rtype: dict
+    :raises keystone.exception.Unauthorized: If neither `user_name` nor
+        `user_id` is set.
+    :returns: tuple with user identification
+    :rtype: tuple
 
     """
     user = mapped_properties['user']
@@ -248,5 +255,4 @@ def setup_username(context, mapped_properties):
         user_id = user_name
 
     user['id'] = parse.quote(user_id)
-
-    return user
+    return (user['id'], user['name'])
diff --git a/keystone-moon/keystone/auth/plugins/oauth1.py b/keystone-moon/keystone/auth/plugins/oauth1.py
index e081cd62..bf60f91c 100644
--- a/keystone-moon/keystone/auth/plugins/oauth1.py
+++ b/keystone-moon/keystone/auth/plugins/oauth1.py
@@ -12,26 +12,21 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from oslo_log import log
 from oslo_utils import timeutils
 
 from keystone import auth
 from keystone.common import controller
 from keystone.common import dependency
-from keystone.contrib.oauth1 import core as oauth
-from keystone.contrib.oauth1 import validator
 from keystone import exception
 from keystone.i18n import _
-
-
-LOG = log.getLogger(__name__)
+from keystone.oauth1 import core as oauth
+from keystone.oauth1 import validator
 
 
 @dependency.requires('oauth_api')
 class OAuth(auth.AuthMethodHandler):
     def authenticate(self, context, auth_info, auth_context):
         """Turn a signed request with an access key into a keystone token."""
-
         headers = context['headers']
         oauth_headers = oauth.get_oauth_headers(headers)
         access_token_id = oauth_headers.get('oauth_token')
diff --git a/keystone-moon/keystone/auth/plugins/password.py b/keystone-moon/keystone/auth/plugins/password.py
index 16492a32..a16887b4 100644
--- a/keystone-moon/keystone/auth/plugins/password.py
+++ b/keystone-moon/keystone/auth/plugins/password.py
@@ -12,8 +12,6 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from oslo_log import log
-
 from keystone import auth
 from keystone.auth import plugins as auth_plugins
 from keystone.common import dependency
@@ -23,8 +21,6 @@ from keystone.i18n import _
 
 METHOD_NAME = 'password'
 
-LOG = log.getLogger(__name__)
-
 
 @dependency.requires('identity_api')
 class Password(auth.AuthMethodHandler):
@@ -33,8 +29,6 @@ class Password(auth.AuthMethodHandler):
         """Try to authenticate against the identity backend."""
         user_info = auth_plugins.UserAuthInfo.create(auth_payload, METHOD_NAME)
 
-        # FIXME(gyee): identity.authenticate() can use some refactoring since
-        # all we care is password matches
         try:
             self.identity_api.authenticate(
                 context,
diff --git a/keystone-moon/keystone/auth/plugins/saml2.py b/keystone-moon/keystone/auth/plugins/saml2.py
index cf7a8a50..0e7ec6bc 100644
--- a/keystone-moon/keystone/auth/plugins/saml2.py
+++ b/keystone-moon/keystone/auth/plugins/saml2.py
@@ -10,17 +10,26 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from oslo_log import versionutils
+
 from keystone.auth.plugins import mapped
 
-""" Provide an entry point to authenticate with SAML2
 
-This plugin subclasses mapped.Mapped, and may be specified in keystone.conf:
+@versionutils.deprecated(
+    versionutils.deprecated.MITAKA,
+    what='keystone.auth.plugins.saml2.Saml2',
+    in_favor_of='keystone.auth.plugins.mapped.Mapped',
+    remove_in=+2)
+class Saml2(mapped.Mapped):
+    """Provide an entry point to authenticate with SAML2.
+
+    This plugin subclasses ``mapped.Mapped``, and may be specified in
+    keystone.conf::
 
-  [auth]
-  methods = external,password,token,saml2
-  saml2 = keystone.auth.plugins.mapped.Mapped
-"""
+        [auth]
+        methods = external,password,token,saml2
+        saml2 = keystone.auth.plugins.mapped.Mapped
 
+    """
 
-class Saml2(mapped.Mapped):
     pass
diff --git a/keystone-moon/keystone/auth/plugins/totp.py b/keystone-moon/keystone/auth/plugins/totp.py
new file mode 100644
index 00000000..d0b61b3b
--- /dev/null
+++ b/keystone-moon/keystone/auth/plugins/totp.py
@@ -0,0 +1,99 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Time-based One-time Password Algorithm (TOTP) auth plugin
+
+TOTP is an algorithm that computes a one-time password from a shared secret
+key and the current time.
+
+TOTP is an implementation of a hash-based message authentication code (HMAC).
+It combines a secret key with the current timestamp using a cryptographic hash
+function to generate a one-time password. The timestamp typically increases in
+30-second intervals, so passwords generated close together in time from the
+same secret key will be equal.
+"""
+
+import base64
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.twofactor import totp as crypto_totp
+from oslo_log import log
+from oslo_utils import timeutils
+import six
+
+from keystone import auth
+from keystone.auth import plugins
+from keystone.common import dependency
+from keystone import exception
+from keystone.i18n import _
+
+
+METHOD_NAME = 'totp'
+
+LOG = log.getLogger(__name__)
+
+
+def _generate_totp_passcode(secret):
+    """Generate TOTP passcode.
+
+    :param bytes secret: A base32 encoded secret for the TOTP authentication
+    :returns: totp passcode as bytes
+    """
+    if isinstance(secret, six.text_type):
+        # NOTE(dstanek): since this may be coming from the JSON stored in the
+        # database it may be UTF-8 encoded
+        secret = secret.encode('utf-8')
+
+    # NOTE(nonameentername): cryptography takes a non base32 encoded value for
+    # TOTP. Add the correct padding to be able to base32 decode
+    while len(secret) % 8 != 0:
+        secret = secret + b'='
+
+    decoded = base64.b32decode(secret)
+    totp = crypto_totp.TOTP(
+        decoded, 6, hashes.SHA1(), 30, backend=default_backend())
+    return totp.generate(timeutils.utcnow_ts(microsecond=True))
+
+
+@dependency.requires('credential_api')
+class TOTP(auth.AuthMethodHandler):
+
+    def authenticate(self, context, auth_payload, auth_context):
+        """Try to authenticate using TOTP"""
+        user_info = plugins.TOTPUserInfo.create(auth_payload, METHOD_NAME)
+        auth_passcode = auth_payload.get('user').get('passcode')
+
+        credentials = self.credential_api.list_credentials_for_user(
+            user_info.user_id, type='totp')
+
+        valid_passcode = False
+        for credential in credentials:
+            try:
+                generated_passcode = _generate_totp_passcode(
+                    credential['blob'])
+                if auth_passcode == generated_passcode:
+                    valid_passcode = True
+                    break
+            except (ValueError, KeyError):
+                LOG.debug('No TOTP match; credential id: %s, user_id: %s',
+                          credential['id'], user_info.user_id)
+            except (TypeError):
+                LOG.debug('Base32 decode failed for TOTP credential %s',
+                          credential['id'])
+
+        if not valid_passcode:
+            # authentication failed because of invalid username or passcode
+            msg = _('Invalid username or TOTP passcode')
+            raise exception.Unauthorized(msg)
+
+        auth_context['user_id'] = user_info.user_id