Restructcture of the directory layout

Restructure of the directory layout due to move of Fuel into it's own repo
JIRA: FUEL-85

Change-Id: I3647e1992a508f29dce06a5d6c790725c527f6f5
Signed-off-by: Jonas Bjurel <jonas.bjurel@ericsson.com>

6 files changed, 559 insertions, 0 deletions

diff --git a/deploy/cloud/configure_environment.py b/deploy/cloud/configure_environment.py
new file mode 100644
index 000000000..ed89fa600
--- /dev/null
+++ b/deploy/cloud/configure_environment.py
@@ -0,0 +1,76 @@
+###############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# szilard.cserey@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+from configure_settings import ConfigureSettings
+from configure_network import ConfigureNetwork
+from configure_nodes import ConfigureNodes
+
+from common import (
+    E,
+    exec_cmd,
+    parse,
+    err,
+    log,
+    delete,
+    create_dir_if_not_exists,
+)
+
+
+class ConfigureEnvironment(object):
+
+    def __init__(self, dea, yaml_config_dir, release_id, node_id_roles_dict):
+        self.env_id = None
+        self.dea = dea
+        self.yaml_config_dir = yaml_config_dir
+        self.release_id = release_id
+        self.node_id_roles_dict = node_id_roles_dict
+        self.required_networks = []
+
+    def env_exists(self, env_name):
+        env_list = parse(exec_cmd('fuel env --list'))
+        for env in env_list:
+            if env[E['name']] == env_name and env[E['status']] == 'new':
+                self.env_id = env[E['id']]
+                return True
+        return False
+
+    def configure_environment(self):
+        log('Configure environment')
+        delete(self.yaml_config_dir)
+        create_dir_if_not_exists(self.yaml_config_dir)
+        env_name = self.dea.get_env_name()
+        env_mode = self.dea.get_env_mode()
+        env_net_segment_type = self.dea.get_env_net_segment_type()
+        log('Creating environment %s release %s, mode %s, network-mode neutron'
+            ', net-segment-type %s'
+            % (env_name, self.release_id, env_mode, env_net_segment_type))
+        exec_cmd('fuel env create --name %s --release %s --mode %s '
+                 '--network-mode neutron --net-segment-type %s'
+                 % (env_name, self.release_id, env_mode, env_net_segment_type))
+
+        if not self.env_exists(env_name):
+            err('Failed to create environment %s' % env_name)
+        self.config_settings()
+        self.config_network()
+        self.config_nodes()
+
+    def config_settings(self):
+        settings = ConfigureSettings(self.yaml_config_dir, self.env_id,
+                                     self.dea)
+        settings.config_settings()
+
+    def config_network(self):
+        network = ConfigureNetwork(self.yaml_config_dir, self.env_id, self.dea)
+        network.config_network()
+
+    def config_nodes(self):
+        nodes = ConfigureNodes(self.yaml_config_dir, self.env_id,
+                               self.node_id_roles_dict, self.dea)
+        nodes.config_nodes()
diff --git a/deploy/cloud/configure_network.py b/deploy/cloud/configure_network.py
new file mode 100644
index 000000000..b3ff9e9e0
--- /dev/null
+++ b/deploy/cloud/configure_network.py
@@ -0,0 +1,69 @@
+###############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# szilard.cserey@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+import yaml
+import io
+
+from common import (
+    exec_cmd,
+    check_file_exists,
+    log,
+    backup,
+)
+
+
+class ConfigureNetwork(object):
+
+    def __init__(self, yaml_config_dir, env_id, dea):
+        self.yaml_config_dir = yaml_config_dir
+        self.env_id = env_id
+        self.dea = dea
+        self.required_networks = []
+
+    def download_network_config(self):
+        log('Download network config for environment %s' % self.env_id)
+        exec_cmd('fuel network --env %s --download --dir %s'
+                 % (self.env_id, self.yaml_config_dir))
+
+    def upload_network_config(self):
+        log('Upload network config for environment %s' % self.env_id)
+        exec_cmd('fuel network --env %s --upload --dir %s'
+                 % (self.env_id, self.yaml_config_dir))
+
+    def config_network(self):
+        log('Configure network')
+        self.download_network_config()
+        self.modify_network_config()
+        self.upload_network_config()
+
+    def modify_network_config(self):
+        log('Modify network config for environment %s' % self.env_id)
+        network_yaml = ('%s/network_%s.yaml'
+                        % (self.yaml_config_dir, self.env_id))
+        check_file_exists(network_yaml)
+        backup(network_yaml)
+
+        network_config = self.dea.get_property('network')
+
+        with io.open(network_yaml) as stream:
+            network = yaml.load(stream)
+
+        net_names = self.dea.get_network_names()
+        net_id = {}
+        for net in network['networks']:
+            if net['name'] in net_names:
+                net_id[net['name']] = {'id': net['id'],
+                                       'group_id': net['group_id']}
+
+        for network in network_config['networks']:
+            network.update(net_id[network['name']])
+
+        with io.open(network_yaml, 'w') as stream:
+            yaml.dump(network_config, stream, default_flow_style=False)
diff --git a/deploy/cloud/configure_nodes.py b/deploy/cloud/configure_nodes.py
new file mode 100644
index 000000000..a456679ab
--- /dev/null
+++ b/deploy/cloud/configure_nodes.py
@@ -0,0 +1,109 @@
+###############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# szilard.cserey@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+import yaml
+import io
+import glob
+
+from common import (
+    exec_cmd,
+    check_file_exists,
+    log,
+    backup,
+)
+
+
+class ConfigureNodes(object):
+
+    def __init__(self, yaml_config_dir, env_id, node_id_roles_dict, dea):
+        self.yaml_config_dir = yaml_config_dir
+        self.env_id = env_id
+        self.node_id_roles_dict = node_id_roles_dict
+        self.dea = dea
+
+    def config_nodes(self):
+        log('Configure nodes')
+        for node_id, roles_blade in self.node_id_roles_dict.iteritems():
+            exec_cmd('fuel node set --node-id %s --role %s --env %s'
+                     % (node_id, roles_blade[0], self.env_id))
+
+        self.download_deployment_config()
+        for node_id, roles_blade in self.node_id_roles_dict.iteritems():
+            self.download_interface_config(node_id)
+            self.modify_node_interface(node_id, roles_blade)
+            self.modify_node_network_schemes(node_id, roles_blade)
+            self.upload_interface_config(node_id)
+        self.upload_deployment_config()
+
+    def modify_node_network_schemes(self, node_id, roles_blade):
+        log('Modify network transformations for node %s' % node_id)
+        type = self.dea.get_node_property(roles_blade[1], 'transformations')
+        transformations = self.dea.get_property(type)
+        deployment_dir = '%s/deployment_%s' % (
+            self.yaml_config_dir, self.env_id)
+        backup(deployment_dir)
+        for node_file in glob.glob(deployment_dir + '/*_%s.yaml' % node_id):
+            with io.open(node_file) as stream:
+                node = yaml.load(stream)
+
+            node['network_scheme'].update(transformations)
+
+            with io.open(node_file, 'w') as stream:
+                yaml.dump(node, stream, default_flow_style=False)
+
+    def download_deployment_config(self):
+        log('Download deployment config for environment %s' % self.env_id)
+        exec_cmd('fuel deployment --env %s --default --dir %s'
+                 % (self.env_id, self.yaml_config_dir))
+
+    def upload_deployment_config(self):
+        log('Upload deployment config for environment %s' % self.env_id)
+        exec_cmd('fuel deployment --env %s --upload --dir %s'
+                 % (self.env_id, self.yaml_config_dir))
+
+    def download_interface_config(self, node_id):
+        log('Download interface config for node %s' % node_id)
+        exec_cmd('fuel node --env %s --node %s --network --download '
+                 '--dir %s' % (self.env_id, node_id, self.yaml_config_dir))
+
+    def upload_interface_config(self, node_id):
+        log('Upload interface config for node %s' % node_id)
+        exec_cmd('fuel node --env %s --node %s --network --upload '
+                 '--dir %s' % (self.env_id, node_id, self.yaml_config_dir))
+
+    def modify_node_interface(self, node_id, roles_blade):
+        log('Modify interface config for node %s' % node_id)
+        interface_yaml = ('%s/node_%s/interfaces.yaml'
+                          % (self.yaml_config_dir, node_id))
+        check_file_exists(interface_yaml)
+        backup('%s/node_%s' % (self.yaml_config_dir, node_id))
+
+        with io.open(interface_yaml) as stream:
+            interfaces = yaml.load(stream)
+
+        net_name_id = {}
+        for interface in interfaces:
+            for network in interface['assigned_networks']:
+                net_name_id[network['name']] = network['id']
+
+        type = self.dea.get_node_property(roles_blade[1], 'interfaces')
+        interface_config = self.dea.get_property(type)
+
+        for interface in interfaces:
+            interface['assigned_networks'] = []
+            if interface['name'] in interface_config:
+                for net_name in interface_config[interface['name']]:
+                    net = {}
+                    net['id'] = net_name_id[net_name]
+                    net['name'] = net_name
+                    interface['assigned_networks'].append(net)
+
+        with io.open(interface_yaml, 'w') as stream:
+            yaml.dump(interfaces, stream, default_flow_style=False)
diff --git a/deploy/cloud/configure_settings.py b/deploy/cloud/configure_settings.py
new file mode 100644
index 000000000..c773e1b61
--- /dev/null
+++ b/deploy/cloud/configure_settings.py
@@ -0,0 +1,55 @@
+###############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# szilard.cserey@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+import yaml
+import io
+
+from common import (
+    exec_cmd,
+    check_file_exists,
+    log,
+    backup,
+)
+
+
+class ConfigureSettings(object):
+
+    def __init__(self, yaml_config_dir, env_id, dea):
+        self.yaml_config_dir = yaml_config_dir
+        self.env_id = env_id
+        self.dea = dea
+
+    def download_settings(self):
+        log('Download settings for environment %s' % self.env_id)
+        exec_cmd('fuel settings --env %s --download --dir %s'
+                 % (self.env_id, self.yaml_config_dir))
+
+    def upload_settings(self):
+        log('Upload settings for environment %s' % self.env_id)
+        exec_cmd('fuel settings --env %s --upload --dir %s'
+                 % (self.env_id, self.yaml_config_dir))
+
+    def config_settings(self):
+        log('Configure settings')
+        self.download_settings()
+        self.modify_settings()
+        self.upload_settings()
+
+    def modify_settings(self):
+        log('Modify settings for environment %s' % self.env_id)
+        settings_yaml = ('%s/settings_%s.yaml'
+                         % (self.yaml_config_dir, self.env_id))
+        check_file_exists(settings_yaml)
+        backup(settings_yaml)
+
+        settings = self.dea.get_property('settings')
+
+        with io.open(settings_yaml, 'w') as stream:
+            yaml.dump(settings, stream, default_flow_style=False)
diff --git a/deploy/cloud/deploy.py b/deploy/cloud/deploy.py
new file mode 100644
index 000000000..14d6a9621
--- /dev/null
+++ b/deploy/cloud/deploy.py
@@ -0,0 +1,94 @@
+###############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# szilard.cserey@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+import yaml
+import io
+
+from dea import DeploymentEnvironmentAdapter
+from configure_environment import ConfigureEnvironment
+from deployment import Deployment
+
+from common import (
+    R,
+    exec_cmd,
+    parse,
+    check_file_exists,
+    commafy,
+    ArgParser,
+)
+
+YAML_CONF_DIR = '/var/lib/opnfv'
+
+
+class Deploy(object):
+
+    def __init__(self, dea_file, no_health_check):
+        self.dea = DeploymentEnvironmentAdapter(dea_file)
+        self.no_health_check = no_health_check
+        self.macs_per_blade = {}
+        self.blades = self.dea.get_node_ids()
+        self.blade_node_dict = self.dea.get_blade_node_map()
+        self.node_roles_dict = {}
+        self.env_id = None
+        self.wanted_release = self.dea.get_property('wanted_release')
+
+    def assign_roles_to_cluster_node_ids(self):
+        self.node_roles_dict = {}
+        for blade, node in self.blade_node_dict.iteritems():
+            roles = commafy(self.dea.get_node_role(blade))
+            self.node_roles_dict[node] = (roles, blade)
+
+    def configure_environment(self):
+        release_list = parse(exec_cmd('fuel release -l'))
+        for release in release_list:
+            if release[R['name']] == self.wanted_release:
+                break
+        config_env = ConfigureEnvironment(self.dea, YAML_CONF_DIR,
+                                          release[R['id']],
+                                          self.node_roles_dict)
+        config_env.configure_environment()
+        self.env_id = config_env.env_id
+
+    def deploy_cloud(self):
+        dep = Deployment(self.dea, YAML_CONF_DIR, self.env_id,
+                         self.node_roles_dict, self.no_health_check)
+        dep.deploy()
+
+    def deploy(self):
+
+        self.assign_roles_to_cluster_node_ids()
+
+        self.configure_environment()
+
+        self.deploy_cloud()
+
+
+def parse_arguments():
+    parser = ArgParser(prog='python %s' % __file__)
+    parser.add_argument('-nh', dest='no_health_check', action='store_true',
+                        default=False,
+                        help='Don\'t run health check after deployment')
+    parser.add_argument('dea_file', action='store',
+                        help='Deployment Environment Adapter: dea.yaml')
+    args = parser.parse_args()
+    check_file_exists(args.dea_file)
+
+    kwargs = {'dea_file': args.dea_file,
+              'no_health_check': args.no_health_check}
+    return kwargs
+
+
+def main():
+    kwargs = parse_arguments()
+    deploy = Deploy(**kwargs)
+    deploy.deploy()
+
+if __name__ == '__main__':
+    main()
diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py
new file mode 100644
index 000000000..12c1f8978
--- /dev/null
+++ b/deploy/cloud/deployment.py
@@ -0,0 +1,156 @@
+###############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# szilard.cserey@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+import time
+import re
+
+from common import (
+    N,
+    E,
+    exec_cmd,
+    run_proc,
+    parse,
+    err,
+    log,
+    delete,
+)
+
+SEARCH_TEXT = 'Puppet (err)'
+LOG_FILE = '/var/log/puppet.log'
+GREP_LINES_OF_LEADING_CONTEXT = 100
+GREP_LINES_OF_TRAILING_CONTEXT = 100
+LIST_OF_CHAR_TO_BE_ESCAPED = ['[', ']', '"']
+
+class Deployment(object):
+
+    def __init__(self, dea, yaml_config_dir, env_id, node_id_roles_dict,
+                 no_health_check):
+        self.dea = dea
+        self.yaml_config_dir = yaml_config_dir
+        self.env_id = env_id
+        self.node_id_roles_dict = node_id_roles_dict
+        self.no_health_check = no_health_check
+        self.pattern = re.compile(
+            '\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d')
+
+    def collect_error_logs(self):
+        for node_id, roles_blade in self.node_id_roles_dict.iteritems():
+            log_list = []
+            cmd = ('ssh -q node-%s grep \'"%s"\' %s'
+                   % (node_id, SEARCH_TEXT, LOG_FILE))
+            results, _ = exec_cmd(cmd, False)
+            for result in results.splitlines():
+                log_msg = ''
+
+                sub_cmd = '"%s" %s' % (result, LOG_FILE)
+                for c in LIST_OF_CHAR_TO_BE_ESCAPED:
+                    sub_cmd = sub_cmd.replace(c, '\%s' % c)
+                grep_cmd = ('grep -B%s %s'
+                            % (GREP_LINES_OF_LEADING_CONTEXT, sub_cmd))
+                cmd = ('ssh -q node-%s "%s"' % (node_id, grep_cmd))
+
+                details, _ = exec_cmd(cmd, False)
+                details_list = details.splitlines()
+
+                found_prev_log = False
+                for i in range(len(details_list) - 2, -1, -1):
+                    if self.pattern.match(details_list[i]):
+                        found_prev_log = True
+                        break
+                if found_prev_log:
+                    log_msg += '\n'.join(details_list[i:-1]) + '\n'
+
+                grep_cmd = ('grep -A%s %s'
+                            % (GREP_LINES_OF_TRAILING_CONTEXT, sub_cmd))
+                cmd = ('ssh -q node-%s "%s"' % (node_id, grep_cmd))
+
+                details, _ = exec_cmd(cmd, False)
+                details_list = details.splitlines()
+
+                found_next_log = False
+                for i in range(1, len(details_list)):
+                    if self.pattern.match(details_list[i]):
+                        found_next_log = True
+                        break
+                if found_next_log:
+                    log_msg += '\n'.join(details_list[:i])
+                else:
+                    log_msg += details
+
+                if log_msg:
+                   log_list.append(log_msg)
+
+            if log_list:
+                role = ('controller' if 'controller' in roles_blade[0]
+                        else 'compute host')
+                log('_' * 40 + 'Errors in node-%s %s' % (node_id, role)
+                    + '_' * 40)
+                for log_msg in log_list:
+                    print(log_msg + '\n')
+
+    def run_deploy(self):
+        WAIT_LOOP = 180
+        SLEEP_TIME = 60
+        LOG_FILE = 'cloud.log'
+
+        log('Starting deployment of environment %s' % self.env_id)
+        run_proc('fuel --env %s deploy-changes | strings | tee %s'
+                 % (self.env_id, LOG_FILE))
+
+        ready = False
+        for i in range(WAIT_LOOP):
+            env = parse(exec_cmd('fuel env --env %s' % self.env_id))
+            log('Environment status: %s' % env[0][E['status']])
+            r, _ = exec_cmd('tail -2 %s | head -1' % LOG_FILE, False)
+            if r:
+                log(r)
+            if env[0][E['status']] == 'operational':
+                ready = True
+                break
+            elif (env[0][E['status']] == 'error'
+                  or env[0][E['status']] == 'stopped'):
+                break
+            else:
+                time.sleep(SLEEP_TIME)
+        delete(LOG_FILE)
+
+        if ready:
+            log('Environment %s successfully deployed' % self.env_id)
+        else:
+            self.collect_error_logs()
+            err('Deployment failed, environment %s is not operational'
+                % self.env_id)
+
+    def verify_node_status(self):
+        node_list = parse(exec_cmd('fuel node list'))
+        failed_nodes = []
+        for node in node_list:
+            if node[N['status']] != 'ready':
+                failed_nodes.append((node[N['id']], node[N['status']]))
+
+        if failed_nodes:
+            summary = ''
+            for node, status in failed_nodes:
+                summary += '[node %s, status %s]\n' % (node, status)
+            err('Deployment failed: %s' % summary)
+
+    def health_check(self):
+        log('Now running sanity and smoke health checks')
+        r = exec_cmd('fuel health --env %s --check sanity,smoke --force'
+                     % self.env_id)
+        log(r)
+        if 'failure' in r:
+            err('Healthcheck failed!')
+
+    def deploy(self):
+        self.run_deploy()
+        self.verify_node_status()
+        if not self.no_health_check:
+            self.health_check()