summaryrefslogtreecommitdiffstats
path: root/deploy/cloud
diff options
context:
space:
mode:
authorJonas Bjurel <jonas.bjurel@ericsson.com>2015-11-25 11:32:57 +0100
committerJonas Bjurel <jonas.bjurel@ericsson.com>2015-11-27 10:24:23 +0100
commit24a95306d2564b272b5320e9149d9aea70b4061c (patch)
tree8c1a2c9f7acbfed41f8ebc56a8ae1f7d316f7cff /deploy/cloud
parent7077376d6e0ff0dec77080fa21b75911b811475d (diff)
Restructcture of the directory layout
Restructure of the directory layout due to move of Fuel into it's own repo JIRA: FUEL-85 Change-Id: I3647e1992a508f29dce06a5d6c790725c527f6f5 Signed-off-by: Jonas Bjurel <jonas.bjurel@ericsson.com>
Diffstat (limited to 'deploy/cloud')
-rw-r--r--deploy/cloud/configure_environment.py76
-rw-r--r--deploy/cloud/configure_network.py69
-rw-r--r--deploy/cloud/configure_nodes.py109
-rw-r--r--deploy/cloud/configure_settings.py55
-rw-r--r--deploy/cloud/deploy.py94
-rw-r--r--deploy/cloud/deployment.py156
6 files changed, 559 insertions, 0 deletions
diff --git a/deploy/cloud/configure_environment.py b/deploy/cloud/configure_environment.py
new file mode 100644
index 000000000..ed89fa600
--- /dev/null
+++ b/deploy/cloud/configure_environment.py
@@ -0,0 +1,76 @@
+###############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# szilard.cserey@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+from configure_settings import ConfigureSettings
+from configure_network import ConfigureNetwork
+from configure_nodes import ConfigureNodes
+
+from common import (
+ E,
+ exec_cmd,
+ parse,
+ err,
+ log,
+ delete,
+ create_dir_if_not_exists,
+)
+
+
+class ConfigureEnvironment(object):
+
+ def __init__(self, dea, yaml_config_dir, release_id, node_id_roles_dict):
+ self.env_id = None
+ self.dea = dea
+ self.yaml_config_dir = yaml_config_dir
+ self.release_id = release_id
+ self.node_id_roles_dict = node_id_roles_dict
+ self.required_networks = []
+
+ def env_exists(self, env_name):
+ env_list = parse(exec_cmd('fuel env --list'))
+ for env in env_list:
+ if env[E['name']] == env_name and env[E['status']] == 'new':
+ self.env_id = env[E['id']]
+ return True
+ return False
+
+ def configure_environment(self):
+ log('Configure environment')
+ delete(self.yaml_config_dir)
+ create_dir_if_not_exists(self.yaml_config_dir)
+ env_name = self.dea.get_env_name()
+ env_mode = self.dea.get_env_mode()
+ env_net_segment_type = self.dea.get_env_net_segment_type()
+ log('Creating environment %s release %s, mode %s, network-mode neutron'
+ ', net-segment-type %s'
+ % (env_name, self.release_id, env_mode, env_net_segment_type))
+ exec_cmd('fuel env create --name %s --release %s --mode %s '
+ '--network-mode neutron --net-segment-type %s'
+ % (env_name, self.release_id, env_mode, env_net_segment_type))
+
+ if not self.env_exists(env_name):
+ err('Failed to create environment %s' % env_name)
+ self.config_settings()
+ self.config_network()
+ self.config_nodes()
+
+ def config_settings(self):
+ settings = ConfigureSettings(self.yaml_config_dir, self.env_id,
+ self.dea)
+ settings.config_settings()
+
+ def config_network(self):
+ network = ConfigureNetwork(self.yaml_config_dir, self.env_id, self.dea)
+ network.config_network()
+
+ def config_nodes(self):
+ nodes = ConfigureNodes(self.yaml_config_dir, self.env_id,
+ self.node_id_roles_dict, self.dea)
+ nodes.config_nodes()
diff --git a/deploy/cloud/configure_network.py b/deploy/cloud/configure_network.py
new file mode 100644
index 000000000..b3ff9e9e0
--- /dev/null
+++ b/deploy/cloud/configure_network.py
@@ -0,0 +1,69 @@
+###############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# szilard.cserey@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+import yaml
+import io
+
+from common import (
+ exec_cmd,
+ check_file_exists,
+ log,
+ backup,
+)
+
+
+class ConfigureNetwork(object):
+
+ def __init__(self, yaml_config_dir, env_id, dea):
+ self.yaml_config_dir = yaml_config_dir
+ self.env_id = env_id
+ self.dea = dea
+ self.required_networks = []
+
+ def download_network_config(self):
+ log('Download network config for environment %s' % self.env_id)
+ exec_cmd('fuel network --env %s --download --dir %s'
+ % (self.env_id, self.yaml_config_dir))
+
+ def upload_network_config(self):
+ log('Upload network config for environment %s' % self.env_id)
+ exec_cmd('fuel network --env %s --upload --dir %s'
+ % (self.env_id, self.yaml_config_dir))
+
+ def config_network(self):
+ log('Configure network')
+ self.download_network_config()
+ self.modify_network_config()
+ self.upload_network_config()
+
+ def modify_network_config(self):
+ log('Modify network config for environment %s' % self.env_id)
+ network_yaml = ('%s/network_%s.yaml'
+ % (self.yaml_config_dir, self.env_id))
+ check_file_exists(network_yaml)
+ backup(network_yaml)
+
+ network_config = self.dea.get_property('network')
+
+ with io.open(network_yaml) as stream:
+ network = yaml.load(stream)
+
+ net_names = self.dea.get_network_names()
+ net_id = {}
+ for net in network['networks']:
+ if net['name'] in net_names:
+ net_id[net['name']] = {'id': net['id'],
+ 'group_id': net['group_id']}
+
+ for network in network_config['networks']:
+ network.update(net_id[network['name']])
+
+ with io.open(network_yaml, 'w') as stream:
+ yaml.dump(network_config, stream, default_flow_style=False)
diff --git a/deploy/cloud/configure_nodes.py b/deploy/cloud/configure_nodes.py
new file mode 100644
index 000000000..a456679ab
--- /dev/null
+++ b/deploy/cloud/configure_nodes.py
@@ -0,0 +1,109 @@
+###############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# szilard.cserey@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+import yaml
+import io
+import glob
+
+from common import (
+ exec_cmd,
+ check_file_exists,
+ log,
+ backup,
+)
+
+
+class ConfigureNodes(object):
+
+ def __init__(self, yaml_config_dir, env_id, node_id_roles_dict, dea):
+ self.yaml_config_dir = yaml_config_dir
+ self.env_id = env_id
+ self.node_id_roles_dict = node_id_roles_dict
+ self.dea = dea
+
+ def config_nodes(self):
+ log('Configure nodes')
+ for node_id, roles_blade in self.node_id_roles_dict.iteritems():
+ exec_cmd('fuel node set --node-id %s --role %s --env %s'
+ % (node_id, roles_blade[0], self.env_id))
+
+ self.download_deployment_config()
+ for node_id, roles_blade in self.node_id_roles_dict.iteritems():
+ self.download_interface_config(node_id)
+ self.modify_node_interface(node_id, roles_blade)
+ self.modify_node_network_schemes(node_id, roles_blade)
+ self.upload_interface_config(node_id)
+ self.upload_deployment_config()
+
+ def modify_node_network_schemes(self, node_id, roles_blade):
+ log('Modify network transformations for node %s' % node_id)
+ type = self.dea.get_node_property(roles_blade[1], 'transformations')
+ transformations = self.dea.get_property(type)
+ deployment_dir = '%s/deployment_%s' % (
+ self.yaml_config_dir, self.env_id)
+ backup(deployment_dir)
+ for node_file in glob.glob(deployment_dir + '/*_%s.yaml' % node_id):
+ with io.open(node_file) as stream:
+ node = yaml.load(stream)
+
+ node['network_scheme'].update(transformations)
+
+ with io.open(node_file, 'w') as stream:
+ yaml.dump(node, stream, default_flow_style=False)
+
+ def download_deployment_config(self):
+ log('Download deployment config for environment %s' % self.env_id)
+ exec_cmd('fuel deployment --env %s --default --dir %s'
+ % (self.env_id, self.yaml_config_dir))
+
+ def upload_deployment_config(self):
+ log('Upload deployment config for environment %s' % self.env_id)
+ exec_cmd('fuel deployment --env %s --upload --dir %s'
+ % (self.env_id, self.yaml_config_dir))
+
+ def download_interface_config(self, node_id):
+ log('Download interface config for node %s' % node_id)
+ exec_cmd('fuel node --env %s --node %s --network --download '
+ '--dir %s' % (self.env_id, node_id, self.yaml_config_dir))
+
+ def upload_interface_config(self, node_id):
+ log('Upload interface config for node %s' % node_id)
+ exec_cmd('fuel node --env %s --node %s --network --upload '
+ '--dir %s' % (self.env_id, node_id, self.yaml_config_dir))
+
+ def modify_node_interface(self, node_id, roles_blade):
+ log('Modify interface config for node %s' % node_id)
+ interface_yaml = ('%s/node_%s/interfaces.yaml'
+ % (self.yaml_config_dir, node_id))
+ check_file_exists(interface_yaml)
+ backup('%s/node_%s' % (self.yaml_config_dir, node_id))
+
+ with io.open(interface_yaml) as stream:
+ interfaces = yaml.load(stream)
+
+ net_name_id = {}
+ for interface in interfaces:
+ for network in interface['assigned_networks']:
+ net_name_id[network['name']] = network['id']
+
+ type = self.dea.get_node_property(roles_blade[1], 'interfaces')
+ interface_config = self.dea.get_property(type)
+
+ for interface in interfaces:
+ interface['assigned_networks'] = []
+ if interface['name'] in interface_config:
+ for net_name in interface_config[interface['name']]:
+ net = {}
+ net['id'] = net_name_id[net_name]
+ net['name'] = net_name
+ interface['assigned_networks'].append(net)
+
+ with io.open(interface_yaml, 'w') as stream:
+ yaml.dump(interfaces, stream, default_flow_style=False)
diff --git a/deploy/cloud/configure_settings.py b/deploy/cloud/configure_settings.py
new file mode 100644
index 000000000..c773e1b61
--- /dev/null
+++ b/deploy/cloud/configure_settings.py
@@ -0,0 +1,55 @@
+###############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# szilard.cserey@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+import yaml
+import io
+
+from common import (
+ exec_cmd,
+ check_file_exists,
+ log,
+ backup,
+)
+
+
+class ConfigureSettings(object):
+
+ def __init__(self, yaml_config_dir, env_id, dea):
+ self.yaml_config_dir = yaml_config_dir
+ self.env_id = env_id
+ self.dea = dea
+
+ def download_settings(self):
+ log('Download settings for environment %s' % self.env_id)
+ exec_cmd('fuel settings --env %s --download --dir %s'
+ % (self.env_id, self.yaml_config_dir))
+
+ def upload_settings(self):
+ log('Upload settings for environment %s' % self.env_id)
+ exec_cmd('fuel settings --env %s --upload --dir %s'
+ % (self.env_id, self.yaml_config_dir))
+
+ def config_settings(self):
+ log('Configure settings')
+ self.download_settings()
+ self.modify_settings()
+ self.upload_settings()
+
+ def modify_settings(self):
+ log('Modify settings for environment %s' % self.env_id)
+ settings_yaml = ('%s/settings_%s.yaml'
+ % (self.yaml_config_dir, self.env_id))
+ check_file_exists(settings_yaml)
+ backup(settings_yaml)
+
+ settings = self.dea.get_property('settings')
+
+ with io.open(settings_yaml, 'w') as stream:
+ yaml.dump(settings, stream, default_flow_style=False)
diff --git a/deploy/cloud/deploy.py b/deploy/cloud/deploy.py
new file mode 100644
index 000000000..14d6a9621
--- /dev/null
+++ b/deploy/cloud/deploy.py
@@ -0,0 +1,94 @@
+###############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# szilard.cserey@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+import yaml
+import io
+
+from dea import DeploymentEnvironmentAdapter
+from configure_environment import ConfigureEnvironment
+from deployment import Deployment
+
+from common import (
+ R,
+ exec_cmd,
+ parse,
+ check_file_exists,
+ commafy,
+ ArgParser,
+)
+
+YAML_CONF_DIR = '/var/lib/opnfv'
+
+
+class Deploy(object):
+
+ def __init__(self, dea_file, no_health_check):
+ self.dea = DeploymentEnvironmentAdapter(dea_file)
+ self.no_health_check = no_health_check
+ self.macs_per_blade = {}
+ self.blades = self.dea.get_node_ids()
+ self.blade_node_dict = self.dea.get_blade_node_map()
+ self.node_roles_dict = {}
+ self.env_id = None
+ self.wanted_release = self.dea.get_property('wanted_release')
+
+ def assign_roles_to_cluster_node_ids(self):
+ self.node_roles_dict = {}
+ for blade, node in self.blade_node_dict.iteritems():
+ roles = commafy(self.dea.get_node_role(blade))
+ self.node_roles_dict[node] = (roles, blade)
+
+ def configure_environment(self):
+ release_list = parse(exec_cmd('fuel release -l'))
+ for release in release_list:
+ if release[R['name']] == self.wanted_release:
+ break
+ config_env = ConfigureEnvironment(self.dea, YAML_CONF_DIR,
+ release[R['id']],
+ self.node_roles_dict)
+ config_env.configure_environment()
+ self.env_id = config_env.env_id
+
+ def deploy_cloud(self):
+ dep = Deployment(self.dea, YAML_CONF_DIR, self.env_id,
+ self.node_roles_dict, self.no_health_check)
+ dep.deploy()
+
+ def deploy(self):
+
+ self.assign_roles_to_cluster_node_ids()
+
+ self.configure_environment()
+
+ self.deploy_cloud()
+
+
+def parse_arguments():
+ parser = ArgParser(prog='python %s' % __file__)
+ parser.add_argument('-nh', dest='no_health_check', action='store_true',
+ default=False,
+ help='Don\'t run health check after deployment')
+ parser.add_argument('dea_file', action='store',
+ help='Deployment Environment Adapter: dea.yaml')
+ args = parser.parse_args()
+ check_file_exists(args.dea_file)
+
+ kwargs = {'dea_file': args.dea_file,
+ 'no_health_check': args.no_health_check}
+ return kwargs
+
+
+def main():
+ kwargs = parse_arguments()
+ deploy = Deploy(**kwargs)
+ deploy.deploy()
+
+if __name__ == '__main__':
+ main()
diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py
new file mode 100644
index 000000000..12c1f8978
--- /dev/null
+++ b/deploy/cloud/deployment.py
@@ -0,0 +1,156 @@
+###############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# szilard.cserey@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+import time
+import re
+
+from common import (
+ N,
+ E,
+ exec_cmd,
+ run_proc,
+ parse,
+ err,
+ log,
+ delete,
+)
+
+SEARCH_TEXT = 'Puppet (err)'
+LOG_FILE = '/var/log/puppet.log'
+GREP_LINES_OF_LEADING_CONTEXT = 100
+GREP_LINES_OF_TRAILING_CONTEXT = 100
+LIST_OF_CHAR_TO_BE_ESCAPED = ['[', ']', '"']
+
+class Deployment(object):
+
+ def __init__(self, dea, yaml_config_dir, env_id, node_id_roles_dict,
+ no_health_check):
+ self.dea = dea
+ self.yaml_config_dir = yaml_config_dir
+ self.env_id = env_id
+ self.node_id_roles_dict = node_id_roles_dict
+ self.no_health_check = no_health_check
+ self.pattern = re.compile(
+ '\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d')
+
+ def collect_error_logs(self):
+ for node_id, roles_blade in self.node_id_roles_dict.iteritems():
+ log_list = []
+ cmd = ('ssh -q node-%s grep \'"%s"\' %s'
+ % (node_id, SEARCH_TEXT, LOG_FILE))
+ results, _ = exec_cmd(cmd, False)
+ for result in results.splitlines():
+ log_msg = ''
+
+ sub_cmd = '"%s" %s' % (result, LOG_FILE)
+ for c in LIST_OF_CHAR_TO_BE_ESCAPED:
+ sub_cmd = sub_cmd.replace(c, '\%s' % c)
+ grep_cmd = ('grep -B%s %s'
+ % (GREP_LINES_OF_LEADING_CONTEXT, sub_cmd))
+ cmd = ('ssh -q node-%s "%s"' % (node_id, grep_cmd))
+
+ details, _ = exec_cmd(cmd, False)
+ details_list = details.splitlines()
+
+ found_prev_log = False
+ for i in range(len(details_list) - 2, -1, -1):
+ if self.pattern.match(details_list[i]):
+ found_prev_log = True
+ break
+ if found_prev_log:
+ log_msg += '\n'.join(details_list[i:-1]) + '\n'
+
+ grep_cmd = ('grep -A%s %s'
+ % (GREP_LINES_OF_TRAILING_CONTEXT, sub_cmd))
+ cmd = ('ssh -q node-%s "%s"' % (node_id, grep_cmd))
+
+ details, _ = exec_cmd(cmd, False)
+ details_list = details.splitlines()
+
+ found_next_log = False
+ for i in range(1, len(details_list)):
+ if self.pattern.match(details_list[i]):
+ found_next_log = True
+ break
+ if found_next_log:
+ log_msg += '\n'.join(details_list[:i])
+ else:
+ log_msg += details
+
+ if log_msg:
+ log_list.append(log_msg)
+
+ if log_list:
+ role = ('controller' if 'controller' in roles_blade[0]
+ else 'compute host')
+ log('_' * 40 + 'Errors in node-%s %s' % (node_id, role)
+ + '_' * 40)
+ for log_msg in log_list:
+ print(log_msg + '\n')
+
+ def run_deploy(self):
+ WAIT_LOOP = 180
+ SLEEP_TIME = 60
+ LOG_FILE = 'cloud.log'
+
+ log('Starting deployment of environment %s' % self.env_id)
+ run_proc('fuel --env %s deploy-changes | strings | tee %s'
+ % (self.env_id, LOG_FILE))
+
+ ready = False
+ for i in range(WAIT_LOOP):
+ env = parse(exec_cmd('fuel env --env %s' % self.env_id))
+ log('Environment status: %s' % env[0][E['status']])
+ r, _ = exec_cmd('tail -2 %s | head -1' % LOG_FILE, False)
+ if r:
+ log(r)
+ if env[0][E['status']] == 'operational':
+ ready = True
+ break
+ elif (env[0][E['status']] == 'error'
+ or env[0][E['status']] == 'stopped'):
+ break
+ else:
+ time.sleep(SLEEP_TIME)
+ delete(LOG_FILE)
+
+ if ready:
+ log('Environment %s successfully deployed' % self.env_id)
+ else:
+ self.collect_error_logs()
+ err('Deployment failed, environment %s is not operational'
+ % self.env_id)
+
+ def verify_node_status(self):
+ node_list = parse(exec_cmd('fuel node list'))
+ failed_nodes = []
+ for node in node_list:
+ if node[N['status']] != 'ready':
+ failed_nodes.append((node[N['id']], node[N['status']]))
+
+ if failed_nodes:
+ summary = ''
+ for node, status in failed_nodes:
+ summary += '[node %s, status %s]\n' % (node, status)
+ err('Deployment failed: %s' % summary)
+
+ def health_check(self):
+ log('Now running sanity and smoke health checks')
+ r = exec_cmd('fuel health --env %s --check sanity,smoke --force'
+ % self.env_id)
+ log(r)
+ if 'failure' in r:
+ err('Healthcheck failed!')
+
+ def deploy(self):
+ self.run_deploy()
+ self.verify_node_status()
+ if not self.no_health_check:
+ self.health_check()