Add a expansion functionality

Through modify the virtual_cluster_expansion.yml, include
host's name and mac, you can decide to how many compute
nodes you need to add.

And you also need to modify network.yml. Note that external
subnet's ip_range should be changed as the first 6 IPs are
already taken by the first deployment.

Edit ``add.sh``, check the environment variable. Note that
the OS version and OpenStack version should be same as the
first deployment.

Run ``add.sh``.

JIRA:COMPASS-481

Change-Id: Id85f02518667e0ff80c2475e70856cd30cf1b9b7
Signed-off-by: liyuenan <liyuenan@huawei.com>

9 files changed, 161 insertions, 60 deletions

diff --git a/deploy/client.py b/deploy/client.py
index e5623c8b..f6a07a77 100644
--- a/deploy/client.py
+++ b/deploy/client.py
@@ -49,6 +49,9 @@ def byteify(input):
         return input
 
 opts = [
+    cfg.StrOpt('expansion',
+              help='is this an expansion?',
+              default='false'),
     cfg.StrOpt('compass_server',
               help='compass server url',
               default='http://127.0.0.1/api'),
@@ -365,19 +368,24 @@ class CompassClient(object):
         assert(subnets)
 
         subnet_mapping = {}
+        _, subnets_in_db = self.client.list_subnets()
         for subnet in subnets:
             try:
                 netaddr.IPNetwork(subnet)
             except:
                 raise RuntimeError('subnet %s format is invalid' % subnet)
 
-            status, resp = self.client.add_subnet(subnet)
-            LOG.info('add subnet %s status %s response %s',
-                         subnet, status, resp)
-            if not self.is_ok(status):
-                raise RuntimeError('failed to add subnet %s' % subnet)
-
-            subnet_mapping[resp['subnet']] = resp['id']
+            if CONF.expansion == "false":
+                status, resp = self.client.add_subnet(subnet)
+                LOG.info('add subnet %s status %s response %s',
+                             subnet, status, resp)
+                if not self.is_ok(status):
+                    raise RuntimeError('failed to add subnet %s' % subnet)
+                subnet_mapping[resp['subnet']] = resp['id']
+            else:
+                for subnet_in_db in subnets_in_db:
+                    if subnet == subnet_in_db['subnet']:
+                        subnet_mapping[subnet] = subnet_in_db['id']
 
         self.subnet_mapping = subnet_mapping
 
@@ -418,6 +426,7 @@ class CompassClient(object):
             if hostname
         ]
 
+        machines = machines[-len(hostnames):]
         assert(len(machines) == len(hostnames))
 
         machines_dict = []
@@ -439,9 +448,11 @@ class CompassClient(object):
             raise RuntimeError("add host to cluster failed")
 
         for host in resp['hosts']:
-            self.host_mapping[host['hostname']] = host['id']
+            if host['hostname'] in hostnames:
+                self.host_mapping[host['hostname']] = host['id']
 
-        assert(len(self.host_mapping) == len(machines))
+        if CONF.expansion == "false":
+            assert(len(self.host_mapping) == len(machines))
 
     def set_cluster_os_config(self, cluster_id):
         """set cluster os config."""
@@ -620,7 +631,7 @@ class CompassClient(object):
                 ])
 
                 LOG.info(
-                    'add host %s interface %s ip %s network proprties %s',
+                    'add host %s interface %s ip %s network properties %s',
                     hostname, interface, ip_str, properties)
 
                 status, response = self.client.add_host_network(
@@ -932,29 +943,56 @@ def kill_print_proc():
     os.system("ps aux|grep -v grep|grep -E 'ssh.+root@192.168.200.2'|awk '{print $2}'|xargs kill -9")
 
 def deploy():
-    client = CompassClient()
-    machines = client.get_machines()
+    if CONF.expansion == "false":
+        client = CompassClient()
+        machines = client.get_machines()
 
-    LOG.info('machines are %s', machines)
+        LOG.info('machines are %s', machines)
 
-    client.add_subnets()
-    adapter_id, os_id, flavor_id = client.get_adapter()
-    cluster_id = client.add_cluster(adapter_id, os_id, flavor_id)
+        client.add_subnets()
+        adapter_id, os_id, flavor_id = client.get_adapter()
+        cluster_id = client.add_cluster(adapter_id, os_id, flavor_id)
 
-    client.add_cluster_hosts(cluster_id, machines)
-    client.set_host_networking()
-    client.set_cluster_os_config(cluster_id)
+        client.add_cluster_hosts(cluster_id, machines)
+        client.set_host_networking()
+        client.set_cluster_os_config(cluster_id)
+
+        if flavor_id:
+            client.set_cluster_package_config(cluster_id)
+
+        client.set_all_hosts_roles(cluster_id)
+        client.deploy_clusters(cluster_id)
+
+        LOG.info("compass OS installtion is begin")
+        threading.Thread(target=print_ansible_log).start()
+        client.get_installing_progress(cluster_id)
+        client.check_dashboard_links(cluster_id)
+
+    else:
+        client = CompassClient()
+        machines = client.get_machines()
+
+        LOG.info('machines are %s', machines)
+
+        client.add_subnets()
+
+        status, response = client.client.list_clusters()
+        cluster_id = 1
+        for cluster in response:
+            if cluster['name'] == CONF.cluster_name:
+                cluster_id = cluster['id']
+
+        client.add_cluster_hosts(cluster_id, machines)
+        client.set_host_networking()
+        client.set_cluster_os_config(cluster_id)
 
-    if flavor_id:
         client.set_cluster_package_config(cluster_id)
 
-    client.set_all_hosts_roles(cluster_id)
-    client.deploy_clusters(cluster_id)
+        client.set_all_hosts_roles(cluster_id)
+        client.deploy_clusters(cluster_id)
 
-    LOG.info("compass OS installtion is begin")
-    threading.Thread(target=print_ansible_log).start()
-    client.get_installing_progress(cluster_id)
-    client.check_dashboard_links(cluster_id)
+        threading.Thread(target=print_ansible_log).start()
+        client.get_installing_progress(cluster_id)
 
 def redeploy():
     client = CompassClient()
diff --git a/deploy/conf/base.conf b/deploy/conf/base.conf
index 24eb703d..d60e68b1 100644
--- a/deploy/conf/base.conf
+++ b/deploy/conf/base.conf
@@ -25,6 +25,8 @@ export ENABLE_SECGROUP=${ENABLE_SECGROUP:-"true"}
 export ENABLE_VPNAAS="false"
 export ENABLE_FWAAS="false"
 
+export EXPANSION=${EXPANSION:-"false"}
+
 function next_ip {
     ip_addr=$1
     ip_base="$(echo $ip_addr | cut -d. -f'1 2 3')"
diff --git a/deploy/conf/hardware_environment/expansion-sample/hardware_cluster_expansion.yml b/deploy/conf/hardware_environment/expansion-sample/hardware_cluster_expansion.yml
new file mode 100644
index 00000000..b160b32a
--- /dev/null
+++ b/deploy/conf/hardware_environment/expansion-sample/hardware_cluster_expansion.yml
@@ -0,0 +1,15 @@
+TYPE: baremetal
+FLAVOR: cluster
+POWER_TOOL: ipmitool
+
+ipmiUser: root
+ipmiPass: Huawei@123
+
+hosts:
+  - name: host6
+     mac: 'E8:4D:D0:BA:60:45'
+     interfaces:
+        - eth1: '08:4D:D0:BA:60:44'
+     ipmiIp: 172.16.131.23
+     roles:
+       - compute
diff --git a/deploy/conf/virtual.conf b/deploy/conf/virtual.conf
index 0475ee9b..7f4fcf02 100644
--- a/deploy/conf/virtual.conf
+++ b/deploy/conf/virtual.conf
@@ -1,7 +1,7 @@
-export VIRT_NUMBER=5
-export VIRT_CPUS=4
-export VIRT_MEM=16384
-export VIRT_DISK=200G
+export VIRT_NUMBER=${VIRT_NUMBER:-5}
+export VIRT_CPUS=${VIRT_CPU:-4}
+export VIRT_MEM=${VIRT_MEM:-16384}
+export VIRT_DISK=${VIRT_DISK:-200G}
 
 export SWITCH_IPS="1.1.1.1"
 export SWITCH_CREDENTIAL="version=2c,community=public"
diff --git a/deploy/conf/vm_environment/virtual_cluster_expansion.yml b/deploy/conf/vm_environment/virtual_cluster_expansion.yml
new file mode 100644
index 00000000..3d3298d3
--- /dev/null
+++ b/deploy/conf/vm_environment/virtual_cluster_expansion.yml
@@ -0,0 +1,7 @@
+TYPE: virtual
+FLAVOR: cluster
+
+hosts:
+  - name: host6
+    roles:
+      - compute
diff --git a/deploy/deploy_host.sh b/deploy/deploy_host.sh
index b38f4209..031b56a2 100755
--- a/deploy/deploy_host.sh
+++ b/deploy/deploy_host.sh
@@ -44,7 +44,7 @@ function deploy_host(){
     --machines=${machines//\'} --switch_credential="${SWITCH_CREDENTIAL}" --deploy_type="${TYPE}" \
     --deployment_timeout="${DEPLOYMENT_TIMEOUT}" --${POLL_SWITCHES_FLAG} --dashboard_url="${DASHBOARD_URL}" \
     --cluster_vip="${VIP}" --network_cfg="$NETWORK" --neutron_cfg="$NEUTRON" \
-    --enable_secgroup="${ENABLE_SECGROUP}" --enable_fwaas="${ENABLE_FWAAS}" \
+    --enable_secgroup="${ENABLE_SECGROUP}" --enable_fwaas="${ENABLE_FWAAS}" --expansion="${EXPANSION}" \
     --rsa_file="$rsa_file" --enable_vpnaas="${ENABLE_VPNAAS}" --odl_l3_agent="${odl_l3_agent}" --moon="${moon}" --onos_sfc="${onos_sfc}"
 
     RET=$?
diff --git a/deploy/host_baremetal.sh b/deploy/host_baremetal.sh
index fc02875f..4c63f823 100755
--- a/deploy/host_baremetal.sh
+++ b/deploy/host_baremetal.sh
@@ -15,6 +15,14 @@ function reboot_hosts() {
 }
 
 function get_host_macs() {
-    machines=`echo $HOST_MACS | sed -e 's/,/'\',\''/g' -e 's/^/'\''/g' -e 's/$/'\''/g'`
+    if [[ "$EXPANSION" == "false" ]]; then
+        machines=`echo $HOST_MACS | sed -e 's/,/'\',\''/g' -e 's/^/'\''/g' -e 's/$/'\''/g'`
+        echo $machines > $WORK_DIR/switch_machines
+    else
+        machines_old=`cat $WORK_DIR/switch_machines`
+        machines_add=`echo $HOST_MACS | sed -e 's/,/'\',\''/g' -e 's/^/'\''/g' -e 's/$/'\''/g'`
+        echo $machines_add $machines_old > $WORK_DIR/switch_machines
+        machines=`echo $machines_add $machines_old|sed 's/ /,/g'`
+    fi
     echo $machines
 }
diff --git a/deploy/host_virtual.sh b/deploy/host_virtual.sh
index 97896bf0..2fab2c9d 100755
--- a/deploy/host_virtual.sh
+++ b/deploy/host_virtual.sh
@@ -60,14 +60,32 @@ function get_host_macs() {
 
     if [[ $REDEPLOY_HOST == "true" ]]; then
         mac_array=`cat $WORK_DIR/switch_machines`
+        machines=`echo $mac_array|sed 's/ /,/g'`
     else
-        chmod +x $mac_generator
-        mac_array=`$mac_generator $VIRT_NUMBER`
-        echo $mac_array > $WORK_DIR/switch_machines
+        if [[ -z $HOST_MACS ]]; then
+            if [[ "$EXPANSION" == "false" ]]; then
+                chmod +x $mac_generator
+                mac_array=`$mac_generator $VIRT_NUMBER`
+                echo $mac_array > $WORK_DIR/switch_machines
+                machines=`echo $mac_array|sed 's/ /,/g'`
+            else
+                machines_old=`cat $WORK_DIR/switch_machines`
+                chmod +x $mac_generator
+                machines_add=`$mac_generator $VIRT_NUMBER`
+                echo $machines_add $machines_old > $WORK_DIR/switch_machines
+                machines=`echo $machines_add $machines_old|sed 's/ /,/g'`
+            fi
+        else
+            if [[ "$EXPANSION" == "false" ]]; then
+                machines=`echo $HOST_MACS | sed -e 's/,/'\',\''/g' -e 's/^/'\''/g' -e 's/$/'\''/g'`
+            else
+                machines_old=`cat $WORK_DIR/switch_machines`
+                machines_add=`echo $HOST_MACS | sed -e 's/,/'\',\''/g' -e 's/^/'\''/g' -e 's/$/'\''/g'`
+                echo $machines_add $machines_old > $WORK_DIR/switch_machines
+                machines=`echo $machines_add $machines_old|sed 's/ /,/g'`
+            fi
+        fi
     fi
-
-    machines=`echo $mac_array|sed 's/ /,/g'`
-
     echo $machines
 }
 
diff --git a/deploy/launch.sh b/deploy/launch.sh
index 488e0fd7..ed65f40d 100755
--- a/deploy/launch.sh
+++ b/deploy/launch.sh
@@ -30,39 +30,52 @@ source ${COMPASS_DIR}/deploy/compass_vm.sh
 source ${COMPASS_DIR}/deploy/deploy_host.sh
 
 ######################### main process
-print_logo
+if [[ "$EXPANSION" == "false" ]]
+then
 
-if [[ ! -z $VIRT_NUMBER ]];then
-    tear_down_machines
-fi
-
-log_info "########## get host mac begin #############"
-machines=`get_host_macs`
-if [[ -z $machines ]]; then
-    log_error "get_host_macs failed"
-    exit 1
-fi
+    print_logo
 
-export machines
-
-if [[ "$DEPLOY_COMPASS" == "true" ]]; then
-    if ! prepare_env;then
-        echo "prepare_env failed"
-        exit 1
+    if [[ ! -z $VIRT_NUMBER ]];then
+        tear_down_machines
     fi
 
-    log_info "########## set up network begin #############"
-    if ! create_nets;then
-        log_error "create_nets failed"
+    log_info "########## get host mac begin #############"
+    machines=`get_host_macs`
+    if [[ -z $machines ]]; then
+        log_error "get_host_macs failed"
         exit 1
     fi
 
-    if ! launch_compass;then
-        log_error "launch_compass failed"
+    export machines
+
+    if [[ "$DEPLOY_COMPASS" == "true" ]]; then
+        if ! prepare_env;then
+            echo "prepare_env failed"
+            exit 1
+        fi
+
+        log_info "########## set up network begin #############"
+        if ! create_nets;then
+            log_error "create_nets failed"
+            exit 1
+        fi
+
+        if ! launch_compass;then
+            log_error "launch_compass failed"
+            exit 1
+        fi
+    fi
+else
+    machines=`get_host_macs`
+    if [[ -z $machines ]];then
+        log_error "get_host_macs failed"
         exit 1
     fi
+
+    log_info "deploy host macs: $machines"
 fi
 
+
 if [[ -z "$REDEPLOY_HOST" || "$REDEPLOY_HOST" == "false" ]]; then
     if ! set_compass_machine; then
         log_error "set_compass_machine fail"