Adapt compass4nfv scripts to new compass-deck

JIRA: COMPASS-612

1. pass machine info through file
2. generate mac address in config_parse.py

Change-Id: Iaaf0f2b1f4ab97e8200c8c046746168d991570e3
Signed-off-by: Harry Huang <huangxiangyu5@huawei.com>

10 files changed, 131 insertions, 123 deletions

diff --git a/deploy/client.py b/deploy/client.py
index e612160b..3a9b0dec 100644
--- a/deploy/client.py
+++ b/deploy/client.py
@@ -79,8 +79,8 @@ opts = [
     cfg.BoolOpt('poll_switches',
                 help='if the client polls switches',
                 default=True),
-    cfg.StrOpt('machines',
-               help='comma separated mac addresses of machines',
+    cfg.StrOpt('machine_file',
+               help='mac addresses and ipmi info of machines',
                default=''),
     cfg.StrOpt('subnets',
                help='comma seperated subnets',
@@ -327,12 +327,12 @@ class CompassClient(object):
                 'get all machines status: %s, resp: %s', status, resp)
             raise RuntimeError('failed to get machines')
 
-        machines_to_add = list(set([
-            machine for machine in CONF.machines.split(',')
-            if machine
-        ]))
+        with open(CONF.machine_file) as fd:
+            machines_to_add = [str(m["mac"]) for m in yaml.load(fd)]
 
+        resp = byteify(resp)
         machines_db = [str(m["mac"]) for m in resp]
+
         LOG.info(
             'machines in db: %s\n to add: %s',
             machines_db,
@@ -983,8 +983,6 @@ class CompassClient(object):
         ansible_log = "%s/work/deploy/docker/ansible/run/%s-%s/ansible.log" \
                       % (compass_dir, CONF.adapter_name, CONF.cluster_name)
         os.system("sudo touch %s" % ansible_log)
-        os.system("sudo chmod +x -R %s/work/deploy/docker/ansible/run/"
-                  % compass_dir)
         ansible_print = multiprocessing.Process(target=print_log,
                                                 args=(ansible_log,))
         ansible_print.start()
diff --git a/deploy/compass_conf/templates/cobbler/ubuntu-16.04.3-server-x86_64/system.tmpl b/deploy/compass_conf/templates/cobbler/ubuntu-16.04.3-server-x86_64/system.tmpl
index cfcc883e..366cbc13 100755
--- a/deploy/compass_conf/templates/cobbler/ubuntu-16.04.3-server-x86_64/system.tmpl
+++ b/deploy/compass_conf/templates/cobbler/ubuntu-16.04.3-server-x86_64/system.tmpl
@@ -11,36 +11,38 @@
     "proxy": "$getVar('http_proxy', '')",
     "modify_interface":
         #set networks = $networks
+        #set mac = $mac
         #set rekeyed = {}
         #set promic_nic = ""
-        #for $nic, $val in $networks.iteritems():
-            #set ip_key = '-'.join(('ipaddress', $nic))
-            #set netmask_key = '-'.join(('netmask', $nic))
-            #set mgmt_key = '-'.join(('management', $nic))
-            #set static_key = '-'.join(('static', $nic))
-            #set $rekeyed[$ip_key] = $val.ip
-            #set $rekeyed[$netmask_key] = $val.netmask
-            #set $rekeyed[$static_key] = True
+        #for $nic in $mac
+            #set mac_key = '-'.join(('macaddress', $nic))
+            #set $rekeyed[$mac_key] = $mac[$nic]
+            #if $nic in $networks
+                #set val = $networks[$nic]
+                #set ip_key = '-'.join(('ipaddress', $nic))
+                #set netmask_key = '-'.join(('netmask', $nic))
+                #set mgmt_key = '-'.join(('management', $nic))
+                #set static_key = '-'.join(('static', $nic))
+                #set $rekeyed[$ip_key] = $val.ip
+                #set $rekeyed[$netmask_key] = $val.netmask
+                #set $rekeyed[$static_key] = True
 
-            #set dns_key = '-'.join(('dnsname', $nic))
-            #if $val.is_mgmt
-                #set $rekeyed[$dns_key] = $dns
-            #else
-                #if '.' in $dns
-                    #set $dns_name, $dns_domain = $dns.split('.', 1)
-                    #set $dns_nic = '%s-%s.%s' % ($dns_name, $nic, $dns_domain)
+                #set dns_key = '-'.join(('dnsname', $nic))
+                #if $val.is_mgmt
+                    #set $rekeyed[$dns_key] = $dns
                 #else
-                    #set $dns_nic = '%s-%s' % ($dns, $nic)
+                    #if '.' in $dns
+                        #set $dns_name, $dns_domain = $dns.split('.', 1)
+                        #set $dns_nic = '%s-%s.%s' % ($dns_name, $nic, $dns_domain)
+                    #else
+                        #set $dns_nic = '%s-%s' % ($dns, $nic)
+                    #end if
+                    #set $rekeyed[$dns_key] = $dns_nic
                 #end if
-                #set $rekeyed[$dns_key] = $dns_nic
-            #end if
 
-            #if $val.is_promiscuous:
-                #set promic_nic = $nic
-            #end if
-            #if $val.is_mgmt:
-                #set mac_key = '-'.join(('macaddress', $nic))
-                #set $rekeyed[$mac_key] = $mac
+                #if $val.is_promiscuous
+                    #set promic_nic = $nic
+                #end if
             #end if
         #end for
         #set nic_info = json.dumps($rekeyed, encoding='utf-8')
@@ -71,5 +73,9 @@
         "ignore_proxy": "$no_proxy",
         "local_repo": "$getVar('local_repo', '')",
         "disk_num": "1"
-    }
+    },
+    "power_address": "$power_manage.ip",
+    "power_user": "$power_manage.username",
+    "power_pass": "$power_manage.password",
+    "power_type": "$getVar('power_type', 'ipmilan')"
 }
diff --git a/deploy/compass_vm.sh b/deploy/compass_vm.sh
index f7bc378e..b203889b 100755
--- a/deploy/compass_vm.sh
+++ b/deploy/compass_vm.sh
@@ -47,9 +47,7 @@ function install_compass_core() {
 }
 
 function set_compass_machine() {
-    local config_file=$WORK_DIR/installer/docker-compose/group_vars/all
-    sed -i '/pxe_boot_macs/d' $config_file
-    echo "pxe_boot_macs: [${machines}]" >> $config_file
+    cp $WORK_DIR/script/machine $WORK_DIR/docker/
 
     ansible-playbook $WORK_DIR/installer/docker-compose/add_machine.yml
 }
diff --git a/deploy/config_parse.py b/deploy/config_parse.py
index 9cdf1acb..88a31e9a 100644
--- a/deploy/config_parse.py
+++ b/deploy/config_parse.py
@@ -11,27 +11,44 @@ import os
 import netaddr
 import yaml
 import sys
+import random
 from Cheetah.Template import Template
 
 
-def init(file):
+def load_yaml(file):
     with open(file) as fd:
         return yaml.safe_load(fd)
 
 
-def export_env_dict(env_dict, ofile, direct=False):
-    if not os.path.exists(ofile):
-        raise IOError("output file: %s not exist" % ofile)
+def dump_yaml(data, file):
+    with open(file, "w") as fd:
+        yaml.safe_dump(data, fd, default_flow_style=False)
+
+
+def mac_generator():
+    def random_hex():
+        return random.choice("0123456789ABCDEF")
+    mac = "00:00"
+    for i in xrange(4):
+        mac += ":{0}{1}".format(random_hex(), random_hex())
+    return mac
+
+
+def export_env_dict(env_dict, output_path, direct=False):
+    if not os.path.exists(output_path):
+        raise IOError("output file: %s not exist" % output_path)
     if direct:
         for k, v in env_dict.items():
-            os.system("echo 'export %s=\"%s\"' >> %s" % (k, v, ofile))
+            os.system("echo 'export %s=\"%s\"' >> %s" %
+                      (k, v, output_path))
     else:
         for k, v in env_dict.items():
-            os.system("echo 'export %s=${%s:-%s}' >> %s" % (k, k, v, ofile))
+            os.system("echo 'export %s=${%s:-%s}' >> %s" %
+                      (k, k, v, output_path))
 
 
 def decorator(func):
-    def wrapter(s, seq):
+    def wrapter(s, seq=None):
         host_list = s.get('hosts', [])
         result = []
         for host in host_list:
@@ -41,8 +58,10 @@ def decorator(func):
             result.append(s)
         if len(result) == 0:
             return ""
-        else:
+        elif seq:
             return "\"" + seq.join(result) + "\""
+        else:
+            return result
     return wrapter
 
 
@@ -57,11 +76,15 @@ def hostroles(s, seq, host=None):
 
 
 @decorator
-def hostmacs(s, seq, host=None):
-    return host.get('mac', '')
+def hostmachines(s, seq, host=None):
+    return {'mac': host.get('interfaces', {}),
+            'power_type': host.get('power_type', ''),
+            'power_ip': host.get('power_ip', ''),
+            'power_user': host.get('power_user', ''),
+            'power_pass': host.get('power_pass', '')}
 
 
-def export_network_file(dha, network, ofile):
+def export_network_file(dha, network, output_path):
     install_network_env = {}
     host_network_env = {}
     ip_settings = network['ip_settings']
@@ -79,7 +102,7 @@ def export_network_file(dha, network, ofile):
     install_network_env.update({'INSTALL_NETMASK': mgmt_netmask})
     install_network_env.update({'INSTALL_IP_RANGE': dhcp_ip_range})
     install_network_env.update({'VIP': internal_vip})
-    export_env_dict(install_network_env, ofile)
+    export_env_dict(install_network_env, output_path)
 
     pxe_nic = os.environ['PXE_NIC']
     host_ip_range = mgmt_net['ip_ranges'][0]
@@ -94,10 +117,10 @@ def export_network_file(dha, network, ofile):
     host_network_env.update({'NETWORK_MAPPING': "install=" + pxe_nic})
     host_network_env.update({'HOST_NETWORKS': ';'.join(host_networks)})
     host_network_env.update({'SUBNETS': ','.join(host_subnets)})
-    export_env_dict(host_network_env, ofile, True)
+    export_env_dict(host_network_env, output_path, True)
 
 
-def export_dha_file(dha, dha_file, ofile):
+def export_dha_file(dha, output_path, machine_path):
     env = {}
     env.update(dha)
     if env.get('hosts', []):
@@ -121,19 +144,28 @@ def export_dha_file(dha, dha_file, ofile):
     env.update({'FLAVOR': dha.get('FLAVOR', "cluster")})
     env.update({'HOSTNAMES': hostnames(dha, ',')})
     env.update({'HOST_ROLES': hostroles(dha, ';')})
-    env.update({'DHA': dha_file})
 
-    value = hostmacs(dha, ',')
-    if len(value) > 0:
-        env.update({'HOST_MACS': value})
+    machine = []
+    if dha.get('TYPE') == "virtual":
+        virtual_mac = []
+        for host in dha.get('hosts'):
+            mac = mac_generator()
+            machine.append({"mac": {"eth0": mac}, "power_type": "libvirt"})
+            virtual_mac.append(mac)
+        env.update({'HOST_MACS': ",".join(virtual_mac)})
+    else:
+        value = hostmachines(dha)
+        for item in value:
+            machine.append(item)
+    dump_yaml(machine, machine_path)
 
     if dha.get('TYPE', "virtual") == "virtual":
         env.update({'VIRT_NUMBER': len(dha['hosts'])})
 
-    export_env_dict(env, ofile)
+    export_env_dict(env, output_path)
 
 
-def export_reset_file(dha, tmpl_dir, output_dir, ofile):
+def export_reset_file(dha, tmpl_dir, output_dir, output_path):
     tmpl_file_name = dha.get('POWER_TOOL', '')
     if not tmpl_file_name:
         return
@@ -151,28 +183,31 @@ def export_reset_file(dha, tmpl_dir, output_dir, ofile):
         f.write(tmpl.respond())
 
     power_manage_env = {'POWER_MANAGE': reset_file_name}
-    export_env_dict(power_manage_env, ofile, True)
+    export_env_dict(power_manage_env, output_path, True)
 
 if __name__ == "__main__":
-    if len(sys.argv) != 6:
-        print("parameter wrong%d %s" % (len(sys.argv), sys.argv))
+    if len(sys.argv) != 7:
+        print("parameter wrong %d %s" % (len(sys.argv), sys.argv))
         sys.exit(1)
 
-    _, dha_file, network_file, tmpl_dir, output_dir, output_file = sys.argv
+    _, dha_file, network_file, tmpl_dir, output_dir, output_file,\
+        machine_file = sys.argv
 
     if not os.path.exists(dha_file):
         print("%s is not exist" % dha_file)
         sys.exit(1)
 
-    ofile = os.path.join(output_dir, output_file)
-    os.system("touch %s" % ofile)
-    os.system("echo \#config file deployment parameter > %s" % ofile)
+    output_path = os.path.join(output_dir, output_file)
+    machine_path = os.path.join(output_dir, machine_file)
+    os.system("touch %s" % output_path)
+    os.system("echo \#config file deployment parameter > %s" % output_path)
+    os.system("touch %s" % machine_path)
 
-    dha_data = init(dha_file)
-    network_data = init(network_file)
+    dha_data = load_yaml(dha_file)
+    network_data = load_yaml(network_file)
 
-    export_dha_file(dha_data, dha_file, ofile)
-    export_network_file(dha_data, network_data, ofile)
-    export_reset_file(dha_data, tmpl_dir, output_dir, ofile)
+    export_dha_file(dha_data, output_path, machine_path)
+    export_network_file(dha_data, network_data, output_path)
+    export_reset_file(dha_data, tmpl_dir, output_dir, output_path)
 
     sys.exit(0)
diff --git a/deploy/deploy_host.sh b/deploy/deploy_host.sh
index 94abf768..7a7b5dd5 100755
--- a/deploy/deploy_host.sh
+++ b/deploy/deploy_host.sh
@@ -43,7 +43,7 @@ function deploy_host(){
     --console_credentials="${CONSOLE_CREDENTIALS}" --host_networks="${HOST_NETWORKS}" \
     --network_mapping="${NETWORK_MAPPING}" --package_config_json_file="${PACKAGE_CONFIG_FILENAME}" \
     --host_roles="${HOST_ROLES}" --default_roles="${DEFAULT_ROLES}" --switch_ips="${SWITCH_IPS}" \
-    --machines=${machines//\'} --switch_credential="${SWITCH_CREDENTIAL}" --deploy_type="${TYPE}" \
+    --machine_file=${machine_file} --switch_credential="${SWITCH_CREDENTIAL}" --deploy_type="${TYPE}" \
     --deployment_timeout="${DEPLOYMENT_TIMEOUT}" --${POLL_SWITCHES_FLAG} --dashboard_url="${DASHBOARD_URL}" \
     --cluster_vip="${VIP}" --network_cfg="$NETWORK" --neutron_cfg="$NEUTRON" \
     --enable_secgroup="${ENABLE_SECGROUP}" --enable_fwaas="${ENABLE_FWAAS}" --expansion="${EXPANSION}" \
diff --git a/deploy/deploy_parameter.sh b/deploy/deploy_parameter.sh
index 78223fcd..d45d4171 100755
--- a/deploy/deploy_parameter.sh
+++ b/deploy/deploy_parameter.sh
@@ -100,7 +100,7 @@ function process_default_para()
            "$DHA" "$NETWORK" \
            "${COMPASS_DIR}/deploy/template" \
            "${WORK_DIR}/script" \
-           "deploy_config.sh"
+           "deploy_config.sh" machine
 
     echo ${WORK_DIR}/script/deploy_config.sh
 }
diff --git a/deploy/host_baremetal.sh b/deploy/host_baremetal.sh
index 4c63f823..3c303567 100755
--- a/deploy/host_baremetal.sh
+++ b/deploy/host_baremetal.sh
@@ -15,14 +15,6 @@ function reboot_hosts() {
 }
 
 function get_host_macs() {
-    if [[ "$EXPANSION" == "false" ]]; then
-        machines=`echo $HOST_MACS | sed -e 's/,/'\',\''/g' -e 's/^/'\''/g' -e 's/$/'\''/g'`
-        echo $machines > $WORK_DIR/switch_machines
-    else
-        machines_old=`cat $WORK_DIR/switch_machines`
-        machines_add=`echo $HOST_MACS | sed -e 's/,/'\',\''/g' -e 's/^/'\''/g' -e 's/$/'\''/g'`
-        echo $machines_add $machines_old > $WORK_DIR/switch_machines
-        machines=`echo $machines_add $machines_old|sed 's/ /,/g'`
-    fi
+    local machines=`echo $HOST_MACS | sed -e 's/,/'\',\''/g' -e 's/^/'\''/g' -e 's/$/'\''/g'`
     echo $machines
 }
diff --git a/deploy/host_virtual.sh b/deploy/host_virtual.sh
index d955b747..c3133cf6 100755
--- a/deploy/host_virtual.sh
+++ b/deploy/host_virtual.sh
@@ -25,20 +25,21 @@ function tear_down_machines() {
 }
 
 function reboot_hosts() {
+    echo "reboot"
     # We do need it for aarch64
-    if [ "$COMPASS_ARCH" = "aarch64" ]; then
-        old_ifs=$IFS
-        IFS=,
-        for i in $HOSTNAMES; do
-            sudo virsh destroy $i
-            sleep 3
-            sudo virsh start $i
-            sleep 3
-        done
-        IFS=$old_ifs
-    else
-        log_warn "reboot_hosts do nothing"
-    fi
+#    if [ "$COMPASS_ARCH" = "aarch64" ]; then
+#        old_ifs=$IFS
+#        IFS=,
+#        for i in $HOSTNAMES; do
+#            sudo virsh destroy $i
+#            sleep 3
+#            sudo virsh start $i
+#            sleep 3
+#        done
+#        IFS=$old_ifs
+#    else
+#        log_warn "reboot_hosts do nothing"
+#    fi
 }
 
 function launch_host_vms() {
@@ -97,37 +98,7 @@ function recover_host_vms() {
 }
 
 function get_host_macs() {
-    local mac_generator=${COMPASS_DIR}/deploy/mac_generator.sh
-    local machines=
-
-    if [[ $REDEPLOY_HOST == "true" ]]; then
-        mac_array=`cat $WORK_DIR/switch_machines`
-        machines=`echo $mac_array|sed 's/ /,/g'`
-    else
-        if [[ -z $HOST_MACS ]]; then
-            if [[ "$EXPANSION" == "false" ]]; then
-                chmod +x $mac_generator
-                mac_array=`$mac_generator $VIRT_NUMBER`
-                echo $mac_array > $WORK_DIR/switch_machines
-                machines=`echo $mac_array|sed 's/ /,/g'`
-            else
-                machines_old=`cat $WORK_DIR/switch_machines`
-                chmod +x $mac_generator
-                machines_add=`$mac_generator $VIRT_NUMBER`
-                echo $machines_add $machines_old > $WORK_DIR/switch_machines
-                machines=`echo $machines_add $machines_old|sed 's/ /,/g'`
-            fi
-        else
-            if [[ "$EXPANSION" == "false" ]]; then
-                machines=`echo $HOST_MACS | sed -e 's/,/'\',\''/g' -e 's/^/'\''/g' -e 's/$/'\''/g'`
-            else
-                machines_old=`cat $WORK_DIR/switch_machines`
-                machines_add=`echo $HOST_MACS | sed -e 's/,/'\',\''/g' -e 's/^/'\''/g' -e 's/$/'\''/g'`
-                echo $machines_add $machines_old > $WORK_DIR/switch_machines
-                machines=`echo $machines_add $machines_old|sed 's/ /,/g'`
-            fi
-        fi
-    fi
+    local machines=`echo $HOST_MACS | sed -e 's/,/'\',\''/g' -e 's/^/'\''/g' -e 's/$/'\''/g'`
     echo $machines
 }
 
diff --git a/deploy/launch.sh b/deploy/launch.sh
index 8a009003..b11127ca 100755
--- a/deploy/launch.sh
+++ b/deploy/launch.sh
@@ -53,6 +53,7 @@ if [[ "$EXPANSION" == "false" ]]; then
         exit 1
     fi
 
+    export machine_file=$WORK_DIR/script/machine
     export machines
 
     CONTAINER_ALIVE=$(check_container_alive)
diff --git a/deploy/prepare.sh b/deploy/prepare.sh
index 38d98e1d..7447ae72 100755
--- a/deploy/prepare.sh
+++ b/deploy/prepare.sh
@@ -48,6 +48,13 @@ function extract_tar()
 function prepare_env() {
     sudo sed -i -e 's/^#user =.*/user = "root"/g' /etc/libvirt/qemu.conf
     sudo sed -i -e 's/^#group =.*/group = "root"/g' /etc/libvirt/qemu.conf
+    sudo sed -i 's/^.\?listen_tls.*/listen_tls = 0/g' /etc/libvirt/libvirtd.conf
+    sudo sed -i 's/^.\?listen_tcp.*/listen_tcp = 1/g' /etc/libvirt/libvirtd.conf
+    sudo sed -i 's/^.\?tcp_port.*/tcp_port = "16509"/g' /etc/libvirt/libvirtd.conf
+    sudo sed -i 's/^.\?listen_addr.*/listen_addr = "0.0.0.0"/g' /etc/libvirt/libvirtd.conf
+    sudo sed -i 's/^.\?auth_tcp.*/auth_tcp = "none"/g' /etc/libvirt/libvirtd.conf
+    sudo sed -i 's#\(exec /usr/sbin/libvirtd \$libvirtd_opts\)$#\1 -l#g' /etc/init/libvirt-bin.conf
+
     sudo service libvirt-bin restart
     if sudo service openvswitch-switch status|grep stop; then
         sudo service openvswitch-switch start