Add a common tool for creating libvirt vms w/o puppet

1.It requires two argumets, input config and output folder.
2.It can bring libvirt vms with puppet and trusted ssh keys.
3.It can also bring normal libvirt vms.

JIRA: BOTTLENECK-57

Change-Id: I47fd61c328b45d03b4d0daf25fd9e4fe79303f88
Signed-off-by: wangyaoguang <sunshine.wang@huawei.com>

8 files changed, 424 insertions, 0 deletions

diff --git a/utils/dev_env/deploy/create_libvirt_vm.sh b/utils/dev_env/deploy/create_libvirt_vm.sh
new file mode 100755
index 00000000..45ba2b8e
--- /dev/null
+++ b/utils/dev_env/deploy/create_libvirt_vm.sh
@@ -0,0 +1,237 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+##############################################
+#  Usage: ./deploy.sh paras_conf outout_dir
+##############################################
+SCRIPT_DIR=$(cd "$(dirname "$0")"; pwd)
+tool_dir=${0%/*}
+
+function print_usage()
+{
+    echo "Usage: ./deploy.sh paras_conf output_dir"
+}
+
+## sanity check
+if [ ! -f $tool_dir/mac_generator.sh ] || \
+    [ ! -f $tool_dir/libvirt_template.xml ] || \
+    [ ! -f $tool_dir/meta-data.template ] || \
+    [ ! -f $tool_dir/user-data.template ] || \
+    [ ! -f $tool_dir/p-master-user-data.template ] || \
+    [ ! -f $tool_dir/p-agent-user-data.template ]; then
+      echo "Lack some necessary files for this tool!"
+      echo "deploy.sh"
+      echo "mac_generator.sh"
+      echo "1 xml, and 4 template"
+      exit 1
+fi
+
+## Check input
+if [ $# != 2 ]; then
+    print_usage
+    exit 1
+fi
+if [ ! -f $1 ]; then
+    echo "Cannot find file: "$1
+    exit 1
+fi
+if [ -d $2 ]; then
+    echo "Ouput dir $2 exist!"
+    exit 1
+fi
+
+## Assign parameters
+host_names=""
+puppet_enable="true"
+master_host=""
+vm_mem=
+vm_cpu_cores=
+image_url=
+image_name=
+ipaddr_start=
+trusted_ssh_pub_keys_file=
+while read line
+do
+    line=(${line//=/ })
+    case ${line[0]} in
+        "host_names" )
+            host_names=${line[1]}
+        ;;
+        "puppet_enable" )
+            puppet_enable=${line[1]}
+        ;;
+        "master_host" )
+            master_host=${line[1]}
+        ;;
+        "vm_mem" )
+            vm_mem=${line[1]}
+        ;;
+        "vm_cpu_cores" )
+            vm_cpu_cores=${line[1]}
+        ;;
+        "image_url" )
+            image_url=${line[1]}
+        ;;
+        "image_name" )
+            image_name=${line[1]}
+        ;;
+        "ipaddr_start" )
+            ipaddr_start=${line[1]}
+        ;;
+        "trusted_ssh_pub_keys_file" )
+            trusted_ssh_pub_keys_file=${line[1]}
+        ;;
+    esac
+done < $1
+
+echo "puppet_enable="$puppet_enable
+
+# Check parameters in conf file
+if [ $puppet_enable == "true" ] ; then
+    if [ ${#master_host} == 0 ];then
+        echo "Should specify master_host!"
+        exit 1
+    else
+        result=$(echo ${host_names} | grep  "${master_host}")
+        if [ ${result} == "" ]; then
+            echo "Specified master_host is invalid!"
+            exit 1
+        fi
+    fi
+fi
+
+# Define and Prepare needed data
+mac_arr=
+hostname_arr=(${host_names//,/ })
+virt_num=${#hostname_arr[@]}
+ip_arr=()
+replaced_hosts=""
+replaced_ssh_keys=""
+output_dir=$2
+work_dir=
+host_vm_dir=
+cache_dir=
+function init(){
+    # Generate mac address
+    local mac_generator=${tool_dir}/mac_generator.sh
+    chmod +x $mac_generator
+    mac_str=$($mac_generator $virt_num)
+    mac_arr=($mac_str)
+
+    # Generate hosts info
+    local ip=""
+    i=0
+    for host in "${hostname_arr[@]}"; do
+        ip=${ipaddr_prefix}$((i+$ipaddr_idx))
+        ip_arr+=($ip)
+        # Note the format, especially the space
+        replaced_hosts=${replaced_hosts}"      "${ip}" "${host}"\n"
+        let i=i+1
+    done
+
+    # Generate ssh public keys
+    echo "## trusted_ssh_pub_keys_file --> "${trusted_ssh_pub_keys_file}
+    if [ ${trusted_ssh_pub_keys_file} != "" ] && [ -f ${trusted_ssh_pub_keys_file} ]; then
+        while read line
+        do
+            # Note the format, especially space
+            replaced_ssh_keys=${replaced_ssh_keys}"    - "${line}"\n"
+        done < ${trusted_ssh_pub_keys_file}
+        # delete last "\n" in replaced_ssh_keysi
+        replaced_ssh_keys=${replaced_ssh_keys%\\n}
+    fi
+
+    # Prepare needed folder and files
+    if [ ${output_dir:0-1} == "/" ]; then
+        output_dir=${output_dir%/*}
+    fi
+    output_file=$output_dir/hosts.info
+    mkdir -p $output_dir
+    touch $output_file
+    echo "## Output host_info file --> "$output_file
+
+    work_dir=$output_dir/work
+    host_vm_dir=$work_dir/vm
+    cache_dir=$work_dir/cache
+    mkdir -p $work_dir
+    mkdir -p $host_vm_dir
+    mkdir -p $cache_dir
+
+    # Cache img file
+    echo "## Cache img file"
+    curl --connect-timeout 10 -o ${cache_dir}/$image_name $image_url
+}
+
+# Bring up instances/vms
+sub_ip_arr=(${ipaddr_start//./ })
+ipaddr_prefix=${sub_ip_arr[0]}"."${sub_ip_arr[1]}"."${sub_ip_arr[2]}"."
+ipaddr_idx=${sub_ip_arr[3]}
+function bring_up() {
+    i=0
+    while (($i < $virt_num))
+    do
+        echo "Bring up a vm, hostname: ${hostname_arr[$i]}, ip: ${ip_arr[$i]}, mac: ${mac_arr[$i]}"
+        vm_dir=$host_vm_dir/${hostname_arr[$i]}
+        mkdir -p $vm_dir
+        cp ${cache_dir}/$image_name $vm_dir
+
+        sed -e "s/REPLACE_IPADDR/${ip_arr[$i]}/g" \
+            -e "s/REPLACE_GATEWAY/${ipaddr_prefix}1/g" \
+            -e "s/REPLACE_HOSTNAME/${hostname_arr[$i]}/g" \
+            ${tool_dir}/meta-data.template > ${cache_dir}/meta-data
+
+        if [ ${puppet_enable} == "true" ]; then
+            # Use puppet user data
+            echo "hostname: "${hostname_arr[$i]}
+            if [ ${hostname_arr[$i]} == ${master_host} ]; then
+                cp ${tool_dir}/p-master-user-data.template ${cache_dir}/user-data.template
+            else
+                cp ${tool_dir}/p-agent-user-data.template ${cache_dir}/user-data.templatate
+            fi
+            sed -e "s#REPLACED_TRUSTED_PUB_SSH_KEYS#${replaced_ssh_keys}#g" \
+                -e "s#REPLACED_HOSTS_INFO#${replaced_hosts}#g" \
+                -e "s/REPLACED_PUPPET_MASTER_SERVER/${master_host}/g" \
+                ${cache_dir}/user-data.template > ${cache_dir}/user-data
+        else
+           # Use common user data
+           echo "## Use common user-data.template"
+           cp ${tool_dir}/user-data.template ${cache_dir}/user-data
+        fi
+
+        genisoimage -output seed.iso -volid cidata -joliet -rock ${cache_dir}/user-data ${cache_dir}/meta-data
+        mv seed.iso ${vm_dir}/
+        # Create vm xml
+        sed -e "s/REPLACE_MEM/$vm_mem/g" \
+            -e "s/REPLACE_CPU/$vm_cpu_cores/g" \
+            -e "s/REPLACE_NAME/${hostname_arr[$i]}/g" \
+            -e "s#REPLACE_IMAGE#$vm_dir/disk.img#g" \
+            -e "s#REPLACE_SEED_IMAGE#$vm_dir/seed.iso#g" \
+            -e "s/REPLACE_MAC_ADDR/${mac_arr[$i]}/g" \
+            ${tool_dir}/libvirt_template.xml > ${vm_dir}/libvirt.xml
+
+        echo "${ip_arr[$i]} ${hostname_arr[$i]}" >> $output_file
+
+        echo "Will define xml from:"${vm_dir}"/libvirt.xml"
+        echo "start: "${hostname_arr[$i]}
+        sudo virsh define ${vm_dir}/libvirt.xml
+        sudo virsh start ${hostname_arr[$i]}
+        let i=i+1
+        rm -rf ${cache_dir}/meta-data ${cache_dir}/user-data
+    done
+}
+
+function clean(){
+    rm -rf ${cache_dir}
+}
+
+init
+bring_up
+clean
diff --git a/utils/dev_env/deploy/libvirt_template.xml b/utils/dev_env/deploy/libvirt_template.xml
new file mode 100644
index 00000000..eddeff00
--- /dev/null
+++ b/utils/dev_env/deploy/libvirt_template.xml
@@ -0,0 +1,41 @@
+<domain type='kvm'>
+  <name>REPLACE_NAME</name>
+  <memory unit='MiB'>REPLACE_MEM</memory>
+  <currentMemory unit='MiB'>REPLACE_MEM</currentMemory>
+  <vcpu placement='static'>REPLACE_CPU</vcpu>
+  <os>
+    <type arch='x86_64'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <features>
+    <acpi/><apic/><pae/>
+  </features>
+  <clock offset="utc"/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>restart</on_crash>
+  <devices>
+    <disk type='file' device='disk'>
+      <driver name='qemu' type='qcow2'/>
+      <source file='REPLACE_IMAGE'/>
+      <target dev='vda' bus='virtio'/>
+    </disk>
+    <disk type='file' device='cdrom'>
+      <driver name='qemu'/>
+      <source file='REPLACE_SEED_IMAGE'/>
+      <target dev='hdb'/>
+      <readonly/>
+    </disk>
+    <interface type='network'>
+      <source network='default'/>
+      <mac address='REPLACE_MAC_ADDR'/>
+      <model type='virtio'/>
+    </interface>
+    <input type='tablet' bus='usb'/>
+    <graphics type='vnc' port='-1' autoport='yes' listen='0.0.0.0'/>
+    <console type='pty'/>
+    <video>
+      <model type='cirrus'/>
+    </video>
+  </devices>
+</domain>
diff --git a/utils/dev_env/deploy/mac_generator.sh b/utils/dev_env/deploy/mac_generator.sh
new file mode 100755
index 00000000..f8bc5e08
--- /dev/null
+++ b/utils/dev_env/deploy/mac_generator.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+## This script is used in 'create_libvirt_vm.sh'
+function mac_address_part() {
+    hex_number=$(printf '%02x' $RANDOM)
+    number_length=${#hex_number}
+    number_start=$(expr $number_length - 2)
+    echo ${hex_number:$number_start:2}
+}
+
+function mac_address() {
+    echo "00:00:$(mac_address_part):$(mac_address_part):$(mac_address_part):$(mac_address_part)"
+}
+
+machines=''
+for i in `seq $1`; do
+  mac=$(mac_address)
+
+  if [[ -z $machines ]]; then
+    machines="${mac}"
+  else
+    machines="${machines} ${mac}"
+  fi
+done
+
+echo ${machines}
diff --git a/utils/dev_env/deploy/meta-data.template b/utils/dev_env/deploy/meta-data.template
new file mode 100644
index 00000000..43db6597
--- /dev/null
+++ b/utils/dev_env/deploy/meta-data.template
@@ -0,0 +1,11 @@
+instance-id: iid-abcdeff
+network-interfaces: |
+  # The primary network interface
+  auto eth0
+  iface eth0 inet static
+  address REPLACE_IPADDR
+  netmask 255.255.255.0
+  gateway REPLACE_GATEWAY
+  dns-nameservers 8.8.8.8
+local-hostname: REPLACE_HOSTNAME
+
diff --git a/utils/dev_env/deploy/p-agent-user-data.template b/utils/dev_env/deploy/p-agent-user-data.template
new file mode 100644
index 00000000..3d778f1a
--- /dev/null
+++ b/utils/dev_env/deploy/p-agent-user-data.template
@@ -0,0 +1,40 @@
+#cloud-config
+
+users:
+  - name: ubuntu
+    ssh-authorized-keys:
+REPLACED_TRUSTED_PUB_SSH_KEYS
+    - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwmx8/86Vc1yqqplMPkm/w6wB2r5Yl6CjON85Ko5SzPEpnysJwO5ygbMClQTbzCUztIpFeO8D6xdWlhPmhMEJIWtRh+Dx0swB+ByMraGTP92xxKo2ukWzYI6eu25LmJX+a1lmVPmewQq+6dPzIwgCZvwD8UVe5L+d0EP2rb3iniQVoBzksUmhL3N6jp2Lgpscfi1yXFgjFdLpDsCcFcM+Q6wQUC2+qGBYa5RxqercWzISSbvAsYwC7UepmjFpcj3W/WX2QcOCDIlAzeURvC8PbOdzCtgeMhuhH2499BcIUSzPMpjzvpRiHr5SeF/zrPeW+2Ei1MfHi1qrBCxS8JBqt bottlenecks@bottlenecks.opnfv.org
+    sudo: ['ALL=(ALL) NOPASSWD:ALL']
+    groups: sudo
+    shell: /bin/bash
+
+runcmd:
+  - restart ssh
+  - wget https://apt.puppetlabs.com/puppetlabs-release-trusty.deb
+  - sudo dpkg -i puppetlabs-release-trusty.deb
+  - sudo apt-get update
+  - sudo apt-get -y install puppet
+  - sed -i '/^factpath/a basemodulepath=\/etc\/puppet\/modules' /etc/puppet/puppet.conf
+  - sed -i '/^factpath/a server=REPLACED_PUPPET_MASTER_SERVER' /etc/puppet/puppet.conf
+  - sed -i '/^factpath/a runinterval=1d' /etc/puppet/puppet.conf
+  - sed -i '/^templatedir/d' /etc/puppet/puppet.conf
+  - sed -i 's/START=no/START=yes/g' /etc/default/puppet
+
+write_files:
+  - path: /etc/hosts
+    content: |
+      127.0.0.1 localhost
+
+      # The following lines are desirable for IPv6 capable hosts
+      ::1 ip6-localhost ip6-loopback
+      fe00::0 ip6-localnet
+      ff00::0 ip6-mcastprefix
+      ff02::1 ip6-allnodes
+      ff02::2 ip6-allrouters
+      ff02::3 ip6-allhosts
+
+      # hosts info for all vms
+REPLACED_HOSTS_INFO
+
+final_message: "The system with puppet is finally up, after $UPTIME seconds."
diff --git a/utils/dev_env/deploy/p-master-user-data.template b/utils/dev_env/deploy/p-master-user-data.template
new file mode 100644
index 00000000..446c9698
--- /dev/null
+++ b/utils/dev_env/deploy/p-master-user-data.template
@@ -0,0 +1,39 @@
+#cloud-config
+
+users:
+  - name: ubuntu
+    ssh-authorized-keys:
+REPLACED_TRUSTED_PUB_SSH_KEYS
+    - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwmx8/86Vc1yqqplMPkm/w6wB2r5Yl6CjON85Ko5SzPEpnysJwO5ygbMClQTbzCUztIpFeO8D6xdWlhPmhMEJIWtRh+Dx0swB+ByMraGTP92xxKo2ukWzYI6eu25LmJX+a1lmVPmewQq+6dPzIwgCZvwD8UVe5L+d0EP2rb3iniQVoBzksUmhL3N6jp2Lgpscfi1yXFgjFdLpDsCcFcM+Q6wQUC2+qGBYa5RxqercWzISSbvAsYwC7UepmjFpcj3W/WX2QcOCDIlAzeURvC8PbOdzCtgeMhuhH2499BcIUSzPMpjzvpRiHr5SeF/zrPeW+2Ei1MfHi1qrBCxS8JBqt bottlenecks@bottlenecks.opnfv.org
+    sudo: ['ALL=(ALL) NOPASSWD:ALL']
+    groups: sudo
+    shell: /bin/bash
+
+runcmd:
+  - restart ssh
+  - wget https://apt.puppetlabs.com/puppetlabs-release-trusty.deb
+  - sudo dpkg -i puppetlabs-release-trusty.deb
+  - sudo apt-get update
+  - sudo apt-get -y install puppetserver
+  - sed -i '/^factpath/a basemodulepath=\/etc\/puppet\/modules' /etc/puppet/puppet.conf
+  - sed -i '/^factpath/a server=REPLACED_PUPPET_MASTER_SERVER' /etc/puppet/puppet.conf
+  - sed -i '/^factpath/a runinterval=1d' /etc/puppet/puppet.conf
+  - sed -i '/^templatedir/d' /etc/puppet/puppet.conf
+
+write_files:
+  - path: /etc/hosts
+    content: |
+      127.0.0.1 localhost
+
+      # The following lines are desirable for IPv6 capable hosts
+      ::1 ip6-localhost ip6-loopback
+      fe00::0 ip6-localnet
+      ff00::0 ip6-mcastprefix
+      ff02::1 ip6-allnodes
+      ff02::2 ip6-allrouters
+      ff02::3 ip6-allhosts
+
+      # hosts info for all vms
+REPLACED_HOSTS_INFO
+
+final_message: "The system with puppet is finally up, after $UPTIME seconds."
diff --git a/utils/dev_env/deploy/user-data.template b/utils/dev_env/deploy/user-data.template
new file mode 100644
index 00000000..8692cde1
--- /dev/null
+++ b/utils/dev_env/deploy/user-data.template
@@ -0,0 +1 @@
+#cloud-config
diff --git a/utils/dev_env/paras.conf b/utils/dev_env/paras.conf
new file mode 100644
index 00000000..1ebf104a
--- /dev/null
+++ b/utils/dev_env/paras.conf
@@ -0,0 +1,20 @@
+##############################################################################
+# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+## This is a sample configuration file, as one input parameter for 'create_libvirt_vm.sh'
+host_names=test-master,test-agent
+puppet_enable=true
+master_host=test-master
+vm_mem=4096
+vm_cpu_cores=4
+image_url=file:///home/ubuntu/trusty-server-cloudimg-amd64-btnks.img
+image_name=disk.img
+ipaddr_start=192.168.122.201
+trusted_ssh_pub_keys_file=/home/ubuntu/trusted_ssh_pub_keys