Prototype of automated libvirt installation

This is a prototype to experiment with the DEA concept and
how to use it to automatically deploy a KVM environment using
libvirt.

It is now possible to perform a fully automated deploy from
an ISO file.

Highlights:

  - Fully automated Fuel deploy installing three controllers,
    two computes in full HA mode including Ceilometer
  - create_dea.sh will create a DEA YAML file with an
    existing deployment as template.

  - The install will configure settings, networks and the
    interface config of all nodes.

JIRA:

Change-Id: Icc8922afca8508c9062aa5be95e2482086a4f624
Signed-off-by: Stefan K. Berg <stefan.k.berg@ericsson.com>

6 files changed, 343 insertions, 0 deletions

diff --git a/fuel/prototypes/libvirt/deploy/tools/transplant1.sh b/fuel/prototypes/libvirt/deploy/tools/transplant1.sh
new file mode 100755
index 0000000..9cead7a
--- /dev/null
+++ b/fuel/prototypes/libvirt/deploy/tools/transplant1.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+cleanup () {
+  if [ -n "$tmpDir" ]; then
+    rm -Rf $tmpDir
+  fi
+}
+
+trap cleanup exit
+
+error_exit () {
+  echo "Error: $@" >&2
+  exit 1
+}
+
+tmpDir=`mktemp -d /tmp/deaXXXX`
+
+export PATH=`dirname $0`:$PATH
+
+if [ $# -lt 1 ]; then
+  error_exit "Argument error"
+fi
+deafile=$1
+shift
+
+if [ $# -ne 0 ]; then
+  comment="$@"
+fi
+
+if [ ! -f "$deafile" ]; then
+  error_exit "Can't find $deafile"
+fi
+
+if [ `fuel env | tail -n +3 | grep -v '^$' | wc -l` -ne 1 ]; then
+  error_exit "Not exactly one environment"
+fi
+envId=`fuel env | tail -n +3 | grep -v '^$' | awk '{ print $1 }'`
+
+fuel settings --env $envId --download --dir $tmpDir > /dev/null || \
+  error_exit "Could not get settings"
+fuel network --env $envId --download --dir $tmpDir > /dev/null || \
+  error_exit "Could not get network settings"
+
+cp $tmpDir/network_${envId}.yaml network_before.yaml
+transplant_network_settings.py $tmpDir/network_${envId}.yaml $deafile || \
+  error_exit "Could not transplant network settings"
+fuel network --env $envId --upload --dir $tmpDir || \
+  error_exit "Could not update network settings"
+cp $tmpDir/network_${envId}.yaml network_after.yaml
+
+cp $tmpDir/settings_${envId}.yaml settings_before.yaml
+transplant_settings.py $tmpDir/settings_${envId}.yaml $deafile || \
+  error_exit "Could not transplant settings"
+fuel settings --env $envId --upload --dir $tmpDir || \
+  error_exit "Could not update settings"
+cp $tmpDir/settings_${envId}.yaml settings_after.yaml
+
+
diff --git a/fuel/prototypes/libvirt/deploy/tools/transplant2.sh b/fuel/prototypes/libvirt/deploy/tools/transplant2.sh
new file mode 100755
index 0000000..5049f88
--- /dev/null
+++ b/fuel/prototypes/libvirt/deploy/tools/transplant2.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+cleanup () {
+  if [ -n "$tmpDir" ]; then
+    rm -Rf $tmpDir
+  fi
+}
+
+trap cleanup exit
+
+error_exit () {
+  echo "Error: $@" >&2
+  exit 1
+}
+
+tmpDir=`mktemp -d /tmp/deaXXXX`
+
+export PATH=`dirname $0`:$PATH
+
+if [ $# -ne 1 ]; then
+  error_exit "Argument error"
+fi
+deaFile=$1
+
+if [ ! -f "$deaFile" ]; then
+  error_exit "Can't find $deaFile"
+fi
+
+
+if [ `fuel env | tail -n +3 | grep -v '^$' | wc -l` -ne 1 ]; then
+  error_exit "Not exactly one environment"
+fi
+envId=`fuel env | tail -n +3 | grep -v '^$' | awk '{ print $1 }'`
+
+# Phase 1: Graft deployment information
+if [ "a" == "b" ]; then
+fuel deployment --env $envId --default --dir $tmpDir || \
+  error_exit "Could not dump environment"
+
+for controller in `find $tmpDir -type f | grep -v compute`
+do
+  transplant_network_scheme.py $controller $deaFile controller || \
+    error_exit "Failed to graft `basename $controller`"
+done
+
+for compute in `find $tmpDir -type f | grep compute`
+do
+  transplant_network_scheme.py $compute $deaFile compute || \
+    error_exit "Failed to graft `basename $compute`"
+done
+
+fuel deployment --env $envId --upload --dir $tmpDir || \
+  error_exit "Could not upload environment"
+fi
+# Phase 2: Graft interface information
+
+for nodeId in `fuel node | grep True | awk '{ print $1}'`
+do
+  echo "Node $nodeId"
+  fuel node --node-id $nodeId --network --download --dir $tmpDir || \
+     error_exit "Could not download node $nodeId"
+
+  transplant_interfaces.py ${tmpDir}/node_${nodeId}/interfaces.yaml $deaFile || \
+     error_exit "Failed to graft interfaces"
+
+  fuel node --node-id $nodeId --network --upload --dir $tmpDir || \
+     error_exit "Could not upload node $nodeId"
+done
+
+
+
diff --git a/fuel/prototypes/libvirt/deploy/tools/transplant_interfaces.py b/fuel/prototypes/libvirt/deploy/tools/transplant_interfaces.py
new file mode 100755
index 0000000..8d076ff
--- /dev/null
+++ b/fuel/prototypes/libvirt/deploy/tools/transplant_interfaces.py
@@ -0,0 +1,66 @@
+#!/usr/bin/python
+##############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import yaml
+import re
+import sys
+import os
+
+if len(sys.argv) != 3:
+    sys.stderr.write("Usage: "+sys.argv[0]+" <infile> <deafile>\n")
+    sys.exit(1)
+
+infile = sys.argv[1]
+if not os.path.exists(infile):
+    sys.stderr.write("ERROR: The file "+infile+" could not be opened\n")
+    sys.exit(1)
+
+deafile = sys.argv[2]
+if not os.path.exists(deafile):
+    sys.stderr.write("ERROR: The file "+deafile+" could not be opened\n")
+    sys.exit(1)
+deafile = sys.argv[2]
+
+namespace = "interfaces"
+
+f1 = open(infile, 'r')
+doc1 = yaml.load(f1)
+f1.close()
+
+f2 = open(deafile, 'r')
+doc2 = yaml.load(f2)
+f2.close()
+
+# Create lookup table network name -> id for current setup
+nwlookup = {}
+for interface in doc1:
+  iface = {}
+  networks = []
+  for network in interface["assigned_networks"]:
+    nwlookup[network["name"]] = network["id"]
+  
+
+out = {}
+out["interfaces"] = {}
+
+for interface in doc1:
+  assigned = []
+  nw = {}
+  interface["assigned_networks"] = []
+  for nwname in doc2["interfaces"][interface["name"]]:
+    iface = {}
+    iface["id"] = nwlookup[nwname]
+    iface["name"] = nwname
+    interface["assigned_networks"].append(iface)
+
+f3 = open(infile, 'w')
+f3.write(yaml.dump(doc1, default_flow_style=False))
+f3.close()
diff --git a/fuel/prototypes/libvirt/deploy/tools/transplant_network_scheme.py b/fuel/prototypes/libvirt/deploy/tools/transplant_network_scheme.py
new file mode 100755
index 0000000..7d50cbe
--- /dev/null
+++ b/fuel/prototypes/libvirt/deploy/tools/transplant_network_scheme.py
@@ -0,0 +1,42 @@
+#!/usr/bin/python
+##############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import yaml
+import re
+import sys
+import os
+
+if len(sys.argv) != 4:
+    sys.stderr.write("Usage: "+sys.argv[0]+" <file> <deafile> [compute|controller]\n")
+    sys.exit(1)
+
+file = sys.argv[1]
+if not os.path.exists(file):
+    sys.stderr.write("ERROR: The file "+file+" could not be opened\n")
+    sys.exit(1)
+
+deafile = sys.argv[2]
+namespace = sys.argv[3]
+
+f1 = open(file, 'r')
+doc1 = yaml.load(f1)
+f1.close()
+
+f2 = open(deafile, 'r')
+doc2 = yaml.load(f2)
+f1.close()
+
+doc1["network_scheme"]["transformations"] = doc2[namespace]
+
+f2 = open(file, 'w')
+f2.write(yaml.dump(doc1, default_flow_style=False))
+f2.close()
+
diff --git a/fuel/prototypes/libvirt/deploy/tools/transplant_network_settings.py b/fuel/prototypes/libvirt/deploy/tools/transplant_network_settings.py
new file mode 100755
index 0000000..c0a46be
--- /dev/null
+++ b/fuel/prototypes/libvirt/deploy/tools/transplant_network_settings.py
@@ -0,0 +1,52 @@
+#!/usr/bin/python
+##############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import yaml
+import re
+import sys
+import os
+
+if len(sys.argv) != 3:
+    sys.stderr.write("Usage: "+sys.argv[0]+" <file> <deafile>\n")
+    sys.exit(1)
+
+file = sys.argv[1]
+if not os.path.exists(file):
+    sys.stderr.write("ERROR: The file "+file+" could not be opened\n")
+    sys.exit(1)
+
+deafile = sys.argv[2]
+if not os.path.exists(deafile):
+    sys.stderr.write("ERROR: The file "+deafile+" could not be opened\n")
+    sys.exit(1)
+
+f1 = open(file, 'r')
+doc1 = yaml.load(f1)
+f1.close()
+
+f2 = open(deafile, 'r')
+doc2 = yaml.load(f2)
+f2.close()
+
+# Grab IDs from Fuel version, graft onto DEA version and save
+id = []
+groupid = []
+for nw in doc1["networks"]:
+  id.append(nw["id"])
+  groupid.append(nw["group_id"])
+
+for nw in doc2["network"]["networks"]:
+  nw["id"] = id.pop(0)
+  nw["group_id"] = groupid.pop(0)
+
+f3 = open(file, 'w')
+f3.write(yaml.dump(doc2["network"], default_flow_style=False))
+f3.close()
diff --git a/fuel/prototypes/libvirt/deploy/tools/transplant_settings.py b/fuel/prototypes/libvirt/deploy/tools/transplant_settings.py
new file mode 100755
index 0000000..7f5c0d8
--- /dev/null
+++ b/fuel/prototypes/libvirt/deploy/tools/transplant_settings.py
@@ -0,0 +1,36 @@
+#!/usr/bin/python
+##############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import yaml
+import re
+import sys
+import os
+
+if len(sys.argv) != 3:
+    sys.stderr.write("Usage: "+sys.argv[0]+" <file> <deafile>\n")
+    sys.exit(1)
+
+file = sys.argv[1]
+if not os.path.exists(file):
+    sys.stderr.write("ERROR: The file "+file+" could not be opened\n")
+    sys.exit(1)
+
+deafile = sys.argv[2]
+
+f1 = open(deafile, 'r')
+doc = yaml.load(f1)
+f1.close()
+
+out = doc["settings"]
+f2 = open(file, 'w')
+f2.write(yaml.dump(out, default_flow_style=False))
+f2.close()
+