From dd2ff7be51548c62f5f708b7e323e23ca5171b95 Mon Sep 17 00:00:00 2001 From: AakashKT Date: Wed, 21 Mar 2018 14:12:24 +0530 Subject: Code for charm-k8s-ovn under sourcecode/JOID/charm-k8s-ovn Change-Id: I07209b0955436d06cdcd8b19f0dfe8f2c146a36c Signed-off-by: AakashKT --- sourcecode/JOID/charm-k8s-ovn/build_ovn.sh | 5 + sourcecode/JOID/charm-k8s-ovn/bundle/bundle.yaml | 101 ++++ .../interfaces/master-config/copyright | 13 + .../interfaces/master-config/interface.yaml | 4 + .../interfaces/master-config/peers.py | 189 +++++++ sourcecode/JOID/charm-k8s-ovn/layers/ovn/README.ex | 21 + .../JOID/charm-k8s-ovn/layers/ovn/config.yaml | 5 + sourcecode/JOID/charm-k8s-ovn/layers/ovn/copyright | 13 + sourcecode/JOID/charm-k8s-ovn/layers/ovn/icon.svg | 581 +++++++++++++++++++++ .../JOID/charm-k8s-ovn/layers/ovn/layer.yaml | 2 + .../JOID/charm-k8s-ovn/layers/ovn/metadata.yaml | 19 + .../JOID/charm-k8s-ovn/layers/ovn/reactive/ovn.py | 491 +++++++++++++++++ .../JOID/charm-k8s-ovn/layers/ovn/tests/00-setup | 5 + .../JOID/charm-k8s-ovn/layers/ovn/tests/10-deploy | 35 ++ 14 files changed, 1484 insertions(+) create mode 100755 sourcecode/JOID/charm-k8s-ovn/build_ovn.sh create mode 100644 sourcecode/JOID/charm-k8s-ovn/bundle/bundle.yaml create mode 100644 sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/copyright create mode 100644 sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/interface.yaml create mode 100644 sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/peers.py create mode 100644 sourcecode/JOID/charm-k8s-ovn/layers/ovn/README.ex create mode 100644 sourcecode/JOID/charm-k8s-ovn/layers/ovn/config.yaml create mode 100644 sourcecode/JOID/charm-k8s-ovn/layers/ovn/copyright create mode 100644 sourcecode/JOID/charm-k8s-ovn/layers/ovn/icon.svg create mode 100644 sourcecode/JOID/charm-k8s-ovn/layers/ovn/layer.yaml create mode 100644 sourcecode/JOID/charm-k8s-ovn/layers/ovn/metadata.yaml create mode 100644 sourcecode/JOID/charm-k8s-ovn/layers/ovn/reactive/ovn.py create mode 100755 sourcecode/JOID/charm-k8s-ovn/layers/ovn/tests/00-setup create mode 100755 sourcecode/JOID/charm-k8s-ovn/layers/ovn/tests/10-deploy diff --git a/sourcecode/JOID/charm-k8s-ovn/build_ovn.sh b/sourcecode/JOID/charm-k8s-ovn/build_ovn.sh new file mode 100755 index 0000000..8943091 --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/build_ovn.sh @@ -0,0 +1,5 @@ +#!bin/bash + +export JUJU_REPOSITORY=$(pwd) +export LAYER_PATH=$JUJU_REPOSITORY/layers +export INTERFACE_PATH=$JUJU_REPOSITORY/interfaces diff --git a/sourcecode/JOID/charm-k8s-ovn/bundle/bundle.yaml b/sourcecode/JOID/charm-k8s-ovn/bundle/bundle.yaml new file mode 100644 index 0000000..dfe3926 --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/bundle/bundle.yaml @@ -0,0 +1,101 @@ +series: xenial + +applications: + + kubeapi-load-balancer: + charm: "cs:~containers/kubeapi-load-balancer-16" + expose: true + num_units: 1 + to: + - "3" + annotations: + gui-x: '450' + gui-y: '250' + + kubernetes-master: + charm: "cs:~containers/kubernetes-master-35" + num_units: 1 + to: + - "0" + expose: true + options: + service-cidr: 192.168.200.0/24 + channel: 1.5/stable + annotations: + gui-x: '800' + gui-y: '850' + + kubernetes-worker: + charm: "cs:~containers/kubernetes-worker-40" + num_units: 2 + to: + - "1" + - "2" + expose: true + options: + channel: 1.5/stable + annotations: + gui-x: '100' + gui-y: '850' + + ovn: + charm: "cs:~aakashkt/ovn-16" + options: + gateway-physical-interface: "none" + annotations: + gui-x: '450' + gui-y: '750' + + etcd: + charm: "cs:~containers/etcd-40" + num_units: 1 + to: + - "0" + annotations: + gui-x: '800' + gui-y: '550' + + easyrsa: + charm: "cs:~containers/easyrsa-12" + num_units: 1 + to: + - "1" + annotations: + gui-x: '450' + gui-y: '550' + +relations: + - - "kubernetes-master:kube-api-endpoint" + - "kubeapi-load-balancer:apiserver" + - - "kubernetes-master:loadbalancer" + - "kubeapi-load-balancer:loadbalancer" + - - "kubernetes-worker:kube-api-endpoint" + - "kubeapi-load-balancer:website" + - - "kubeapi-load-balancer:certificates" + - "easyrsa:client" + - - "kubernetes-master:kube-api-endpoint" + - "kubernetes-worker:kube-api-endpoint" + - - "kubernetes-master:kube-control" + - "kubernetes-worker:kube-control" + - - "kubernetes-master:cni" + - "ovn:cni" + - - "kubernetes-worker:cni" + - "ovn:cni" + - - "etcd:certificates" + - "easyrsa:client" + - - "kubernetes-worker:certificates" + - "easyrsa:client" + - - "kubernetes-master:etcd" + - "etcd:db" + - - "kubernetes-master:certificates" + - "easyrsa:client" + +machines: + "0": + series: xenial + "1": + series: xenial + "2": + series: xenial + "3": + series: xenial \ No newline at end of file diff --git a/sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/copyright b/sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/copyright new file mode 100644 index 0000000..e04f1d3 --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/copyright @@ -0,0 +1,13 @@ +Copyright 2017 Aakash KT + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/interface.yaml b/sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/interface.yaml new file mode 100644 index 0000000..2ccf05b --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/interface.yaml @@ -0,0 +1,4 @@ +name : master-config +summary : Master config peer relation for OVN charm +version : 1 +maintainer : Aakash KT diff --git a/sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/peers.py b/sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/peers.py new file mode 100644 index 0000000..1cea378 --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/peers.py @@ -0,0 +1,189 @@ +import os +import json +import re +import sys +import subprocess +import time +import urllib.request as urllib2 +import multiprocessing as mp + +from charmhelpers.core import host + +from charmhelpers.core.hookenv import ( + open_port, + open_ports, + status_set, + config, + unit_public_ip, + unit_private_ip, +) + +from charmhelpers.core.host import ( + service_start, + service_stop, + log, + mkdir, + write_file, +) + +from charmhelpers.fetch import ( + apt_install, + apt_update, + apt_upgrade +) + +from charms.reactive.helpers import ( + mark_invoked, + was_invoked, +) + +from charms.reactive import ( + when, + when_not, + when_file_changed, + hook, + RelationBase, + scopes, + set_state, + remove_state +) + + + +CONF_FILE = '/tmp'; + + +######################################################################### +# Common functions +######################################################################### + +def run_command(command=None): + + if command is None: + return False; + + log('Running Command "%s"' % command); + try: + return subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT).decode('utf-8').replace('\n', ''); + except subprocess.CalledProcessError as e: + log('Error running "%s" : %s' % (command, e.output)); + + return False; + +def get_config(key): + conf = config(key); + return conf; + +def retrieve(key): + try: + conf = open('/tmp/ovn_conf', 'r'); + except: + return ''; + + plain_text = conf.read(); + conf.close(); + if plain_text == '': + return ''; + else: + data = json.loads(plain_text); + return data[key]; + +def store(key, value): + conf = open('/tmp/ovn_conf', 'r'); + plain_text = conf.read(); + conf.close(); + + conf = open('/tmp/ovn_conf', 'w+'); + + data = {}; + if plain_text != '': + data = json.loads(plain_text); + data[key] = value; + + conf.truncate(0); + conf.seek(0, 0); + conf.write(json.dumps(data)); + conf.close(); + + +######################################################################### +# Relation Class +######################################################################### + +class MasterConfigPeer(RelationBase): + + scope = scopes.UNIT; + + @hook("{peers:master-config}-relation-{joined}") + def joined(self): + conv = self.conversation(); + conv.set_state("{relation_name}.connected"); + + @hook("{peers:master-config}-relation-{changed}") + def changed(self): + conv = self.conversation(); + worker_id = conv.get_local(key='worker_id'); + + if worker_id != None and conv.get_remote(worker_id): + conv.set_state("{relation_name}.master.data.available"); + elif conv.get_remote('cert_to_sign'): + conv.set_state("{relation_name}.worker.cert.available"); + + @hook("{peers:master-config}-relation-{departed}") + def departed(self): + conv = self.conversation(); + + conv.remove_state("{relation_name}.connected"); + conv.remove_state("{relation_name}.master.data.available"); + conv.remove_state("{relation_name}.worker.cert.available"); + + def set_worker_id(self, worker_id): + convs = self.conversations(); + + for conv in convs: + conv.set_local(key='worker_id', value=worker_id); + + def get_worker_data(self): + convs = self.conversations(); + + final_data = []; + for conv in convs: + worker_unit = {}; + + cert = conv.get_remote('cert_to_sign'); + worker_hostname = conv.get_remote('worker_hostname'); + + worker_unit['cert_to_sign'] = cert; + worker_unit['worker_hostname'] = worker_hostname; + + final_data.append(worker_unit); + + return final_data; + + def send_worker_data(self, data): + convs = self.conversations(); + + for conv in convs: + conv.set_remote(data=data); + + + def send_signed_certs(self, certs): + convs = self.conversations(); + for conv in convs: + for key, value in certs.items(): + data_str = json.dumps(value); + conv.set_remote(key=key, value=data_str); + + def get_signed_cert(self, worker_hostname): + convs = self.conversations(); + + final = None; + for conv in convs: + data = conv.get_remote(worker_hostname); + + if data != '' and data != None: + data = json.loads(data); + final = data; + break; + + return final; \ No newline at end of file diff --git a/sourcecode/JOID/charm-k8s-ovn/layers/ovn/README.ex b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/README.ex new file mode 100644 index 0000000..f8f5bc0 --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/README.ex @@ -0,0 +1,21 @@ +# Overview + +This charm provides an SDN via the use of OVN and can be used with any principal charm implementing the [kubernetes-cni](https://github.com/juju-solutions/interface-kubernetes-cni) interface. + +# Usage + +This charm is subordinate. + + +juju deploy ovn +juju deploy kubernetes-master +juju deploy kubernetes-worker +juju add-relation ovn kubernetes-master +juju add-relation ovn kubernetes-worker + + +# Configuration + +The "gateway-physical-interface" option will allow you to choose an interface on which to create the gateway bridge. If unsure, leave it to "none" to use the default interface. + + diff --git a/sourcecode/JOID/charm-k8s-ovn/layers/ovn/config.yaml b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/config.yaml new file mode 100644 index 0000000..eb1fa21 --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/config.yaml @@ -0,0 +1,5 @@ +options: + gateway-physical-interface: + type: string + default: "none" + description: "The interface used by gateway nodes for north-south connectivity" diff --git a/sourcecode/JOID/charm-k8s-ovn/layers/ovn/copyright b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/copyright new file mode 100644 index 0000000..e04f1d3 --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/copyright @@ -0,0 +1,13 @@ +Copyright 2017 Aakash KT + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/sourcecode/JOID/charm-k8s-ovn/layers/ovn/icon.svg b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/icon.svg new file mode 100644 index 0000000..7517e0d --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/icon.svg @@ -0,0 +1,581 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + diff --git a/sourcecode/JOID/charm-k8s-ovn/layers/ovn/layer.yaml b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/layer.yaml new file mode 100644 index 0000000..44b8acf --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/layer.yaml @@ -0,0 +1,2 @@ +includes: ['layer:basic', 'interface:kubernetes-cni', 'interface:master-config'] +repo: 'https://github.com/AakashKT/charm-ovn.git' diff --git a/sourcecode/JOID/charm-k8s-ovn/layers/ovn/metadata.yaml b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/metadata.yaml new file mode 100644 index 0000000..767bb13 --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/metadata.yaml @@ -0,0 +1,19 @@ +name: ovn +summary: SDN via OVN +maintainer: Aakash +description: | + Overlay networking via OVN and OVS +tags: + - networking +subordinate: true +requires: + cni: + interface: kubernetes-cni + scope: container +peers: + master-config: + interface: master-config + +series: + - xenial + - trusty diff --git a/sourcecode/JOID/charm-k8s-ovn/layers/ovn/reactive/ovn.py b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/reactive/ovn.py new file mode 100644 index 0000000..675a3dd --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/reactive/ovn.py @@ -0,0 +1,491 @@ +import os +import json +import re +import sys +import subprocess +import time +import urllib.request as urllib2 +import multiprocessing as mp + +from charmhelpers.core import host + +from charmhelpers.core.hookenv import ( + open_port, + open_ports, + status_set, + config, + unit_public_ip, + unit_private_ip, +) + +from charmhelpers.core.host import ( + service_start, + service_stop, + log, + mkdir, + write_file, +) + +from charmhelpers.fetch import ( + apt_install, + apt_update, + apt_upgrade +) + +from charms.reactive.helpers import ( + mark_invoked, + was_invoked, +) + +from charms.reactive import ( + when, + when_not, + when_file_changed, + hook, + RelationBase, + scopes, + set_state, + remove_state +) + + + +CONF_FILE = '/tmp'; + + +######################################################################### +# Common functions +######################################################################### + +def run_command(command=None): + + if command is None: + return False; + + log('Running Command "%s"' % command); + try: + return subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT).decode('utf-8').replace('\n', ''); + except subprocess.CalledProcessError as e: + log('Error running "%s" : %s' % (command, e.output)); + + return False; + +def get_config(key): + conf = config(key); + return conf; + +def retrieve(key): + try: + conf = open('/tmp/ovn_conf', 'r'); + except: + return ''; + + plain_text = conf.read(); + conf.close(); + if plain_text == '': + return ''; + else: + data = json.loads(plain_text); + return data[key]; + +def store(key, value): + conf = open('/tmp/ovn_conf', 'r'); + plain_text = conf.read(); + conf.close(); + + conf = open('/tmp/ovn_conf', 'w+'); + + data = {}; + if plain_text != '': + data = json.loads(plain_text); + data[key] = value; + + conf.truncate(0); + conf.seek(0, 0); + conf.write(json.dumps(data)); + conf.close(); + + +######################################################################### +# Hooks and reactive handlers +######################################################################### + +''' Common reactive handlers ''' + +@when_not('deps.installed') +def install_deps(): + status_set('maintenance', 'Installing dependencies'); + + conf = open('/tmp/ovn_conf', 'w+'); + conf.close(); + + run_command('sudo apt-get update ; sudo apt-get upgrade ; sudo apt-get install git -y'); + run_command('sudo apt-get install -y build-essential fakeroot debhelper \ + autoconf automake bzip2 libssl-dev docker.io \ + openssl graphviz python-all procps \ + python-dev python-setuptools python-pip python3 python3.4 \ + python-twisted-conch libtool git dh-autoreconf \ + linux-headers-$(uname -r) libcap-ng-dev'); + run_command('sudo pip2 install six'); + + status_set('maintenance', 'Configure and make openvswitch'); + run_command('git clone https://github.com/openvswitch/ovs.git /tmp/ovs'); + + os.chdir('/tmp/ovs'); + + run_command('./boot.sh'); + run_command('./configure --prefix=/usr --localstatedir=/var --sysconfdir=/etc \ + --enable-ssl --with-linux=/lib/modules/`uname -r`/build'); + run_command('make -j3 ; sudo make install ; sudo make modules_install'); + + status_set('maintenance', 'Replacing kernel module'); + + run_command('sudo mkdir /etc/depmod.d/'); + run_command('for module in datapath/linux/*.ko; \ + do modname="$(basename ${module})" ; \ + echo "override ${modname%.ko} * extra" >> "/etc/depmod.d/openvswitch.conf" ; \ + echo "override ${modname%.ko} * weak-updates" >> "/etc/depmod.d/openvswitch.conf" ; \ + done'); + run_command('/sbin/modprobe openvswitch'); + + run_command('/usr/share/openvswitch/scripts/ovs-ctl start --system-id=$(uuidgen)'); + status_set('maintenance', 'Open vSwitch Installed'); + + run_command('git clone https://github.com/openvswitch/ovn-kubernetes /tmp/ovn-kubernetes'); + os.chdir('/tmp/ovn-kubernetes'); + run_command('sudo -H pip2 install .'); + + set_state('deps.installed'); + + + +''' Master reactive handlers and functions ''' + +def get_worker_subnet(): + ip3 = int(retrieve('ip3')); + store('ip3', ip3+1); + + return '192.168.%s.0/24' % ip3; + +@when('master.initialised') +def restart_services(): + new_interface = retrieve('new_interface'); + old_interface = retrieve('old_interface'); + + run_command('sudo ovn-k8s-watcher --overlay --pidfile \ + --log-file -vfile:info -vconsole:emer --detach'); + run_command('sudo ovn-k8s-gateway-helper --physical-bridge=%s \ + --physical-interface=%s \ + --pidfile --detach' % (new_interface, old_interface)); + +@when('master.initialised', 'master-config.worker.cert.available') +def sign_and_send(mconfig): + data = mconfig.get_worker_data(); + central_ip = get_my_ip(); + master_hostname = run_command('hostname'); + + signed_certs = {}; + for unit in data: + worker_hostname = unit['worker_hostname']; + + if not was_invoked(worker_hostname): + mark_invoked(worker_hostname); + cert = unit['cert_to_sign']; + worker_subnet = get_worker_subnet(); + + os.chdir('/tmp/'); + cert_file = open('/tmp/ovncontroller-req.pem', 'w+'); + cert_file.truncate(0); + cert_file.seek(0, 0); + cert_file.write(cert); + cert_file.close(); + run_command('sudo ovs-pki -d /certs/pki -b sign ovncontroller switch --force'); + + cert_file = open('ovncontroller-cert.pem', 'r'); + signed_cert = cert_file.read(); + + signed_certs[worker_hostname] = { + "central_ip": central_ip, + "signed_cert": signed_cert, + "master_hostname": master_hostname, + "worker_hostname": worker_hostname, + "worker_subnet": worker_subnet, + }; + + mconfig.send_signed_certs(signed_certs); + +@when('cni.is-master', 'master.initialised') +@when_not('gateway.installed') +def install_gateway(cni): + status_set('maintenance', 'Initialising gateway'); + + run_command('sudo ovs-vsctl set Open_vSwitch . external_ids:k8s-api-server="0.0.0.0:8080"'); + + run_command('git clone https://github.com/openvswitch/ovn-kubernetes /tmp/ovn-kubernetes'); + os.chdir('/tmp/ovn-kubernetes'); + run_command('sudo pip2 install .'); + + old_interface = get_interface(old=True); + new_interface = get_interface(old=False); + + op = run_command('ifconfig %s | grep "inet addr:"' % (new_interface)); + br_ip = op.lstrip().split()[1].replace('addr:', ''); + + gateway_ip = run_command('ip route | grep default').split(' ')[2]; + hostname = run_command('hostname'); + + op = run_command('ovn-k8s-overlay gateway-init \ + --cluster-ip-subnet="192.168.0.0/16" \ + --bridge-interface %s \ + --physical-ip %s/32 \ + --node-name="%s-gateway" \ + --default-gw %s' % (new_interface, br_ip, hostname, gateway_ip)); + log('Gateway init output: %s' % (op)); + + op = run_command('ovn-k8s-gateway-helper --physical-bridge=%s \ + --physical-interface=%s --pidfile --detach' % (new_interface, old_interface)); + log('Gateway Helper start: %s' % (op)); + + status_set('active', 'Master subnet : 192.168.1.0/24'); + set_state('gateway.installed'); + +@when('cni.is-master', 'master.setup.done') +@when_not('master.initialised') +def initialise_master(cni): + status_set('maintenance', 'Initialising master network'); + + central_ip = get_my_ip(); + hostname = run_command('hostname'); + + run_command('sudo ovs-vsctl set Open_vSwitch . external_ids:k8s-api-server="0.0.0.0:8080"'); + run_command('sudo ovn-k8s-overlay master-init --cluster-ip-subnet="192.168.0.0/16" \ + --master-switch-subnet="192.168.1.0/24" \ + --node-name="%s"' % (hostname)); + + run_command('sudo ovn-k8s-watcher --overlay --pidfile --log-file -vfile:info \ + -vconsole:emer --detach'); + + status_set('maintenance', 'Waiting for gateway'); + set_state('master.initialised'); + +@when('cni.is-master', 'bridge.setup.done') +@when_not('master.setup.done') +def master_setup(cni): + status_set('maintenance', 'Setting up master'); + open_port(6641); + open_port(6642); + open_port(8080); + + central_ip = get_my_ip(); + run_command('sudo /usr/share/openvswitch/scripts/ovn-ctl start_northd'); + run_command('sudo ovn-nbctl set-connection pssl:6641'); + run_command('sudo ovn-sbctl set-connection pssl:6642'); + + os.chdir('/etc/openvswitch'); + run_command('sudo ovs-pki -d /certs/pki init --force'); + run_command('sudo cp /certs/pki/switchca/cacert.pem /etc/openvswitch/'); + + run_command('sudo ovs-pki req ovnnb --force && sudo ovs-pki self-sign ovnnb --force'); + run_command('sudo ovn-nbctl set-ssl /etc/openvswitch/ovnnb-privkey.pem \ + /etc/openvswitch/ovnnb-cert.pem /certs/pki/switchca/cacert.pem'); + + run_command('sudo ovs-pki req ovnsb --force && sudo ovs-pki self-sign ovnsb --force'); + run_command('sudo ovn-sbctl set-ssl /etc/openvswitch/ovnsb-privkey.pem \ + /etc/openvswitch/ovnsb-cert.pem /certs/pki/switchca/cacert.pem'); + + run_command('sudo ovs-pki req ovncontroller'); + run_command('sudo ovs-pki -b -d /certs/pki sign ovncontroller switch --force'); + + ovn_host_file = open('/etc/default/ovn-host', 'a'); + ovn_host_file.write('OVN_CTL_OPTS="--ovn-controller-ssl-key=/etc/openvswitch/ovncontroller-privkey.pem \ + --ovn-controller-ssl-cert=/etc/openvswitch/ovncontroller-cert.pem \ + --ovn-controller-ssl-bootstrap-ca-cert=/etc/openvswitch/ovnsb-ca.cert"'); + ovn_host_file.close(); + + run_command('sudo ovs-vsctl set Open_vSwitch . external_ids:ovn-remote="ssl:%s:6642" \ + external_ids:ovn-nb="ssl:%s:6641" \ + external_ids:ovn-encap-ip=%s \ + external_ids:ovn-encap-type="%s"' % (central_ip, central_ip, central_ip, 'geneve')); + run_command('sudo /usr/share/openvswitch/scripts/ovn-ctl \ + --ovn-controller-ssl-key="/etc/openvswitch/ovncontroller-privkey.pem" \ + --ovn-controller-ssl-cert="/etc/openvswitch/ovncontroller-cert.pem" \ + --ovn-controller-ssl-bootstrap-ca-cert="/etc/openvswitch/ovnsb-ca.cert" \ + restart_controller'); + + set_state('master.setup.done'); + +@when('cni.is-master', 'master.kv.setup') +@when_not('bridge.setup.done') +def bridge_setup(cni): + status_set('maintenance', 'Setting up new interface'); + + interface = get_config('gateway-physical-interface'); + if interface == 'none' or interface == None: + op = run_command('ip route | grep default').split(' '); + interface = op[4]; + + store('old_interface', interface); + store('new_interface', 'br%s' % (interface)); + + op = run_command('ovn-k8s-util nics-to-bridge %s' % (interface)); + log('Bridge create output: %s' % (op)); + + op = run_command('dhclient -r br%s' % (interface)); + op = run_command('dhclient br%s' % (interface)); + + status_set('maintenance', 'Waiting to initialise master'); + set_state('bridge.setup.done'); + +@when('cni.is-master', 'deps.installed') +@when_not('master.kv.setup') +def setup_master_kv(cni): + store('ip3', '2'); + + set_state('master.kv.setup'); + + +''' Worker reactive handlers and functions ''' + +@when('cni.is-worker', 'worker.data.registered') +@when_not('k8s.worker.certs.setup') +def setup_k8s_worker_certs(cni): + if os.path.isfile('/root/cdk/kubeconfig') and os.path.isfile('/root/cdk/ca.crt'): + set_state('k8s.worker.certs.setup'); + + master_hostname = retrieve('master_hostname'); + + k8s_api_ip = "%s:6443" % (master_hostname); + api_token = run_command('sudo awk \'$1=="token:" {print $2}\' /root/cdk/kubeconfig'); + + run_command('sudo cp /root/cdk/ca.crt /etc/openvswitch/k8s-ca.crt'); + run_command('sudo ovs-vsctl set Open_vSwitch . \ + external_ids:k8s-api-server="https://%s" \ + external_ids:k8s-api-token="%s"' % (k8s_api_ip, api_token)); + + +@when('cni.is-worker', 'worker.setup.done') +@when_not('worker.initialised') +def initialise_worker(cni): + status_set('maintenance', 'Initialising worker network'); + + local_ip = get_my_ip(); + worker_subnet = retrieve('worker_subnet'); + central_ip = retrieve('central_ip'); + hostname = run_command('hostname').replace('\n', ''); + + run_command('ovs-vsctl set Open_vSwitch . \ + external_ids:k8s-api-server="%s:8080"' % (central_ip)); + run_command('ovn-k8s-overlay minion-init --cluster-ip-subnet="192.168.0.0/16" \ + --minion-switch-subnet="%s" --node-name="%s"' % (worker_subnet, hostname)); + + os.chdir('/tmp/'); + run_command('wget https://github.com/containernetworking/cni/releases/download/v0.5.2/cni-amd64-v0.5.2.tgz'); + run_command('sudo mkdir -p /opt/cni/bin'); + run_command('sudo mkdir -p /etc/cni/net.d'); + os.chdir('/opt/cni/bin/'); + run_command('sudo tar xvzf /tmp/cni-amd64-v0.5.2.tgz'); + + status_set('active', 'Worker subnet : %s' % (worker_subnet)); + set_state('worker.initialised'); + +@when('cni.is-worker', 'worker.data.registered') +@when_not('worker.setup.done') +def worker_setup(cni): + status_set('maintenance', 'Setting up worker'); + open_port(8080); + + central_ip = retrieve('central_ip'); + local_ip = get_my_ip(); + + run_command('sudo ovs-vsctl set Open_vSwitch . \ + external_ids:ovn-remote="ssl:%s:6642" \ + external_ids:ovn-nb="ssl:%s:6641" \ + external_ids:ovn-encap-ip=%s \ + external_ids:ovn-encap-type=%s' % (central_ip, central_ip, local_ip, 'geneve')); + + ovn_host_file = open('/etc/default/ovn-host', 'a'); + ovn_host_file.write('OVN_CTL_OPTS="--ovn-controller-ssl-key=/etc/openvswitch/ovncontroller-privkey.pem \ + --ovn-controller-ssl-cert=/etc/openvswitch/ovncontroller-cert.pem \ + --ovn-controller-ssl-bootstrap-ca-cert=/etc/openvswitch/ovnsb-ca.cert"'); + ovn_host_file.close(); + + run_command('sudo /usr/share/openvswitch/scripts/ovn-ctl \ + --ovn-controller-ssl-key="/etc/openvswitch/ovncontroller-privkey.pem" \ + --ovn-controller-ssl-cert="/etc/openvswitch/ovncontroller-cert.pem" \ + --ovn-controller-ssl-bootstrap-ca-cert="/etc/openvswitch/ovnsb-ca.cert" \ + restart_controller'); + set_state('worker.setup.done'); + +@when('cni.is-worker', 'master-config.master.data.available', 'worker.cert.sent') +@when_not('worker.data.registered') +def receive_data(cni, mconfig): + status_set('maintenance', 'Certificate received') + worker_hostname = run_command('hostname'); + + data = mconfig.get_signed_cert(worker_hostname); + cert = data['signed_cert']; + worker_subnet = data['worker_subnet']; + master_ip = data['central_ip']; + master_hostname = data['master_hostname']; + + store('master_hostname', master_hostname); + store('worker_subnet', worker_subnet); + store('central_ip', master_ip); + cni.set_config(cidr='192.168.0.0/16'); + + os.chdir('/etc/openvswitch'); + cert_file = open('/etc/openvswitch/ovncontroller-cert.pem', 'a'); + cert_file.write(cert); + cert_file.close(); + + set_state('worker.data.registered'); + +@when('cni.is-worker', 'master-config.connected', 'worker.kv.setup') +@when_not('worker.cert.sent') +def send_cert(cni, mconfig): + worker_hostname = run_command('hostname'); + mconfig.set_worker_id(worker_hostname); + + os.chdir('/etc/openvswitch'); + run_command('sudo ovs-pki req ovncontroller'); + + req_file = open('ovncontroller-req.pem', 'r'); + cert = req_file.read(); + mconfig.send_worker_data({ + 'cert_to_sign': cert, + 'worker_hostname': worker_hostname + }); + + status_set('maintenance', 'Waiting for certificate'); + set_state('worker.cert.sent'); + +@when('cni.is-worker', 'deps.installed') +@when_not('worker.kv.setup') +def setup_worker_kv(cni): + hostname = run_command('hostname'); + interface = get_config('gateway-physical-interface'); + if interface == 'none' or interface == None: + op = run_command('ip route | grep default').split(' '); + interface = op[4]; + + store('new_interface', interface); + store('worker_hostname', hostname); + set_state('worker.kv.setup'); + + +######################################################################### +# Helper functions +######################################################################### + + +def get_my_ip(): + interface = get_interface(old=False); + + op = run_command('ifconfig %s | grep "inet addr:"' % (interface)); + br_ip = op.lstrip().split()[1].replace('addr:', ''); + + return br_ip; + +def get_interface(old): + key = 'old_interface' if old == True else 'new_interface'; + return retrieve(key); diff --git a/sourcecode/JOID/charm-k8s-ovn/layers/ovn/tests/00-setup b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/tests/00-setup new file mode 100755 index 0000000..f0616a5 --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/tests/00-setup @@ -0,0 +1,5 @@ +#!/bin/bash + +sudo add-apt-repository ppa:juju/stable -y +sudo apt-get update +sudo apt-get install amulet python-requests -y diff --git a/sourcecode/JOID/charm-k8s-ovn/layers/ovn/tests/10-deploy b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/tests/10-deploy new file mode 100755 index 0000000..1cd905f --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/tests/10-deploy @@ -0,0 +1,35 @@ +#!/usr/bin/python3 + +import amulet +import requests +import unittest + + +class TestCharm(unittest.TestCase): + def setUp(self): + self.d = amulet.Deployment() + + self.d.add('OVN') + self.d.expose('OVN') + + self.d.setup(timeout=900) + self.d.sentry.wait() + + self.unit = self.d.sentry['OVN'][0] + + def test_service(self): + # test we can access over http + page = requests.get('http://{}'.format(self.unit.info['public-address'])) + self.assertEqual(page.status_code, 200) + # Now you can use self.d.sentry[SERVICE][UNIT] to address each of the units and perform + # more in-depth steps. Each self.d.sentry[SERVICE][UNIT] has the following methods: + # - .info - An array of the information of that unit from Juju + # - .file(PATH) - Get the details of a file on that unit + # - .file_contents(PATH) - Get plain text output of PATH file from that unit + # - .directory(PATH) - Get details of directory + # - .directory_contents(PATH) - List files and folders in PATH on that unit + # - .relation(relation, service:rel) - Get relation data from return service + + +if __name__ == '__main__': + unittest.main() -- cgit 1.2.3-korg