diff options
31 files changed, 2045 insertions, 60 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..71d7636 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.tox +docs/_build/* @@ -1,26 +0,0 @@ -Project: OVN for NFV (OVN4NFV) -Project Creation Date: Feb 7th 2017 -Project Category: Collaborative Development -Lifecycle State: Incubation -Primary Contact: Vikram Dham (Huawei Research) -Project Lead: Vikram Dham (Huawei Research) -Jira Name: OVN for NFV -Jira Prefix: [OVN4NFV] -mailing list tag [ovn4nfv] -Repo: ovn4nfv - -Committers: -vikram.dham@huawei.com -trozet@redhat.com -narinder.gupta@canonical.com -chigang@huawei.com -trinath.somanchi@nxp.com -sridhar.pothuganti@nxp.com - -Contributors: -prakash.ramchandran@huawei.com -prasad.gorja@nxp.com - - -Link to TSC approval: http://meetbot.opnfv.org/meetings/opnfv-meeting/2015/opnfv-meeting.2015-12-15-14.59.html -Link to approval of additional submitters: diff --git a/INFO.yaml b/INFO.yaml new file mode 100644 index 0000000..d604879 --- /dev/null +++ b/INFO.yaml @@ -0,0 +1,48 @@ +--- +project: 'OVN for NFV (OVN4NFV)' +project_creation_date: 'Feb 7th 2017' +project_category: 'Collaborative Development' +lifecycle_state: 'Incubation' +project_lead: &opnfv_ovn4nfv_ptl + name: 'Trinath Somanchi' + email: 't.somanchi@f5.com' + company: 'F5 Networks' + id: 'SomanchiTrinath' +primary_contact: *opnfv_ovn4nfv_ptl +issue_tracking: + type: 'jira' + url: 'https://jira.opnfv.org/projects/OVN4NFV' + key: 'OVN4NFV' +mailing_list: + type: 'mailman2' + url: 'opnfv-tech-discuss@lists.opnfv.org' + tag: '[OVN4NFV]' +realtime_discussion: + type: irc + server: 'freenode.net' + channel: '#opnfv-ovn4nfv' +meetings: + - type: 'irc' + agenda: 'https://wiki.opnfv.org/display/OV/Meeting+Agenda' + url: # eg: 'https://global.gotomeeting.com/join/819733085' + server: 'freenode.net' + channel: '#ovn4nfv-meeting' + repeats: 'weekly' + time: '15:30 UTC' +repositories: + - 'ovn4nfv' + - 'ovn4nfv-k8s-plugin' +committers: + - <<: *opnfv_ovn4nfv_ptl + - name: 'Trevor Bramwell' + email: 'tbramwell@linuxfoundation.org' + company: 'linuxfoundation.org' + id: 'bramwelt' + - name: 'Ritu Sood' + email: 'ritu.sood@intel.com' + company: 'intel.com' + id: 'ritusood' +tsc: + # yamllint disable rule:line-length + approval: 'http//meetbot.opnfv.org/meetings/opnfv-meeting/2015/opnfv-meeting.2015-12-15-14.59.html' + # yamllint enable rule:line-length diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..3c4453e --- /dev/null +++ b/docs/conf.py @@ -0,0 +1 @@ +from docs_conf.conf import * diff --git a/docs/conf.yaml b/docs/conf.yaml new file mode 100644 index 0000000..7611686 --- /dev/null +++ b/docs/conf.yaml @@ -0,0 +1,3 @@ +--- +project_cfg: opnfv +project: OVN4NFV diff --git a/docs/development/index.rst b/docs/development/index.rst index 5cf12b5..32be210 100644 --- a/docs/development/index.rst +++ b/docs/development/index.rst @@ -1,3 +1,5 @@ +.. _ovn4nfv-developmentguide: + .. This work is licensed under a Creative Commons Attribution 4.0 International .. License. http://creativecommons.org/licenses/by/4.0 @@ -7,10 +9,10 @@ OVN4NFV Development :Project: Ovn4Nfv, https://wiki.opnfv.org/display/PROJ/Ovn4nfv -:Editors: Vikram Dham (Huawei) -:Authors: Vikram Dham (Huawei) - Trinath Somanchi (NXP) - Prakash Ramchandran (Huawei) +:Editors: Trinath Somanchi (NXP India) +:Authors: Prasad Gorja (NXP India) + Trinath Somanchi (NXP India) + Prakash Ramchandran (Dell) :Abstract: This document provides an overview of networking-ovn,its utilization into NFV and outlines OpenStack official release. @@ -20,3 +22,4 @@ OVN4NFV Development openstack-networking-ovn.rst ovn-for-nfv.rst + ovn-sfc-openstack.rst diff --git a/docs/development/ovn-sfc-openstack.rst b/docs/development/ovn-sfc-openstack.rst new file mode 100644 index 0000000..2d29c5a --- /dev/null +++ b/docs/development/ovn-sfc-openstack.rst @@ -0,0 +1,201 @@ +=================== +OVN-SFC POC Details +=================== + +Purpose +======= +The purpose of this Proof-of-concept is to showcase Service Function +Chaining with OVN. + +Scope +===== + +The Scope of this document is to describe SFC using OVN and discuss +installation and configuration of OVN to instantiate a forwarding path. + +Steps +===== +1. Install CentOS7 minimal install: +----------------------------------- +- Make sure to enable network interface. +- Just create a root password. Don't create any users + +2. Create user: +--------------- +Below are the instructions to create user - stack, for use with Devstack. + +- $ sudo useradd -s /bin/bash -d /opt/stack -m stack +- $ echo 'stack ALL=(ALL) NOPASSWD: ALL' | sudo tee /etc/sudoers.d/stack +- $ sudo su - stack + +3. Install git +-------------- +- $ sudo yum install git -y + +4. clone Devstack and Networking-ovn +------------------------------------ +- $ git clone http://git.openstack.org/openstack-dev/devstack.git +- $ git clone http://git.openstack.org/openstack/networking-ovn.git +- $ cd devstack +- $ cp ../networking-ovn/devstack/local.conf.sample local.conf + +5. Edit the local.conf file: +---------------------------- +- Add (uncomment and edited) + + - OVN_REPO=https://github.com/doonhammer/ovs + - OVN_BRANCH=sfc.v30 +- Uncomment the below line + + - OVN_BUILD_MODULES=False + +We use forked/modifed OVS for SFC usecase from John McDowall. + +6. Devstack Preliminaries: +-------------------------- +- $ ./stack.sh +- $ . ~/devstack/openrc admin +- $ openstack keypair create demo &gt; ~/id_rsa_demo +- $ chmod 600 ~/id_rsa_demo +- $ for group in $(openstack security group list -f value -c ID); + do openstack security group rule create --ingress --ethertype IPv4 --dst-port 22 --protocol tcp $group; + openstack security group rule create --ingress --ethertype IPv4 -- protocol ICMP $group; + done +- $ IMAGE_ID=$(openstack image list -f value -c ID) + +10. Create Neutron network and subnet +-------------------------------------- +- $ openstack network create --project admin --provider-network-type geneve n1 +- $ openstack subnet create --subnet-range 10.1.1.0/24 --network n1 n1subnet + + +10. Spawn VMs +------------- +- Create 5 VMs, 3 VMs to act as communication end-points (a,b, and c) and two + VMs to act as VNFs (vnf1 & vnf2). +- The 2 VNF VMs are created with two NICs to act as ingress and egress ports + (Optional) +- Created two SFCs: + - SFC1: any traffic from VM a to VM b will go through vnf1 + - SFC1: any traffic from VM a to VM c will go through vnf2 then vnf1 + + +A. SFC with OVN - Scenario 1: +----------------------------- + +*********************** +1. create VMs and VNFs: +*********************** +- $ openstack server create --nic net-id=n1,v4-fixed-ip=10.1.1.5 --flavor m1.nano --image $IMAGE_ID --key-name demo a +- $ openstack server create --nic net-id=n1,v4-fixed-ip=10.1.1.6 --flavor m1.nano --image $IMAGE_ID --key-name demo b +- $ openstack server create --nic net-id=n1,v4-fixed-ip=10.1.1.10 --nic net-id=n1,v4-fixed-ip=10.1.1.11 --flavor m1.nano --image $IMAGE_ID --key-name demo vnf1 +- $ openstack port set --name ap $(openstack port list --server a -f value -c ID) +- $ openstack port set --name bp $(openstack port list --server b -f value -c ID) +- $ AP_MAC=$(openstack port show -f value -c mac_address ap) +- $ BP_MAC=$(openstack port show -f value -c mac_address bp) +- $ openstack port set --name vnf1-pin $(openstack port list --server vnf1 --mac-address fa:16:3e:a0:e9:70 -f value -c ID) +- $ openstack port set --name vnf1-pout $(openstack port list --server vnf1 --mac-address fa:16:3e:ae:0c:36 -f value -c ID) +- $ f1_pin_MAC=$(openstack port show -f value -c mac_address vnf1-pin) +- $ f1_pout_MAC=$(openstack port show -f value -c mac_address vnf1-pout) + +*************************************** +2. Create port-pairs, groups and chains +*************************************** +The switch and ports UUIDs below will different in each environment. + +- n1 = f1de57df-04e3-456b-85c0-64fd869507ad +- vnf1-pin = 6ec5aa3d-8440-44c9-acf3-a18914ca9b0d +- vnf1-pout = 3f558a9d-295e-4417-9646-d46b59be97d8 +- ap = 0438495b-7de4-4bbb-b787-dff82615b541 +- bp = 1f004846-3f38-450d-8f4a-e5ed0f7228e6 +- cp = 9a72cc76-4d8d-494c-a959-8d672149c0ea +- vnf2-pin = 6a32edc7-23d4-42ed-9cf8-c6e0009da01d +- vnf2-pout = 8553b6d2-1433-4ab4-ab69-704d318b09af + +**1. Configure the port pair vnf1-PP1** + +- $ ovn-nbctl lsp-pair-add n1 vnf1-pin vnf1-pout vnf1-PP1 (didn't work with names) +- $ ovn-nbctl lsp-pair-add f1de57df-04e3-456b-85c0-64fd869507ad 6ec5aa3d-8440-44c9-acf3-a18914ca9b0d 3f558a9d-295e-4417-9646-d46b59be97d8 vnf1-PP1 + +**2. Configure the port chain PC1** + +- $ ovn-nbctl lsp-chain-add n1 PC1 +- $ ovn-nbctl lsp-chain-add f1de57df-04e3-456b-85c0-64fd869507ad PC1 + +**3. Configure the port pair group PG1 and add to port chain** + +- $ ovn-nbctl lsp-pair-group-add PC1 PG1 + +**4. Add port pair to port chain** + +- $ ovn-nbctl lsp-pair-group-add-port-pair PG1 vnf1-PP1 + +**5. Add port chain to port classifier PCC1** + +- $ lsp-chain-classifier-add SWITCH CHAIN PORT DIRECTION PATH [NAME] [MATCH] +- $ ovn-nbctl lsp-chain-classifier-add n1 PC1 bp 'entry-lport' 'bi-directional' PCC1 ''; +- $ ovn-nbctl lsp-chain-classifier-add f1de57df-04e3-456b-85c0-64fd869507ad PC1 1f004846-3f38-450d-8f4a-e5ed0f7228e6 'entry-lport' 'bi-directional' PCC1 '' + +***************** +3. Validating SFC +***************** + +- $ ovn-trace n1 'inport == "ap" && eth.src == "$AP_MAC" && eth.dst == "$BP_MAC"' + + +B. SFC with OVN - Scenario 2: +----------------------------- + +************* +1. Create VMs +************* +- $ openstack server create --nic net-id=n1,v4-fixed-ip=10.1.1.7 --flavor m1.nano --image $IMAGE_ID --key-name demo c +- $ openstack server create --nic net-id=n1,v4-fixed-ip=10.1.1.20 --nic net-id=n1,v4-fixed-ip=10.1.1.21 --flavor m1.nano --image $IMAGE_ID --key-name demo vnf2 +- $ openstack port set --name cp $(openstack port list --server c -f value -c ID) +- $ CP_MAC=$(openstack port show -f value -c mac_address cp) +- $ openstack port set --name vnf2-pin $(openstack port list --server vnf2 --mac-address fa:16:3e:ff:e5:76 -f value -c ID) +- $ openstack port set --name vnf2-pout $(openstack port list --server vnf2 --mac-address fa:16:3e:4c:a3:58 -f value -c ID) +- $ f2_pin_MAC=$(openstack port show -f value -c mac_address vnf2-pin) +- $ f2_pout_MAC=$(openstack port show -f value -c mac_address vnf2-pout) + +**************** +2. Configure SFC +**************** + +**1. Configure the port pair vnf2-PP1** + +- $ ovn-nbctl lsp-pair-add n1 vnf2-pin vnf2-pout vnf2-PP1 (Didn't work with names) +- $ ovn-nbctl lsp-pair-add f1de57df-04e3-456b-85c0-64fd869507ad 6a32edc7-23d4-42ed-9cf8-c6e0009da01d 8553b6d2-1433-4ab4-ab69-704d318b09af vnf2-PP1 + +**2. Configure the port chain PC2** + +- $ ovn-nbctl lsp-chain-add n1 PC2 +- $ ovn-nbctl lsp-chain-add f1de57df-04e3-456b-85c0-64fd869507ad PC2 + +**3. Configure the port pair group PG2 and add to port chain** + +- $ ovn-nbctl lsp-pair-group-add PC2 PG2 +- $ ovn-nbctl lsp-pair-group-add PC2 PG3 + +**4. Add port pair to port chain** + +- $ ovn-nbctl lsp-pair-group-add-port-pair PG2 vnf2-PP1 +- $ ovn-nbctl lsp-pair-group-add-port-pair PG3 vnf1-PP1 + +**4. Add port chain to port classifier PCC2** + +- $ ovn-nbctl lsp-chain-classifier-add n1 PC2 cp "entry-lport" "bi-directional" PCC2 "" +- $ ovn-nbctl lsp-chain-classifier-add f1de57df-04e3-456b-85c0-64fd869507ad PC2 9a72cc76-4d8d-494c-a959-8d672149c0ea "entry-lport" "bi-directional" PCC2 ""; + +******************** +3. Validate Scenario +******************** + +- $ ovn-trace n1 'inport == "ap" && eth.src == "$AP_MAC" && eth.dst == "$CP_MAC"' + +References: +----------- + +1. http://docs.openvswitch.org/en/latest/tutorials/ovn-openstack/ +2. https://gist.github.com/voyageur/a26943eced3324b302f1ffede45252bd +3. https://github.com/doonhammer/ovs diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..50b855d --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,20 @@ +.. _ovn4nfv: + +.. This work is licensed under a Creative Commons Attribution 4.0 +.. International License. +.. SPDX-License-Identifier: CC-BY-4.0 +.. (c) Open Platform for NFV Project, Inc. and its contributors + +======= +OVN4NFV +======= + +.. toctree:: + :numbered: + :maxdepth: 2 + + development/index + release/configguide/index + release/release-notes/index + scenarios/index + testing/index diff --git a/docs/release/configuration-guide.rst b/docs/release/configguide/configuration-guide.rst index 12a6243..12a6243 100644 --- a/docs/release/configuration-guide.rst +++ b/docs/release/configguide/configuration-guide.rst diff --git a/docs/release/index.rst b/docs/release/configguide/index.rst index 24da448..ddc7f6e 100644 --- a/docs/release/index.rst +++ b/docs/release/configguide/index.rst @@ -1,21 +1,22 @@ +.. _ovn4nfv-configguide: + .. This work is licensed under a Creative Commons Attribution 4.0 International .. License. http://creativecommons.org/licenses/by/4.0 -========================== -OVN4NFV Release Notes -========================== +=========================== +OVN4NFV Configuration guide +=========================== :Project: OVN4NFV, https://wiki.opnfv.org/display/PROJ/Ovn4nfv -:Editors: Vikram Dham (Huawei) -:Authors: Vikram Dham (Huawei) +:Editors: Trinath Somanchi (NXP) +:Authors: Prasad Gorja (NXP) Trinath Somanchi (NXP) - Prakash Ramchandran (Huawei) + Prakash Ramchandran (Dell) -:Abstract: OVN4NFV Release Notes. +:Abstract: OVN4NFV Configuration Guide .. toctree:: :maxdepth: 1 - release-notes.rst configuration-guide.rst diff --git a/docs/release/release-notes/index.rst b/docs/release/release-notes/index.rst new file mode 100644 index 0000000..4c305f3 --- /dev/null +++ b/docs/release/release-notes/index.rst @@ -0,0 +1,23 @@ +.. _ovn4nfv-releasenotes: + +.. This work is licensed under a Creative Commons Attribution 4.0 International +.. License. http://creativecommons.org/licenses/by/4.0 + +============================== +OVN4NFV - Fraser Release Notes +============================== + +:Project: OVN4NFV, https://wiki.opnfv.org/display/PROJ/Ovn4nfv + +:Editors: Trinath Somanchi (NXP India) +:Authors: Prasad Gorja (NXP India) + Trinath Somanchi (NXP India) + Prakash Ramchandran (Dell) + Aakash K T (IIIT Hyderabad, INDIA) + +:Abstract: OVN4NFV Release Notes. + +.. toctree:: + :maxdepth: 1 + + release-notes.rst diff --git a/docs/release/release-notes.rst b/docs/release/release-notes/release-notes.rst index b8b8a65..3c4acfe 100644 --- a/docs/release/release-notes.rst +++ b/docs/release/release-notes/release-notes.rst @@ -1,3 +1,5 @@ +.. _ovn4nfv-releasenotes: + .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 .. (c) Open Platform for NFV Project, Inc. and its contributors @@ -6,18 +8,18 @@ Abstract ======== -This document compiles the release notes for the Euphrates release of -OPNFV when using Apex/Joid as a deployment tool. +This document compiles the release notes for OPNFV release when using Apex, +Joid and Fuel as a deployment tool. =============== Important Notes =============== -These notes provides release information for the use of Apex/Joid as deployment -tool for the Euphrates release of OPNFV. +This notes provides release information for the use of Apex, Joid and Fuel +as deployment tool for the Gambia release of OPNFV. -The goal of the Euphrates release and this Joid/Apex-based deployment process is -to establish a lab ready platform accelerating further development +The goal of the Gambia release and this Apex and Fuel based deployment +process is to establish a lab ready platform accelerating further development of the OPNFV infrastructure. Carefully follow the installation-instructions. @@ -26,9 +28,9 @@ Carefully follow the installation-instructions. Summary ======= -For Euphrates, OVN4NFV is supported with APEX and JOID installers. +For Gambia release, OVN4NFV is supported with APEX and FUEL installers. -This Euphrates artifact provides Apex/Joid as the deployment stage tool in the +This Gambia artifact provides Apex and Fuel as the deployment stage tool in the OPNFV CI pipeline including: - Documentation built by Jenkins @@ -39,7 +41,7 @@ OPNFV CI pipeline including: - installation instructions -- Automated validation of the Euphrates deployment. +- Automated validation of the Fraser deployment. ============ Release Data @@ -49,16 +51,18 @@ Release Data | **Project** | ovn4nfv | | | | +--------------------------------------+--------------------------------------+ -| **Repo/tag** | opnfv-5.0.0 | +| **Repo/tag** | ovn4nfv/opnfv-7.0 | | | | +--------------------------------------+--------------------------------------+ -| **Release designation** | Euphrates 5.1 | +| **Release designation** | Gambia 7.0 | | | | +--------------------------------------+--------------------------------------+ -| **Release date** | December 15 2017 | +| **Release date** | November 09 2018 | | | | +--------------------------------------+--------------------------------------+ -| **Purpose of the delivery** | Bug fixes | +| **Purpose of the delivery** | New Scenario PoC: OVN SFC | +| | Scenario documentation, OVN-CN-VM | +| | OVN support in Fuel installer. | +--------------------------------------+--------------------------------------+ @@ -70,17 +74,20 @@ Software Deliverables - `Apex based installation <https://git.opnfv.org/apex>`_ -- `Joid based installation <https://git.opnfv.org/joid>`_ +- `Fuel based installation <https://git.opnfv.org/fuel>`_ Documentation Deliverables -------------------------- -- `Installation instructions <http://docs.opnfv.org/en/stable-euphrates/submodules/ovn4nfv/docs/development/openstack-networking-ovn.html#install-configuration>`_ +- `Installation instructions <https://git.opnfv.org/ovn4nfv/tree/docs/development/openstack-networking-ovn.rst?h=stable/gambia>`_ - Release notes (This document) -- `User guide and Testing notes <http://docs.opnfv.org/en/stable-euphrates/submodules/ovn4nfv/docs/testing/testing-notes.html>`_ +- `User guide and Testing notes <https://git.opnfv.org/ovn4nfv/tree/docs/testing/testing-notes.rst?h=stable/gambia>`_ + +- `Scenario Documentation (K8S-OVN) <https://git.opnfv.org/ovn4nfv/tree/docs/scenarios/JOID/k8s-ovn-lb-noha.rst?h=stable/gambia>`_ +- `Scenario Documentation (OVN-SFC) <https://git.opnfv.org/ovn4nfv/tree/docs/development/ovn-sfc-openstack.rst?h=stable/gambia>`_ ========== References @@ -93,7 +100,7 @@ OPNFV 1) `OPNFV Home Page <http://www.opnfv.org>`_ 2) `OPNFV Documentation <http://docs.opnfv.org>`_ 3) `OPNFV Software Downloads <https://www.opnfv.org/software/download>`_ -4) `OVN4NFV Project <>`_ +4) `OVN4NFV Project <https://wiki.opnfv.org/display/PROJ/Ovn4nfv>`_ OpenStack ========= diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..9fde2df --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +lfdocs-conf +sphinx_opnfv_theme diff --git a/docs/scenarios/JOID/k8s-ovn-lb-noha.rst b/docs/scenarios/JOID/k8s-ovn-lb-noha.rst new file mode 100644 index 0000000..ea9a35e --- /dev/null +++ b/docs/scenarios/JOID/k8s-ovn-lb-noha.rst @@ -0,0 +1,171 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) <optionally add copywriters name> + + +Abstract +======== + +This document outlines the notes for deploying Kubernetes with OVN as the SDN and a load +balancer. JOID is used as the deployment tool. + +Introduction +============ +Juju OPNFV Infrastructure Deployer (JOID) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +JOID as Juju OPNFV Infrastructure Deployer allows you to deploy different combinations of +OpenStack release and SDN solution in HA or non-HA mode. For OpenStack, JOID supports +Juno and Liberty. For SDN, it supports Openvswitch, OpenContrail, OpenDayLight, and ONOS. +In addition to HA or non-HA mode, it also supports deploying from the latest development +tree. + +JOID heavily utilizes the technology developed in Juju and MAAS. Juju is a +state-of-the-art, open source, universal model for service oriented architecture and +service oriented deployments. Juju allows you to deploy, configure, manage, maintain, +and scale cloud services quickly and efficiently on public clouds, as well as on physical +servers, OpenStack, and containers. You can use Juju from the command line or through its +powerful GUI. MAAS (Metal-As-A-Service) brings the dynamism of cloud computing to the +world of physical provisioning and Ubuntu. Connect, commission and deploy physical servers +in record time, re-allocate nodes between services dynamically, and keep them up to date; +and in due course, retire them from use. In conjunction with the Juju service +orchestration software, MAAS will enable you to get the most out of your physical hardware +and dynamically deploy complex services with ease and confidence. + +For more info on Juju and MAAS, please visit https://jujucharms.com/ +and http://maas.ubuntu.com. + +Production grade container Orchestrator - Kubernetes (K8S) +========================================================== + +Kubernetes is an open-source system for automating deployment, scaling, and +management of containerized applications. + +This is a Kubernetes cluster that includes logging, monitoring, and operational +knowledge. It is comprised of the following components and features: + +- Kubernetes (automated deployment, operations, and scaling) + TLS used for communication between nodes for security. + A CNI plugin - in this scenario, we use OVN (Refer : https://github.com/openvswitch/ovn-kubernetes) + A load balancer for HA kubernetes-master (Experimental) + Optional Ingress Controller (on worker) + Optional Dashboard addon (on master) including Heapster for cluster monitoring + +- EasyRSA + Performs the role of a certificate authority serving self signed certificates + to the requesting units of the cluster. + +- Etcd (distributed key value store) + Minimum Three node cluster for reliability. + +- Kubernetes deployment with Load Balancer + .. code-block:: sh + + ./deploy.sh -m kubernetes -f lb -l custom -s ovn + + +Deployment Diagram +================== + + + + +-------------------------+ + | | + +------------+ Kubeapi-load-balancer +-----------------+ + | | | | + | +------------+------------+ | + | | | + | | | + | +------+------+ +----------+ | + | | | | | | + | +--------------+ EasyRSA +--------+ etcd | | + | | | | | | | + | | +-------------+ +-----+----+ | + | | | | + | | +---------+ | | + | | | | | | + | | +-------------+ OVN +-------------+ | | + | | | | | | | | + | | | +---------+ | | | + | | | | | | + | | | | | | ++-+---+--+----------+ +-----------+--+------+ | +| | | | | +| Kubernetes-worker +-----------------Kubernetes-master +-+ +| | | | ++-------------------+ +---------------------+ + + + +Using Kubernetes after Deployment +================================= + +Once you have finished installing Kubernetes, you can use the +following command to test the deployment. + +To deploy 5 replicas of the microbot web application inside the Kubernetes +cluster run the following command: + +.. code-block:: bash + + juju run-action kubernetes-worker/0 microbot replicas=5 + +This action performs the following steps: + +It creates a deployment titled 'microbots' comprised of 5 replicas defined +during the run of the action. It also creates a service named 'microbots' +which binds an 'endpoint', using all 5 of the 'microbots' pods. +Finally, it will create an ingress resource, which points at a +xip.io domain to simulate a proper DNS service. + +Running the packaged example + +You can run a Juju action to create an example microbot web application: + +.. code-block:: bash + + $ juju run-action kubernetes-worker/0 microbot replicas=3 + Action queued with id: db7cc72b-5f35-4a4d-877c-284c4b776eb8 + + $ juju show-action-output db7cc72b-5f35-4a4d-877c-284c4b776eb8 + results: + address: microbot.104.198.77.197.xip.io + status: completed + timing: + completed: 2016-09-26 20:42:42 +0000 UTC + enqueued: 2016-09-26 20:42:39 +0000 UTC + started: 2016-09-26 20:42:41 +0000 UTC +Note: Your FQDN will be different and contain the address of the cloud +instance. +At this point, you can inspect the cluster to observe the workload coming +online. + +For deploying via juju, refer to : https://jujucharms.com/u/aakashkt/kubernetes-ovn/ + +Supported OVN scenarios +======================= +Name: joid-k8s-ovn-lb-noha +Test Link: https://build.opnfv.org/ci/view/joid/job/joid-k8-ovn-lb-noha-baremetal-daily-master/ + +References +========== + +Juju +---- +- `Juju Charm store <https://jujucharms.com/>` +- `Juju documents <https://jujucharms.com/docs/stable/getting-started>` +- `Canonical Distibuytion of Kubernetes <https://jujucharms.com/canonical-kubernetes/>` + +MAAS +---- +- `Bare metal management (Metal-As-A-Service) <http://maas.io/get-started>` +- `MAAS API documents <http://maas.ubuntu.com/docs/>` + +JOID +---- +- `OPNFV JOID wiki <https://wiki.opnfv.org/joid>` +- `OPNFV JOID Get Started <https://wiki.opnfv.org/display/joid/JOID+Get+Started>` + +Kubernetes +---------- +- `Kubernetes Release artifacts <https://get.k8s.io/>` +- `Kubernetes documentation <https://kubernetes.io/>`
\ No newline at end of file diff --git a/docs/scenarios/index.rst b/docs/scenarios/index.rst new file mode 100644 index 0000000..82a81f0 --- /dev/null +++ b/docs/scenarios/index.rst @@ -0,0 +1,26 @@ +.. _ovn4nfv-scenarios: + +.. This work is licensed under a Creative Commons Attribution 4.0 International Licence. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) <optionally add copywriters name> + +============================= +OVN4NFV Installer Scenarios +============================= + +:Editors: Trinath Somanchi (NXP India) +:Authors: Prasad Gorja (NXP India) + Trinath Somanchi (NXP India) + Prakash Ramchandran (Dell) + Aakash K T (IIIT Hyderabad INDIA) + +:Abstract: Installer scenario documenation. + Currently Apex, Fuel and Joid support OVN no-feature and feature + scenarios. + +.. toctree:: + :numbered: + :maxdepth: 4 + + JOID/k8s-ovn-lb-noha.rst + diff --git a/docs/testing/index.rst b/docs/testing/index.rst index b0d0585..c94f1a7 100644 --- a/docs/testing/index.rst +++ b/docs/testing/index.rst @@ -1,3 +1,5 @@ +.. _ovn4nfv-testing: + .. This work is licensed under a Creative Commons Attribution 4.0 International .. License. http://creativecommons.org/licenses/by/4.0 @@ -7,12 +9,12 @@ OVN4NFV Testing Notes :Project: OVN4NFV, https://wiki.opnfv.org/display/PROJ/Ovn4nfv -:Editors: Vikram Dham (Huawei) -:Authors: Vikram Dham (Huawei) - Trinath Somanchi (NXP) - Prakash Ramchandran (Huawei) +:Editors: Trinath Somanchi (NXP India) +:Authors: Prasad Gorja (NXP India) + Trinath Somanchi (NXP India) + Prakash Ramchandran (Dell) -:Abstract: OVN4NFV Release Notes. +:Abstract: OVN4NFV Testing Notes. .. toctree:: :maxdepth: 1 diff --git a/sourcecode/JOID/charm-k8s-ovn/build_ovn.sh b/sourcecode/JOID/charm-k8s-ovn/build_ovn.sh new file mode 100755 index 0000000..8943091 --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/build_ovn.sh @@ -0,0 +1,5 @@ +#!bin/bash + +export JUJU_REPOSITORY=$(pwd) +export LAYER_PATH=$JUJU_REPOSITORY/layers +export INTERFACE_PATH=$JUJU_REPOSITORY/interfaces diff --git a/sourcecode/JOID/charm-k8s-ovn/bundle/bundle.yaml b/sourcecode/JOID/charm-k8s-ovn/bundle/bundle.yaml new file mode 100644 index 0000000..dfe3926 --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/bundle/bundle.yaml @@ -0,0 +1,101 @@ +series: xenial + +applications: + + kubeapi-load-balancer: + charm: "cs:~containers/kubeapi-load-balancer-16" + expose: true + num_units: 1 + to: + - "3" + annotations: + gui-x: '450' + gui-y: '250' + + kubernetes-master: + charm: "cs:~containers/kubernetes-master-35" + num_units: 1 + to: + - "0" + expose: true + options: + service-cidr: 192.168.200.0/24 + channel: 1.5/stable + annotations: + gui-x: '800' + gui-y: '850' + + kubernetes-worker: + charm: "cs:~containers/kubernetes-worker-40" + num_units: 2 + to: + - "1" + - "2" + expose: true + options: + channel: 1.5/stable + annotations: + gui-x: '100' + gui-y: '850' + + ovn: + charm: "cs:~aakashkt/ovn-16" + options: + gateway-physical-interface: "none" + annotations: + gui-x: '450' + gui-y: '750' + + etcd: + charm: "cs:~containers/etcd-40" + num_units: 1 + to: + - "0" + annotations: + gui-x: '800' + gui-y: '550' + + easyrsa: + charm: "cs:~containers/easyrsa-12" + num_units: 1 + to: + - "1" + annotations: + gui-x: '450' + gui-y: '550' + +relations: + - - "kubernetes-master:kube-api-endpoint" + - "kubeapi-load-balancer:apiserver" + - - "kubernetes-master:loadbalancer" + - "kubeapi-load-balancer:loadbalancer" + - - "kubernetes-worker:kube-api-endpoint" + - "kubeapi-load-balancer:website" + - - "kubeapi-load-balancer:certificates" + - "easyrsa:client" + - - "kubernetes-master:kube-api-endpoint" + - "kubernetes-worker:kube-api-endpoint" + - - "kubernetes-master:kube-control" + - "kubernetes-worker:kube-control" + - - "kubernetes-master:cni" + - "ovn:cni" + - - "kubernetes-worker:cni" + - "ovn:cni" + - - "etcd:certificates" + - "easyrsa:client" + - - "kubernetes-worker:certificates" + - "easyrsa:client" + - - "kubernetes-master:etcd" + - "etcd:db" + - - "kubernetes-master:certificates" + - "easyrsa:client" + +machines: + "0": + series: xenial + "1": + series: xenial + "2": + series: xenial + "3": + series: xenial
\ No newline at end of file diff --git a/sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/copyright b/sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/copyright new file mode 100644 index 0000000..e04f1d3 --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/copyright @@ -0,0 +1,13 @@ +Copyright 2017 Aakash KT <aakashkt0@gmail.com> <aakash.kt@research.iiit.ac.in> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License.
\ No newline at end of file diff --git a/sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/interface.yaml b/sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/interface.yaml new file mode 100644 index 0000000..2ccf05b --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/interface.yaml @@ -0,0 +1,4 @@ +name : master-config +summary : Master config peer relation for OVN charm +version : 1 +maintainer : Aakash KT <aakashkt0@gmail.com> <aakash.kt@research.iiit.ac.in> diff --git a/sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/peers.py b/sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/peers.py new file mode 100644 index 0000000..1cea378 --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/interfaces/master-config/peers.py @@ -0,0 +1,189 @@ +import os +import json +import re +import sys +import subprocess +import time +import urllib.request as urllib2 +import multiprocessing as mp + +from charmhelpers.core import host + +from charmhelpers.core.hookenv import ( + open_port, + open_ports, + status_set, + config, + unit_public_ip, + unit_private_ip, +) + +from charmhelpers.core.host import ( + service_start, + service_stop, + log, + mkdir, + write_file, +) + +from charmhelpers.fetch import ( + apt_install, + apt_update, + apt_upgrade +) + +from charms.reactive.helpers import ( + mark_invoked, + was_invoked, +) + +from charms.reactive import ( + when, + when_not, + when_file_changed, + hook, + RelationBase, + scopes, + set_state, + remove_state +) + + + +CONF_FILE = '/tmp'; + + +######################################################################### +# Common functions +######################################################################### + +def run_command(command=None): + + if command is None: + return False; + + log('Running Command "%s"' % command); + try: + return subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT).decode('utf-8').replace('\n', ''); + except subprocess.CalledProcessError as e: + log('Error running "%s" : %s' % (command, e.output)); + + return False; + +def get_config(key): + conf = config(key); + return conf; + +def retrieve(key): + try: + conf = open('/tmp/ovn_conf', 'r'); + except: + return ''; + + plain_text = conf.read(); + conf.close(); + if plain_text == '': + return ''; + else: + data = json.loads(plain_text); + return data[key]; + +def store(key, value): + conf = open('/tmp/ovn_conf', 'r'); + plain_text = conf.read(); + conf.close(); + + conf = open('/tmp/ovn_conf', 'w+'); + + data = {}; + if plain_text != '': + data = json.loads(plain_text); + data[key] = value; + + conf.truncate(0); + conf.seek(0, 0); + conf.write(json.dumps(data)); + conf.close(); + + +######################################################################### +# Relation Class +######################################################################### + +class MasterConfigPeer(RelationBase): + + scope = scopes.UNIT; + + @hook("{peers:master-config}-relation-{joined}") + def joined(self): + conv = self.conversation(); + conv.set_state("{relation_name}.connected"); + + @hook("{peers:master-config}-relation-{changed}") + def changed(self): + conv = self.conversation(); + worker_id = conv.get_local(key='worker_id'); + + if worker_id != None and conv.get_remote(worker_id): + conv.set_state("{relation_name}.master.data.available"); + elif conv.get_remote('cert_to_sign'): + conv.set_state("{relation_name}.worker.cert.available"); + + @hook("{peers:master-config}-relation-{departed}") + def departed(self): + conv = self.conversation(); + + conv.remove_state("{relation_name}.connected"); + conv.remove_state("{relation_name}.master.data.available"); + conv.remove_state("{relation_name}.worker.cert.available"); + + def set_worker_id(self, worker_id): + convs = self.conversations(); + + for conv in convs: + conv.set_local(key='worker_id', value=worker_id); + + def get_worker_data(self): + convs = self.conversations(); + + final_data = []; + for conv in convs: + worker_unit = {}; + + cert = conv.get_remote('cert_to_sign'); + worker_hostname = conv.get_remote('worker_hostname'); + + worker_unit['cert_to_sign'] = cert; + worker_unit['worker_hostname'] = worker_hostname; + + final_data.append(worker_unit); + + return final_data; + + def send_worker_data(self, data): + convs = self.conversations(); + + for conv in convs: + conv.set_remote(data=data); + + + def send_signed_certs(self, certs): + convs = self.conversations(); + for conv in convs: + for key, value in certs.items(): + data_str = json.dumps(value); + conv.set_remote(key=key, value=data_str); + + def get_signed_cert(self, worker_hostname): + convs = self.conversations(); + + final = None; + for conv in convs: + data = conv.get_remote(worker_hostname); + + if data != '' and data != None: + data = json.loads(data); + final = data; + break; + + return final;
\ No newline at end of file diff --git a/sourcecode/JOID/charm-k8s-ovn/layers/ovn/README.ex b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/README.ex new file mode 100644 index 0000000..f8f5bc0 --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/README.ex @@ -0,0 +1,21 @@ +# Overview + +This charm provides an SDN via the use of OVN and can be used with any principal charm implementing the [kubernetes-cni](https://github.com/juju-solutions/interface-kubernetes-cni) interface. + +# Usage + +This charm is subordinate. + +<code> +juju deploy ovn +juju deploy kubernetes-master +juju deploy kubernetes-worker +juju add-relation ovn kubernetes-master +juju add-relation ovn kubernetes-worker +</code> + +# Configuration + +The "gateway-physical-interface" option will allow you to choose an interface on which to create the gateway bridge. If unsure, leave it to "none" to use the default interface. + + diff --git a/sourcecode/JOID/charm-k8s-ovn/layers/ovn/config.yaml b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/config.yaml new file mode 100644 index 0000000..eb1fa21 --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/config.yaml @@ -0,0 +1,5 @@ +options: + gateway-physical-interface: + type: string + default: "none" + description: "The interface used by gateway nodes for north-south connectivity" diff --git a/sourcecode/JOID/charm-k8s-ovn/layers/ovn/copyright b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/copyright new file mode 100644 index 0000000..e04f1d3 --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/copyright @@ -0,0 +1,13 @@ +Copyright 2017 Aakash KT <aakashkt0@gmail.com> <aakash.kt@research.iiit.ac.in> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License.
\ No newline at end of file diff --git a/sourcecode/JOID/charm-k8s-ovn/layers/ovn/icon.svg b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/icon.svg new file mode 100644 index 0000000..7517e0d --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/icon.svg @@ -0,0 +1,581 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> + +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://creativecommons.org/ns#" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="96" + height="96" + id="svg6517" + version="1.1" + inkscape:version="0.91 r13725" + sodipodi:docname="icon.svg"> + <defs + id="defs6519"> + <linearGradient + id="Background"> + <stop + id="stop4178" + offset="0" + style="stop-color:#b8b8b8;stop-opacity:1" /> + <stop + id="stop4180" + offset="1" + style="stop-color:#c9c9c9;stop-opacity:1" /> + </linearGradient> + <filter + style="color-interpolation-filters:sRGB;" + inkscape:label="Inner Shadow" + id="filter1121"> + <feFlood + flood-opacity="0.59999999999999998" + flood-color="rgb(0,0,0)" + result="flood" + id="feFlood1123" /> + <feComposite + in="flood" + in2="SourceGraphic" + operator="out" + result="composite1" + id="feComposite1125" /> + <feGaussianBlur + in="composite1" + stdDeviation="1" + result="blur" + id="feGaussianBlur1127" /> + <feOffset + dx="0" + dy="2" + result="offset" + id="feOffset1129" /> + <feComposite + in="offset" + in2="SourceGraphic" + operator="atop" + result="composite2" + id="feComposite1131" /> + </filter> + <filter + style="color-interpolation-filters:sRGB;" + inkscape:label="Drop Shadow" + id="filter950"> + <feFlood + flood-opacity="0.25" + flood-color="rgb(0,0,0)" + result="flood" + id="feFlood952" /> + <feComposite + in="flood" + in2="SourceGraphic" + operator="in" + result="composite1" + id="feComposite954" /> + <feGaussianBlur + in="composite1" + stdDeviation="1" + result="blur" + id="feGaussianBlur956" /> + <feOffset + dx="0" + dy="1" + result="offset" + id="feOffset958" /> + <feComposite + in="SourceGraphic" + in2="offset" + operator="over" + result="composite2" + id="feComposite960" /> + </filter> + <clipPath + clipPathUnits="userSpaceOnUse" + id="clipPath873"> + <g + transform="matrix(0,-0.66666667,0.66604479,0,-258.25992,677.00001)" + id="g875" + inkscape:label="Layer 1" + style="fill:#ff00ff;fill-opacity:1;stroke:none;display:inline"> + <path + style="fill:#ff00ff;fill-opacity:1;stroke:none;display:inline" + d="m 46.702703,898.22775 50.594594,0 C 138.16216,898.22775 144,904.06497 144,944.92583 l 0,50.73846 c 0,40.86071 -5.83784,46.69791 -46.702703,46.69791 l -50.594594,0 C 5.8378378,1042.3622 0,1036.525 0,995.66429 L 0,944.92583 C 0,904.06497 5.8378378,898.22775 46.702703,898.22775 Z" + id="path877" + inkscape:connector-curvature="0" + sodipodi:nodetypes="sssssssss" /> + </g> + </clipPath> + <filter + inkscape:collect="always" + id="filter891" + inkscape:label="Badge Shadow"> + <feGaussianBlur + inkscape:collect="always" + stdDeviation="0.71999962" + id="feGaussianBlur893" /> + </filter> + </defs> + <sodipodi:namedview + id="base" + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1.0" + inkscape:pageopacity="0.0" + inkscape:pageshadow="2" + inkscape:zoom="4.0745362" + inkscape:cx="-15.476928" + inkscape:cy="49.018169" + inkscape:document-units="px" + inkscape:current-layer="layer1" + showgrid="true" + fit-margin-top="0" + fit-margin-left="0" + fit-margin-right="0" + fit-margin-bottom="0" + inkscape:window-width="1383" + inkscape:window-height="879" + inkscape:window-x="57" + inkscape:window-y="21" + inkscape:window-maximized="1" + showborder="true" + showguides="true" + inkscape:guide-bbox="true" + inkscape:showpageshadow="false" + inkscape:snap-global="true"> + <inkscape:grid + type="xygrid" + id="grid821" /> + <sodipodi:guide + orientation="1,0" + position="16,48" + id="guide823" /> + <sodipodi:guide + orientation="0,1" + position="64,80" + id="guide825" /> + <sodipodi:guide + orientation="1,0" + position="80,40" + id="guide827" /> + <sodipodi:guide + orientation="0,1" + position="64,16" + id="guide829" /> + </sodipodi:namedview> + <metadata + id="metadata6522"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + <dc:title /> + </cc:Work> + </rdf:RDF> + </metadata> + <g + inkscape:label="BACKGROUND" + inkscape:groupmode="layer" + id="layer1" + transform="translate(268,-635.29076)" + style="display:inline"> + <path + style="fill:#ffffff;fill-opacity:1;stroke:none;display:inline;filter:url(#filter1121)" + d="m -268,700.15563 0,-33.72973 c 0,-27.24324 3.88785,-31.13513 31.10302,-31.13513 l 33.79408,0 c 27.21507,0 31.1029,3.89189 31.1029,31.13513 l 0,33.72973 c 0,27.24325 -3.88783,31.13514 -31.1029,31.13514 l -33.79408,0 C -264.11215,731.29077 -268,727.39888 -268,700.15563 Z" + id="path6455" + inkscape:connector-curvature="0" + sodipodi:nodetypes="sssssssss" /> + <image + y="651.29077" + x="-252" + id="image3466" + xlink:href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgAAADIEAIAAAByquWKAABDBElEQVR4nOzdB1wT6d4+/Emh9y4g +vQgoRboiiCgCggrIqgh2EcuKIiiIYqErdooUxYJIVwRpFlRQkSqoNAXpSC/SIeXNJPt/H/es7qIk +GRLme32e57MHYebac/TnZDK5bzSeCIDNMjh9MMPDw6XDpW0v2h61PapPqreqt6rjrtOp06mX+JL9 +JbsNaBtsG+zg6kB2IAceDtwduDu2dUyCkPdjXWNd2GjsM+yzXz7xBcAFcGG6xfSY6TFzHxiW9WAE +Dgh4CHgIBQqFCYWJf5HQkdCRfiJVIVUhc072oexDcR0xSzFLfiYBPQE9Rj4wwEtiYLMMAh5YdGgY +zHgrmCaBJoJKh4p5FfOq7lY7VjvWKn+e83lOg36DeIN4+7UOQkaA4ZLhElwDbgg3hDAFA/W/AFEX +GHwuPgufxSjKKMIowtXP1cbVJqYn5iTmJP1Q5rPMZ/mF8k7yTvMNlA4rHZ7noRCuEM6rx7uLdxdS +CAzU/wIw8oMHFi17DqZPpo+lj6Wqt+pF1YviY0ULixaWGJSwlbBVZlVGV0b3pPT49/jjmMEglMBA +XZr88MlgmN2ZnZidxDXErcSt1BLUPqp91BzRlNCU0BjRsNawFkdLGEkYMfSAgbov7PfBA4sWVIIZ +bhwuHi5+519qVmqWq5DbmNtYVF50o+hG9cbqpdVLx2rG0sfSgWwgjxAYSSMY5BbkUuRSflZ+RX5F +TUCTQ5ND+6ROnk6evtWSrCVZkkulTkidQAYgg5BBUNeF/Td4YM08xD9mI+EjviO++f35sfmxmQ0Z +Hhkez0Se9T/r7+fqn+yfBO4TA5sGVC+yBlkj7zTPfZ67uZT5enNzkyxTK1N76X3ScdJxSEMwUHeE +/S94YM0I4/PHBccF8+PzV+evTv2c+ib1zfNvOW45bv2nwUDdbrZANiNKEaUKYoriiuKr5q/yXOVp +rmyRapEq4SzxQOIBQhAM1B1nO3hgQQB/B0y1ZHVpdWnSwqSOpI70xEc7H+1sX9uu1q6G0AQDdUcY +CPUQFYAKUH+pzqLOYlNqM2ozajpi5mzmzBnP+ZXzK9TtZiN4YFHJ2DowWfxZ97Pu3z0c3Rjd+O7K +u6vvruIycUm4JKjbwf4bvhgM935uXW5di/0W+Rb59rmbNTZrKFxUGFYYBnqIgVEYPLAog/iWfHdW +96XuS7EysQT3amJex7xuD21f074G6CYGRuNQvagWVMuigkUDiwZ2nt75YWeBfq6BtYE9qg0M1O3o +EzywyKcYTPOr5ivNV26MXw+6HpQcnbwyeeXQ8FDnUCfU5WAUZg1GkRgH5t3eu73NT5r3mfcxpjMW +MhZCXY5+wANreojv6DWyNlY2Vl77EmocapzyKYVg/OS467gr1OVgEOkCf1dIuUjtkdqz5/De13tf +W+pZ1lvWM35kHGQchLocbYMH1m/qCO3w6vAKGQjeH7w/wSmhPaF9fMG4+bg51L1gM0wjvhJfKWMh +ayRrdND4YObBzFUt5gHmAaizYKAuR3vggTU1xA+7DFcOZw5nRu2/0X+jP9Ilkj2SffDAoPugO8AG +dT0YTTABo7ZTbZfaLvfz7qLuotoXdfA6eIQoGKjL0QZ4YP2HccFx5DjylvstxluMd6ui/aL9Wp62 +XG25CnUvGG0Dn75HLtWPWRK2JOyE2UnWk6yy5rJhsmGAIzGwn4AH1o+og8k7kSeQJ+CldibxTGLt +8lqVWhWoa8HoEwOaYYxhzGarTapNquu9IzpHdHgrefG88J/KH4AH1t90q3ezdLP4vvJV8lVKlX7Y +8rAFxwIG6l6w2ULAGoxHpoeYh9ia9LWWay2RUmCg7jVTIKEuAD18EZi0qDTlNGWzvaYppikp8x/0 +POiBRxWM+rrug3GOdJZ0ltx9ySHTIbOtvC21LRXqXjPFrL7C6vvU96TvyengU8KnhNPq0k6nncZX +gk8zQ90LBvs/PKFgTiufeXbm2eqvqzeu3ojQAgN1L2jMvoFFfLwzXyW/P7//CMOR7Ue2t0q3oFpQ +UNeCwf4N6ROm1ibW+tb6J81PWZyy4BTmlOGUgboXtc2igTXZNtk/2R+yITg/OD/kQAhTCBPGDbMD +swPqXjDYr5GeL71AesGl4Us+l3xUnquqqapB3Yh6ZsXA6tjYsbhj8eEU5ybnpjecb8bfjANsxMBg +NIv5AhgPb48hjyH7rs3HNx9HtICBuhdl0e/AIn78uFC1oK6gzinZidmJuWNTh2WHJdS1YDCyIi7l +aMUCxrvDp8GngW0pmxmbGdS1KIUO3yXEZ4CJjo5Wj1bffHVz8+ZmeFTB6BbxQ9cPzMBsbN/wdsPb +Zram+010uxotXV1hTXJOoifRXhVei70W3+0Gn0on/I9pTwgMNmsIAAIjAiPB48H7gvdpf9Y5onOE +nm6A0MnAwgRgjmOOu7w43Hq4NfV56r3Ue4AIMTDYrMS6k1WfVT/y6/XP1z8vDln8dvFbqBuRB82/ +JBzhHEGMIGzlN1pstEgtTk1LTYNHFQw2cmMkbyRvy6LNVput7jMmByUHAW3E0DgavsL6du7b6W+n +98buub7n+pvPb/re9IFXw4TAYLDvMDIwtjG2+XX7EayLtJGykQLWEUODaHJg9a7r1ezV3H5re932 +uvcq5VzlXFA3gsFmOpQ2Sgol5YnwdPF02YraFrstFoghhqbQ2MDqLunO6s7awr556ealVaZVilWK +UDeCwWgJkh2M++1jvMd4He44qDuoA5eJoRE0cw+rF9HT3dO9Zf6WdVvWwaMKBvs9uCEw/lJ+KX4p +N8quI68jAR9iaAQNDKz+1v43/U+3obfpbNOpUqwUqITvUsFg04JXB+Mn6VftV32n87b8bXkgmZgZ +b0YPrGHmoY6hjl0ZOz/u/PJh7gfsByzUjWAw+oF7iavEVXrVe93xupNonBiaGApUEjODzdB7WKN1 +o2WjZXvSHYsci3Iv5frm+kLdCPZjpO1Fp/KdCAEwgAQxsBmGYTuDBYNF0Lsg8yBzkwemZqYz9MM9 +M25gYTCYPkyffbrdsN1wgV3BvIJ58MMKVELcsgyhBIZxDaMboxvvXF41XrU50sJGwkYidSIMIgwC +GgJKAkp8B3nded05xzl1OHXYTrD7s/szFjA+YnzEcJQhnuEmAwNDG0ML7hDOGec5OWeSY5JjYjuY +cZsxQr69GLw9eHtAuF+2X7bbr8e5x7njRfvt9tut61t1WnW6+LsauxoHDww6DTphB7EF2IK/utHR +s9ozGUsYy0WWi1c4rzpcdTBeZNxjPOP2sp5BA2viMxhXRldjV+M0jdTB1EGAgxgYWSG6EQAC4GLg +EuESUTRRPKR4SCVEtU61bn7PfALZOtlJ2UnRo6Juom7sNezN7M1IcBeOcYrXygIzfnXcf9y/71Nf +eV95g37DvIZ51Seqt1Rv+YD5kPMh531teXx5fGNWY0RjxGTI5KnJU4RrNXiQkR3jOMNrhtcJa5OU +kpRUr6kOqA5A3ej/zIiBhT8GxiP6GO4YLq4r7lLcJfhpdTLQAMOxleMkx0mtZi1BLUG99iVrl6zV +5tA21zaXey5XIFfAdBsMrVzD4neB6RHsISizKFtftj5fOf9z/udXva8I6s7Vrq5djX2BvYa9BowQ +A5sGzkhOV07X6EPgIgIqH1UXqy6GuhEI6oFFXAQmojrCJMLEP8XP0M+QtFwGZH1oF3HPO95CXhwv +zkjX6IjREVNFMz8zP933uly6XGwn2A6zHabX+0f4CjCNhxrtG+2ftT3Nf5qfKZ75MvNl+dFyo3Ij +zAaMHEYOvg77PcJRwnHCccnM94vvFwsvEv5T+E9o+0A8sJ6L5NTm1Do0Oog7iGPlwUDVhMYQN3Zl +kWQxYTFZEWXcZdxlLm3uZ+5ncMkgyyCL5TgYqCtCKhdMozh4W+6RahpDGsMjj/Q16WuqN1ZVV1UD +hsSrT9iULfQCE6N5T+qeFIsiiz6LPlRNIBtY9YP1hfWF1uJWS6yW9HP3i/TDLwD/C3HlIyl2KUMp +w61XttZsrbFAWhywOMDXy8/Mzwx1uZkOb47fgN9QbFtsWGwYPxafFp+Wzvso/VH62J4xwTFB+Kb+ +VKx/s4FzA2fAuwACxCow1O8AwcAa1QBjc2Pdk3VPKq0q1SvVqXl2GlMCRnmf8g3lG46seyL2RKwU +W2m10orhDEMgQyDU5Whbm3nbsrZlt9be+nbrW1x/bGps6uD5Qb1BPXhw/QyiG9GIaPQN90P4IWw3 +2eJscRB0oOrAsgJzLOxY57HOOOXYT7Gf4L/ZfsAXzAJpwpjad3CX0wunF0Z9yy2WWyBbwUBdjj51 +F3dHd0cT/lZA30DfdY7uiu4ayhy6OHQR6l4zEUsYyzGWY4mLkwaTBudzzHed70rNs1N1YD1Cp0Wl +RTmJO3k7ec+EdydnGvGLEo0SjS4uh6sPV5ubWwRbBKOCwEDda9Yg3hls/9j+vP15UFmQa5BrIm9C +RkLG5NFJo0kjqMvNLHKP5HLkch5UpbxKecVmw7aFbQt1zkulgdX6sPV663WLbeb7zPfBd6y+x9rF +ysLK4mjvGOkYueuCwzyHeaxsrAKsNPCQAZ0jDq+K+oqiiiLvTG9jb+MCzNvAt4FABC2tbUBpdpn2 +nfadPhd8rHysgEhqLFZD8YGFOw9mq9GWvVv2vlr3av6r+ZQ7F80gPiGly6W7QXeDz0VfFl8WGS6Z +1TKroa4F+zFsOph4o3iueK6z8QFzAuZ8O/NtxbcVUPeCHlILzHWbG09uPFm2fln/sn5Kn5HiA+tu +8V31u+qeJ08InRACqoiZxZgWM61hWuOc62zgbLCrw2GTwybUBBioe8GmqvFl453GsKM7jsgeUSmc +V+hd6A3/rhbZILJaZHX6gYy1GWu5RblXcFNwlFNwtYaWdy2RLZFnVc6Wny2H/0eVfCXVI9UTHxe/ +On61Y/Uesz1m8KiiRRJLJbZI7Ll7KaY7pvuArFOBUwHaHO2KpuqN55mmLaotri0u4Im/lL8UpVfX +osjAwoeBORV0svVk69C8QbFBMUqchQYQnz431TeNMI1I8UsxSzFTxaoZqxnD743SOoY1YA5fPdx7 +uPe61I2WGy18lfwK/ApQ94II8fdz4tzEwsTC11tfI19T8DKIIofO8EoXTRfN4cmpy6mjxPFnPtRJ +VAAqwKXGxcbFJuR96NHQo1whXAlcCVD3gpHfUpelAUsDHtg9AB4AynOUg5Rn6bu6OAfcMdyxUxMn +jU4ajRmMAqMUOQuZB9bgh8Hng899qnwkfCSA+0AeIbMMmwabKZtp0B/BfcF9fwYe4DnAgywDA3Uv +GGWJ3RfLEsuKZY8LiAsw6zZzNnMGFAF9ALKPsEClTrVued3yyObIdZHrSO+0kvf4ZL7p7tfkp++n +H7k0ojmimVzHpBWElwRL+JdEsEZmR2YvTF7YtbAL6kYwaGAZwfha+3T5dN28dUv0liigiOfH80Pd +i3pYrVjVWdUz7bLEs8TFNcQvi5PtQRCyXWH15PVk92Qn3kqwTbAl1zFpBWsXmICegJMBJxfeWdiw +sAHqRjAokd5OOfqnW45bjlmT2WazzQA/IE7IrDHyYKR0pPQO7rb3bW/SwpDkOjLZBtaVx5eVLiv1 +3+w/13+OXMec+VizwIQfi9CP0F9euEJ+hTx8Qx1GwryE2YrZ6krMFcErgmvV1t5eexvAz66xddf5 +rv5dxbrJupy6HHIdkwwvCWvP1u6r3bdq3Axphpy8Oflo8hG5ys1kzMZgwhsikiOSDR4b9Br0Qt0I +NnNNxk1GTUYdKj506dCljDvgc6iEv9RmyRKDphpgromGzQubB1wB/AiZhuldYRFXHbrQeV7vvN7s +GVXos+ggdNDF+othF8PgUQWbCoaNDDsYdly8SPhdU28QsRSxFEHum9Ez12N2MCVsJdEl0dM/2rQG +VnlsuWu56+PSx5GPI6dfZeZDPEDEIGJOzTkdfDrY7N6q3lXwqIL9AiYc0xDTULBwcFRwlMpylWcq +z6BuRA24l2AuKlyIvhBNekJzOkf73YF1CMwV5OXDlw/jGsBMp8TMh2oB45i1x36PvX2jPQGtrIMO +m2k4dMCER0dIREjILJEpkCmAuhE15Afnb8vf9nY/uAb/dI7zmwOrXKa8tbz1hfqL+hf10zk9DVAE +47TC6bnTc1cd1yDXIGAzMTDYNMxBg7l7LqY1ppWrncuIi86Xr8F34QE8EPQsSCZIBn8NzO8d59cH +FvG+Vahc6PPQ53gffDg+/PdOTCtMj5uVmJX8+fUAAWoZygJlAXUjGP2YIzxHfY769es3tG9osx1n +K2UrhboRZb099zbtbVqJQQnB7x3hlwdW7bPaqNqoZ6efcj7l/L1T0gqpGule6d6zV8/qnNWBn1aH +UY6mneYezT1ei70LvQsRmoh1iHVQN6IUfBWYiMhwxXBFwAfwIOQX/fLAunkuqi2qDduBxWEhWNGZ +OpjWMG1k2niF9fKDyw844znTONOgbgQjPwwDdsfkesw5bAGGAeouIGt9aztrO5vrf6z6A4LNHagp +xy1nTc6aurd1N+tu/urP/sLA6pPqQ/YhU1alCKUI/eppaAb4YJ+4cxa4XpUyRmW5ynKoC8EopWfd +twtdihdZksVPWXwLGYnun9bNYHIUAuNZ7TnmOSb5WMpMygziPhSDXYw1xhpH597hu8P3qz/7CwMr +sShRL1FvJG8kayTrV09DKzRLNAU0BXae38W8C944i94RH9usLGkyKJvwVo+Z79LStLuz6AvEexFx +6HCYc5j7N/lL+kuiYlGfUJ+g7UM5D64/8H/g/+3tt5BvIVP/qSkNLGwomDjzWI5YDnr96AmTGhif +Rt/jvsfRq9FGaDp/1wb2vc6w/mdfA/204s4e3Vg879Pj12ZALlAMQPYXs66K7g7dHbZ/2mJtsVB1 +oLRvO75t/LYxLT/tVNqpqf/UlAbWW4a3z94+qy/6kvQl6XfrzWDERTB2fdiltkttHvM8AqgLwaAx ++mpcZ8QxJDQtxP9hitab8XvzcS/wm3BeEFQhPuXnWuY65jomWCqkJaQFQQdKI/47xqnEpsWm4Z3A +TOWHpjSwEgLjueO5AQmEEkJpeh1norlMcwn2Pt23ft96er1+hE0d7g5uCc7yQeTrvrt3w9Y+GjiX +NmY2cWMUgq3tuTDcPNw8R9uOyh6VBUwAe0LoTkVSxW1Chj5e+HhhKt//HwNrIGrg1MCpJzee/PHk +D3LUm2GID4UeZTi6/eh2Nkk2JTY6HMew31QCVAF5BcbVWrms/qVxE248XWED+9s/UL+IZaflSsuV +C+sXblq4ifpnpzR8Fphk+2R88pSWYPiPgZV9OUszS3N03ejmUTp8tnshz0Lbhbbm0hapFqlQd4HN +SMQr7gbTjqFaRu/6GCWXe9X6zdYfuIi3Eai0QCNqGRg3OXc3dzdEKRKBRFDnvNSUUZHhneE9ITTB +McHx79/584FFfKL9YX7qvtR99PdCCaGJUEQouux1feP6BvkUDNSNYDPdQPvwSF/5eeMkPk+h54vK +zTMqAAXgEMBCnbPrfNLp0+lbttWwzJAOH2DusOxw6HB4K5IfnB/879/504HVFd3l0eVRGFrgVuBG +7nrQ092vG68br2erl66XDnUXGC2ZrMPIT8jexj35FHI62ufZm2t3MEbYhMl8ip/4MfAKeHVw7qGV +h1Yi36GEUHT1LCTCAMwj0UcFjwr+fSX4nw6sJ9pPxp6MYUIx5zHnKVMSItlg/mw74HbADWgmBgb7 +RXgNvDOe9Wlc6aq04IsNyayn+r99HWkckKH0eVV2qISohBhGGr4zfEfpc1Ffztec+Jz48TdgfvY9 +PxpYxAn32DFbKZsOb0IvdFsYujB0Ud0ikUUiUHeB0TjirZIK/caqsiAfq3tmLhxNI12W9aIUPKMm +GMfDjtWO1QhnhB9iWqt3zjQ9Yz1CPUIlHsUsxT99of2DgTWIHewY7CiQL/hQAMF7IpS2M32n2k41 +hCcYqLvA6EfHvL7tbSJ+TbHKRweKL3zqfV1DuUdPtZq0EFoIVVW1W2q3KHF8yICLRo883f60/Gn5 +z77lBwMr/33+2fyzYy1jn8bo6mMBIkdF0kTSVmaaiJvMoo0AYNQ06j/OOWwQopa2PmD+wwX5n2Nb +cPX4g7jfXPvpZxBSYLZobXHZ4gKw0tvGFi/DwOA+gvnnr/5gYL0Ue0lAfytq/vF6/bL1yxiWg4G6 +C4ye4e7gTLAr7196hYyuCst49Oe51WMPJlpHfcl7FlMr04umF3lf8HXydZL3yNCqD6u/UH+hCdF0 +uekHuxn+YGAV9hekF9DXe2engXAg3MLW4oDFAairwGaNJkKqCgqrU3J3X8LdHzl9ePT6hMTIEXId +niUbzIqFy88sP0OuY84EeDv8bvzu4ugi/yL/f/7q3wZWx8UO1w7XL++/vP7ymlr1qEHlsUqvSq8s +Vnah7EKou8Bmo+rU5pQP646z3Hy8r649tk+5lWz3T614rPdZ7wPYAQ0Agg8PUc6bbfmL8hf98+vo +7/9DuVY5rhyHVcKuwq5CnARDrXqUtfLtSoGVAkAMsIKQWaDZuSusPv3l7vcbs+l2TaXpG/ljPHUY +hV+AD8c7AJeBEiCC0mfsefwttXMs4ENckDtuv90al2Nz5QZETytVARzE/JaFRQvLF5bzdwksEVjS +bdBFrcfvKa6s9d2xd8fw6fgL+AsIFzCkr/9tYBUtKcIV4RDSCE2EJhQlyQ+RjchD5Jn9sUp6lTTh +P04SMgt0DvfXfq17ElAamQqvQP9fngKdwFtqnrBPeaio59059YQLx9msvPQq7bpWFWpfspkDvAQi +f323QiZmprlMc1c8X8G9gjsOiO2P7adEZ+prqmrqburuLOr07PQUAoRchP4aWH97SfhufimilK4+ +qSR/QP6C/AVpbelt0tug7gKD/Z+JSUzguEw86uXLKN+ITxl550uGvcdeDfH83tFM1E0umVwCWAB+ +QugCZi3GFmP7frI8p/xv29z/NbDG549LjkvWNNek1KRAUY9Slp9Z8X7Fe8LfXGBgsJnmKzAEfH29 +smJnzmHfS/fmux75Ktwr0Fz9q4fRFtJeoL2As5VzPed6StSkPtKHdcqOlP9R/rd1Yv4aWPVJ9R71 +HoMHBy0HLaGoRwH84PMpy64u+7SMrp4mg9Gr1tU915te+YjfE3L9XD7+pa54HtAIdAGVU/lZVgEw +2qk6D3ViKd2Tmj6ofsB8wABdwP9/c+6vgVVRUVFYUYiIQTxAPICuHjlxnuU8xXlKFaNKAHUXGGyq +hhRGtw5eu9KSsspLPmtJsdj9Jrw3fgg/pZWw9PX1ffUv/fp9sJmr5l61Z7Un9gu2APvX/th/Dazq +5qrLVT94TIt2aTZpEjAkMKQz0NczZbBZAOuF3YXBxkY9f3G990ZrtsBltglXjO74vX//qUXCi1Yv +Wo3wp5/Lju6m7ipiIrr/eg/3r4H1yfyz9Gdp6IqRn/YCna06W6FuAYNNw10gD7ib1/3B68mnc3EJ +osft+wyH5LtHfvbt0qnST6Wf8hfzD/PTyVUWdjF2LXZt3Ye6nLq/br0j8QaELPzyrS6xLhHacmRD +3FtQI1NDQIOuPloEm6XApTPZPi9v1a009cqIYXPxb7/ZZ9N6/Z/fiNIBoxqiVqBWQPWWFIHYA6Z2 +8vPbz389eoIc/jz0cqi0PbE9oT0B2nLkwnCaIZAhUKlJqVupG+ouMBh5IEoQlxAj6mGyX3Un+AHO +U0I/fVZezYveVnGoZ67vqO8g/TOyfqx+vH588ujkmUk6+USS9EPpXuleVgFWRVZFqLvAYNOFVkYJ +MXy2D1++ce+rzXcJ//8eejtKGf31Z9+/oGYBagGK9C45NXtSTqNGI38jPzAIBl3rVqdYp4jQROAR +U9q1YuZTaAADpAB+AF0tbwabbdiXsjBzftp72KLyqMqCVsmL6m+AUgD4rzVU5Ovlv8h/Qfojq5BV +OAecHk6PKmUpqIWj5U3LG8AB8Af8kfWLwEBdiZzkMHIr5GbFZwZhdAi8XT4sHMWXIcZ03GTThcDV +C/olA9TvE+9kTemerGCk4GvB11wWXK5crpStSi3djt0e3R7jfOOM44zI5oKmE00noK5ETlLa0obS +hlC3IL/ap22p1QrjgpPmY3TySU/YP7Aq5Ynbqf55YrfttcADIqK8DWKXfvUQqD4w4hri9uJ0svHq +QMJA4EDgcOSQ3ZAdsu1U2+G2w1BXIg98JhjxInECqLuQSTKQC0S/NqjwehZ0VjfB7ZjHJDfGeOIc +1LVg5IRIQ5xGlBg+UBE27XLptTnhlch+g0WB0+g3D2cARuyOeKZ4JllrQgb7BcuMZe5Q7ODq4EJ3 +dHbWddYBKlCXIgsJcCdn0XSRBBGaf8cT+w1XiL2VlJaXdPt8pnnRULIzPgq/A/+Z+ItzIS4HIxN0 +HSoX7bOjxhRz6A89BaXkZenAN+ABgn36RxYtEL0legsA6OHj0Ahwk377r83th9oPIXuSu/27f7Cy +Hy1i72FvZm9m02Q3Y6fhdaCGG8aqh5yu9KSYee/KwBfOSxLHV+IV8clQ94KRk9BjnkCRZx7Kts/P +segJKI0b1QE95BlVfx1/gdB1oR88q0WTJBGEdDF0ZXVloUfyRq6P0Mm/GCeaS5lLmfED4zjjONRd +fsdX5V6l5tGr5ik2Plvbvva4N9cC4E4iMLqicExsl8rT/avWoNwLOTewHuamyMM3/Jn8ufy5lDgy +VHo/9hKg8ep4C7wFkANUA7+8qMVMw8UABhgF+gHaWcasCxgGut4vq+cpXhh26dGLQJ7hu2OvB8Wm +c0hUDXIUZc6iyLiIlU4+okEJ+Dt4FpzD2NbJnLE9xL8WKPlXgzNgDfgurVK5YVK0uW/Fqn2lDBtQ ++QxqlDshTwFPAg94Y4QdINtVG7QGCvsf9j9EAy7ECBND4zi3cuzl2As8grrH1OAbAAV8UlZlUdiD +6kSb3NKbGdi7uEys+/SPrJooU649FpJ0YEH85+kfjV51bOtb1nbco+tmxJ5uPCt+N56VEmdBIZFL +USfX6xqwb79t0q2Va3WI8LIvH7GbEuf6HnsAGGAJcJEQNuKOfzRuSHSIAI0/Dwa4QAyNY81l7WKl +gUWtJ8owRydQtwef2AV/fdX2Efe0FagHMgEyjCoShCDwCjEftQ8JoKLIdUz6g1qD7EOdAngJ/9gC +gA9Nk/nBaVYFpjVs1o7h5rGugFqzzBGd44RrnVfU2jqPpRMMEAHcJQS8vrOmznkpZ7RstG60Dg28 +IIYuMGIY2xnboW7xb/o8hxb3WAczPDzlZ1lb1/a+ahvhiyVQt4KRl6Amt6ywstNay48ntosdEzgt +haJ+B8ajjDcZbyIwiFJEKR7Ak3saQ2Di1kTyRDKa8MelhF7+yCBqkGPIMahb/Fhd2tcNNQbBHx+u +83Xs5Rx83x0DdSMYmQ0DgMKQGE55zv6eNRnH5nKKs0ZyQzCqSJCeyKvIq8AbYJyQu0AxITQOE405 +izmLhroGWfESM8NUnWjmf690GX1/55kjY5wTuFF4VNEh5W6pRI0ip17LkRPrGZPR9UxQb9mLIoaN +GLqA18AL44V/sPMzDbtHzEwQDjwHwh+/Kal5+Ozsqnj3Y0/G6if4R+llxTEYiS2wGnAxCdIstsxw +sV2X72XPmE8YVSFQ1yI6TEwNMXQBeRv5nhDAkBi6gFXD8mB/c6MkcsG54+VxokmWeXNu294beZ4W +oY4Px+PxDtC2gpEXcikyEulvzbEkYHOG7WFDVwdjhD2CE/HLn/ujHMxrzHXMdeAFcJsQuoBahVqK +WkpXA2v86ljAWABUZx/VGG8Z7gyyeNjny5a2/e3N+OX4aMKosoOqD4wSmLGMz1j49o5ZKLuFrJVf +FGjrjriO0EI8gbrX/5pYNq49ro3fjffAe0DdhTyYkpnuM91HIxzBbaAJrw458L+5WfbMMZwLhvAP +uwGKP+fyvY4VfZ/azK8ef2jjg2+J7DraIEx885pO7h3ASPjEOMsEnA/Irv3zRITUtjkWctFQN/o3 +I8AIYgQBXCL+WaCLO1ksaixmLGZI4AJwBqCTtUYHZMFQ6WTD4BPqH9sbht5t9u6MQRze2/Kwa3MD +6ddo/jcH7DvDMvtEghXqPSXt+C5eI44qGrgXObDo2+pvq+ljVJGwt7IXsxejEaWIr4iv9PGkxoDb +wMaBNTg5MMjPYChyGj7ACj/0RKgUkWYY1/+i4/o+jDHWEhNKkXP9A5YJ9xF7C1uAu4a9S50z0iLU +VmQp6iFQDUQCPb95CH1AHNBfdE5R0bBmu6+px8EqJh30Vab7ZK1JQb3veu71gG9AqdDJSiwAFxeX +EZcRmq2QrYitiLhcMtSNpm2gdiBt4Nk4apyABQBDXpP52EWT6OivT89d2/Ty5XuGLF/iRkwRZD7N +z4BPy5UcP3xrcJ8yghURhqDhFSkoqAqoBKoOlK29cOKxPCAKKP3yAZDsCEfkUksfPZ5NzGvUdHNs +UxFzEZOIXgp0paBOn65DXYeAnVD3IB/e97zdvN1o3ptgBv8Y1Buk+bWfJz5PtE609tj0CPYIzgXm +knHZqMbhzl1fZO8uenYgjPWTTcuzj43ES21qjarvDL4Y0Rz4ALwANIEP1D87DQAfkSzGXsVaTj4B +Xv3ajzKnM55mKdrJaHLkYKJ2vkK7QTcgAQgABylTlLK+fmg73Xaa8A85QA7UXaatAU8Iv4rAeoH1 +SEEzwROCdLJEMiISEYOIaVrXpNWkRYbDVQKNQGWR7SfJV6Z+TbHmRwo+bW1p/biEnu4LwEh4L3Ds +FQDceTbo+j/WblMADNiIo0oC6l6/r/lVc0/z774cnmHw4Och785pEYoRikEKfxL5KvLTLYNojAA4 +Shqs6vXqp3W1iMPiw3CJD6rfSMXEhH5JfRsgPRYwUTu6mVw1YTMCcXUGmTfCOvPWnjSz57vgLfV5 +zjP5hVDXmi58PZiG6EbfRl+ou5AH6gnqHuqe0N05snNkkWIdYgRQVyKnWtlahlqG3/vZMdOJp6Ma +obfTogL6Up68DoqpwSngo3ENZC0ImxF42jmU+DrcBtYH+8fwFLGP82tA3Yg8RqxH9Ef0v+a07W/b +D3UX8uB8yPmU8ylbDNthtsNoySDJJMkkwle5CKEL1ZrVo9Wjv/pTXfMGZNp3XA1KCfAJaYrp/OOL +FwCv9knXGF+gvzKFQd2C/L7If5n7Ze4EFxiou5AHHx+fOp868wHmZOZkpLSp9A7pHaTbWlAXI4/q +UjDYD9gCbMF/fCtxD7iqg0333y/z2nW39DCCMKoaiKMKBqNRFcwVQxVDgD09rIFFIooVxYhigDgw +SGmMtKq0KlIbZYgyhLoYefSt71PuU2560HS56fJPv2kJ4AGI5Wwou5oeen5lEosn6tvHkYh+ermX +B5vFyrnKysvKoW5BTmLJ4pXilYAIGCTnJg5CBEwFNgvQy01lXzClbCWXS346sOoYv56rjoypyYkP +Z8UkYD9OMlKzIAxGEZVgSiNLrUvp5NqKRPKyRKlEKemfkchRFBqFloyUfCX5i0+tzGyF1wqfFz7/ +2a/K5AhjFEydbltu8uxhu8+cyI6gZjcYjBK6LnX5dPnUY+sJoO5CHvhcfCW+UjpA5qzMWdJX/loP +S1ZWdrvsduiKkV8+V358fjwwj5ifUF0svV/rpEeX7c5ALiEbnjqRDdRsCIORVxGuqLOoc1JuEjOJ +gboLeSCOI6wR1rI3ZSdlJ0lf+WtgyZ+flzEvA7pi5NeyoIWjhePL2i/yX+T//TvnMvM7StzzfGCX +cQGreEh8SPU9wEoMDEZTciNyFXMViTvk0Mn721xtXANcbXNez2mZ00L6yl8DSxFQ1FfUx4eDga4e +OREuJgl5sfBF/YspXR5zRLE0ce1wdbER8jpvOF812PQucBywBmbQkmww2M9gZcHk3c31zaWTh0VJ +pLcSso/RndGN0Y30lf93hfVRvkO+g0EYDHT1yO8J5+Oyx2VT/350Dmofw8btUStNnOQ2qS277pCE ++gO5BRVLuYYw2PRV2FUsqZjXJttGAHUXcpr/bv7Y/DHAADAD/vqo/18Di7Rnsji/hL6EPnT1yK8k +t2R3ye7uJd0K3Qq/8GOlwGWg2WRE09/q+EFbq3mepay7md3YIPioMww2Fdm+WQJZSoA9YEoIfQDX +mxtWblW2ULb4/sv/bxOKzWDUdqimq6ZDUI5iJkMm4yfjs7Wy8rLyfu8IqoPS/drLjg/b2gZyKyqI +J6nowHe3YDMH9guYTI1Mvkw+qLuQ1WXw4STVFWo71HZ8/+W/7ZqjrqBhqWFJ3V4UJgDm4buHex/u +BbKI+S1zV/LHSLK7uNjM9+4w7FHZbioOOAMmhP+DwSD1XqZ8vHy83rt+Sf0SqLuQE88ZnlCeUOkU +aZT03/Z2/NvA0srWGtMaAzQAfYC+XhhuKTErMWt40eDd4D2d4zAwoqIYQre/Nzl44Jrth2WuDmtR +UkgtFGTbXsBgycX3ne87A46El4P2UHchJ9WNqm6qbgy3GBIYEr7/+t82UpVOlH4q/ZTrK9cQ19CA +w0DewG++jJppcN5g0jUexT6K3Q/8Sci0VAMxCIQpoAlYjbB7Mzdw3I8aya65oohlxX3DVpGn8Q+x +g/+nqC0uplqA3ofyRtPVGhvkgq8B2gA8G5blDSe4+kIzQCdPJP3TxJuJ7Ins7JAs3SxdAPxdWQ11 +I3JST1L/ov4FOEN8jfSdvw0s9Hy0LlpX+4Z2jnbOEwAMPYkzimuPa9+D2UsI+HQ/igy7Xi/xXCC5 +wlp4LZ/LXNR5/8Ruz4AR7/Gg4bXTP/IPgA/AzvvTd037sT3swywVnEYUOQuMRjz2y+bL5utu7E7q +ToK6C1nJg1kUtZhx8Q8+MPeDnZ+1/XQ+6NDh8rstIS3HW46/XvWKgLxHlnkoPKSw1q11Q7C/B38R +5zLBdeQ9Pgz2N9FAOBCeUJDgl+AHDBFDR9iVwcxfNP/U/FP//NUfDKylAgbWBtaIbjCUr0dFxCeA +I1dFjkWOAVbEkJXkUiFX2UDPVXZiF5NkVURUFGl/GyLYjFTpULGyYuXrp68DXtPh/VMtdi1DLUOW +LJZiluJ//uoPBpZsjly7XPvcjLkdczsoX4/aXmu/Hns9Vq5T/qb8DSWOz32eXYg3021gQ7Df4cXc +SiPL/gD0AUXAhBLngs1O4azhBDhL3ErcSqi7kN+ybUYPjR4CSsT8ww8GFkILzPL3KxatWET5etSG +t8Mfwh8K+nzV9aor4dIaDAUweqHTmIx3i6/a6/p6HbBk4ZYUpCTiMBJ+qQibls9Cn8Y+jWUkZXhn +TOv97pkJ+RqMkfSy/ct+urjzDwYWicnFlUUriwBxYujO8/ScAzkHSp1LJEoouDMKwgUhiLi+xngR +48aQfZ9WT7jfYmZifMtCXw/4wagjF8zlmiv+V/wxOzFyGDmoC5HfgrEFyAVIkQLi+qI/8dOBpamp +tU5rnQBCgE2ADre0wrHgFfAK50LOcZ/jxuvhF+IpuVcKuGGUklbsPJYlecfqN74/+5BPj3NckE4e +GYFRxzvku1fvXmVLZOVm5ULdhVJMq8wMzAxIr/B+9j0/HVjoePQj9CMTFpNDJocoUw96BQUFgQWB +2SeysrOyqXNGySdCn2W/nQyw47vwUrZcZEKBJjfphFET7jaYgOv+C/wXYHmx9Vg6WZzve8h7yEhk +pJmSmaeZ539857//8poza9vXtuOLwXX/yFdvxngA5AF5/kL+9/zvjVqPio9S6cUv9xf2vXzqbos2 +3PR/tWi70vFlTwB9gJ++Pl0AI5c0hrTUtNRCscKPhR+h7kIpqrJqa9TWSJpK+kr+x/I4/zGw1IfV +xdXFJcIluiXo6xGH7zSZNvk2+YZeCx0MHSR9Rpw652W8hq5l8nZ8sOqTq7a1/pKkzXlIdsQ6JDn2 +rIbRhYFnA3EDcQEB/nP85wB3gQiAHtcL6QJjtcTSx9IHqCHmX/3HwEItA2OVbDVkRVcPp/0Ncev5 +yHURAhECNZE1mjWa1Dw54gpCEvF6LfOiW7bX9rqsPuUmwvyR8SZLBTU7wGYc4nYSge2BBO2D7ent +dLWGyvdYcGBWPTE3NDecyvf/x8Aisb68Lm9dHuo6KpZ+l7Ibbx2fGJ84dsg90z0TcxSzHUPdFe7B +G/MS2tLzovUd3TM2RAVs5WPmfCIArwYxS+XrvBl/Mx53I1Y4lq4W1Pyn5SLLdyzfwcfCx8/HP5Xv +n9LAEtcXdxR31Lum167XPr16M907rXer3q2KAMJNwyFbCE1KaI6unJfnUju+i0ky60VOKbyEqgmM ++gYXDcoNyh0TPxZ2LAxbj/2Ipdv7VsBxMBsP2N6yvQX+dQ1M6QGjKQ0sEjsf+yr7Kmre5YFAN/jx +nStCVx5feVz2rsyzbA9URXg2scfyhbuvWV/rj1gUp9hgmA1oAIrwjXk6xoInONN+OvB0YGNXY3Yj +ld65horMHBkCXRXdR7qPpv5TvzCwjE4Z5Rvlz30pNiRGv/eziCaCJ55NPDtUdVD8oPgAGgxUTRix +DCJMQ46bzfcdEbY+pMdjX4xYh2BD+kHVB0Y5ibeT1JLU7nff331/N0CHzz7+Lzu8fZh9GOku+dR/ +CoEnmvoPRHRHREdE+2v5efr9xxMT9GFFyAq3FW5hYuHq4eooZZQOSgeyKuD7KY1AFeEasBEwIFxA +G0DWBEZWBSpv89/m75DfcWbHmZGSkeoRulrZ6p+4lLl0uHRyO/Oq8qo433K+53w/9Z/9hSssko1G +G49uPMrxjrOXs+VXf5YWPT319MbTG5cvX8q6lIVlxExiJiGrIkB8pQ+PKjrSZt5m0GawK2aX+i71 +2TCqSNbLrD+y/gjnM858zvxf/dlfHliEefiV8+umC7YVtnT4xO0PdIMJCQ3ZGbLTr9mf3Z8dECMG +BpuGNlRrS2vLpiHbTbaGQ2uG5g39dH9yesI8l5mXmXebzva47XGkx4l+9Qi/PLBIdojsTNiZwOLD +ksmS+XtHoC14BfwS/JIohhu1hETd6LvRR3rgDepeMNozMKd/on9i15JdYbvCGpsaAxrvQN2Ieqwu +WWVaZYqsFjkjcub3jvCbA0twr6CLoIvtGVseW57fOwJNIm7P4b/Hb8xv7MHl+yX3S+j8PVMYWY00 +juSO5Dr6O1Y75lVVVd2tugt1I+phWgFmT/Ve/73+v3dtRfKbA4vE8cGenj09bAJsimyK0zkObcHi +sHOxc92M3erc6rINsgjgsQX7d+NbwPw58KfRn0YFXgVrCnbOhvcBv2ejZeNv4y9uJe4rPq3N9Kc1 +sATng9l6fZvONp3Z9od2cvtk4GTgQauDyQeTn8k8jX0aO9v+G4BNxQTHeO94r9O8A+4H3J+vzZHP +kYe6EbWxdbEWsxbv5/zz9J+np3NtRTKtgUXiaO8Y7hjOZ8V3kG/WLZYyfmP8zfib/YH73+9/n1Wc +pZ+lD48tGMlo9WjeaN7+5fvz9+c/vv7Y5rENQNpXYJbZJgPeYhfeKBwiHDL9o/3yc1g/c2ft7fbb +7afKT+mc0gEQxMwyDGiGCYYJHxYfKx+r9ac2GG0wAtYRA5tlBiwGZAZk9n7ao7VHK78//1n+s+lf +WdAiQTPBzYKbn9g8XfB0AacR5wbODdM/JhmusEg2utmW25aLaoleEr1ErmPSlknMJOMk47E1x0aP +jcYuuGd9z3oqy2XA6Mlg8WDMYMwOxu3Htx/P781/kv9kdo4qEgc2h80Om8k1qkjIdoVFkr0lmyeb +Z0+to5ajFvAV+ETI7GQIZtv4Nt5tvJ76J+VOyiH3goG6FoxSGsTrx+rH7PLs59rPbTNsZW9lh7oR +lGRSZCplKrOWZ4tni6O7wZDryGS7wiIxljV2NnbWZNCU1pQm75FpzAswt9C3um51OfseCjsUNlQ4 +dHvoNtS1YGRFvF/5dv9bo7dGG6I2lGwogUcVaaFB12BXZldm8o4qEjIPLORJMGfSvRZ7LWbYznCa +4TR5j09j8sCkMqQOpA5YK1nvtt5d7VwtXk2HuxDNNtgEMGErr3Vc69gitdlss1mnQ6ddpx3UvaC3 +PG152/I2E37T46bHKXF8Mr8k/N7ZG2e1z2qHHbxWca2C+Dk4AUqchbawVLK8ZHnp4e1R7FG8ydvO +0s4SiQIDdS/YVHUwtte117npuZ13O/+y/mXGy4zZ+RbTP7FHsl9hv5LxJfNj5kex3WLHxSgysMh8 +hfU9JyanTU6bpFNlJmQmKHcW2jKqNLp0dKnnuOdjz8e7jzsscVjSEdpxoOMA1L1g/0odTMaBDJ4M +HvPT5jfMb7xseJn5MhMeVX9RBOMi5JrimkK5UUVCwSsskiKmwm+F32wP2nrbemMTwVDuXLSIV42X +h5fHQ9wj3CPcis86yzqL9LIa6l4wUOeFTuNOY59Kb0tvy0ffHn149AFfjM/CZ0Hda2bROakTpBMU +U33vwb0HqLOoKFQU5c5F8YFF4tvo6+Prc90w8nrkdUqfiyYpgjts6xrprtZdfaLqxOYTm+e7LOhd +0AsoEQOjoomgCdcJ1xizmO6Y7qtaV8avjPdz9+f3//JCKLMB+2H2Pex70k6lWaRZSPZKLZBaQOkz +UmlgjY2Cse62VrRWrDIAtwSB/Qx6DprAqsiq3qreae1BsYNic+/MrZhbMZuf6KE0nBpOHief/Srb +Mdvx4soLhAur2ve1UbVRgAD83/lP2AO7gd3+6wJ6A3o3qm08v/E8dU5LpYFF8snsk9YnLSs2SxNL +k5GSkbsjs+jT6r+HeSGzHrPees710uuldx9y7HbsFpUTPS96Hh5e00caUjmMz94+e0t4RRMbFPu+ +7v2j94+AKmJg/8r8oQXKAhV0Mkg5SBmRAoY656XqwCJJeJ9wLuGc25eji48uBpwJk9qemmenXUzp +TIlMiZa7LM9bnt+Zvktvl57cXrk3cm+AOGJgUzCOBq/0s+5kG2QbRLCFZ4dnV1ZWhlWGETcppcdt +SilAcp3kCckTKWwPHz98zHWGK4ErgZpnh2BgkVbsdKt32+m2M0EkPiQ+BL5e+FWoVShDlKE2s/aY +9tjWRduStiUZtBicMTjDcojlOAsF36OhMVZg6pnrC+sLU46k4FPw9wxjBmMGu+eBIa0lC3VFWsKS +DD6Uk2CUMJAwsKBfWUVZhfodoBhYRGNmYGyfb+TeyF0mXJZfBt/U/F2DYHjv897lvWsubeFqsc+S +2XKJ5UrV1apvVd+i8lHlqHKoK1JFI5h+7T6OPo4nQeAHbh843le6r1Q4r5CpkAnbga3Awvtp/yZE +DJjA4ECVQJV192w4bDggawLVwCL5qgzGysDykuWljoyOXR27oGpCV/jBiA+LN4g3GC829jb2Xqlg +cszkmOoa1UTVRCZRpsVMi2n+qpZ49dTh3aHXoUf4a7/oZVH28SyDLIM3R95se7NtbOmY4Jgg/Lgy +GRB/L+264MDrwHs867jmcU3AjxiIQDywSMo8y2zKbDZZ2SbZJo2uG5UclYS2Dx0aBleg53HnOcBz +QGtUS1BLUPeUbr5uvqaS5gnNE/LYecbzjJm8mK4xXZtpS+Lg5+Hn4ud2O3ev615Xtq1Muky6kLkw +sTAxvz+foOZKtUu1C+YMJhATSNw7eEq7B8OmbsXkio0rNl5zD+MN40X/iT6KPgptnxkxsEgy3TPn +Zs51+nBA5YAKphIM1I1mAWcw7NnsEewRCsMK/Ar8SiNKBkoGsglyX+QKpN5JlUo1iy0R8xHzEdws +eFjwMNMXJgwTBumJDEAGABrETP1JMdK2HaT34HaDmZQFM+QztGloU+t4a3lreaNi43Dj8JfzdV51 +XjXsNa9qXlUkgWkWaibApmNfYF/Ag4k6dFfo+uj6RFZc97ruxf6G/RP7jFh5ZQYNLJJMZMa7jHf7 +Jfa773fHA/gaPLyeFNRMwDCGMu5k3Ml3jC+DL4Mbw/2V+yvXdq6jhMhwG3EbMe9iIgR9liGNkMVo +UbQo7huODcc2mYPJJmTDpMqkyhB+CDeE6z/XH9gfOJA2EDwQ3HsWzMD7gfyBfLw3/iL+ItBEDAxS +cgZyu+R2PYx+6PzQmQXHysY6g24dzLiBRXJb+TbjbUbvy14VXhXYXVhjrDHUjWAw+rfAcMGWBVtu +t94h4H3M28DbAHWj/0XBDz9Px9YPWye2TpwR9brldQvxClGNmBU74sJgUJExl9kns+/G1ajtUdtn +5qgimaEDi8Su087EzuQY1iPZIxnBhpBAwHcuYDAyE8eJs4iz3MbeeXHnhSCHoJSgFNSN/s2MHliA +ARiHJIcOhw7XVa5PXZ8CVQgCqGvBYPRA9Izoc9Hnd8KiHaIdRK+JZohmQN3ov83Qe1g/4AAmeFFw +f3D/Re8LRReK8AAYGAz2q0RviSaIJtwtjnkQ80DSRdJf0h/qRlNFOwOL5BCY8MowhzCHs+fOos+i +8VZ4E7wJ1LVgMNogYQLmTuGdsDth4qUSCAkae70ys18S/tNlMI4nwC3yvc/7LPRZiPoEBupaMNhM +pzCkIKYgFqsTpxynTIujioTWrrC+R9yzJLMh0zvT22Xr4d7DvaM9o49HH0NdCwabWXRcdVJ1Uq/t +CmMOY+Zh4pHnoeHt8ml5YH3n+bfnD58/3HPG8bPj54n7E8ETwVA3gsGgp52r06HTcXPLTYmbEqzP +WatZaf7xIFp7SfgTyziXrV22NhWf1pbWxufH94HvA9SNYDCIsINZL70+dH3ovZp7affS6GNUkdDJ +wCKZdxFMwuvEY4nH5G7JY+QxUDeCwagHaQZmv9mfn/a/85PwT/JPQq0AA3UvcqKTl4T/NGQ8pDak +5t7uds3tWrpY+vn080AVUEIIDEZ3OJdxbuPcdrbsrOtZV9PLZqVmpaRnGKHuRX50O7BI8E5gbiXc +3HhzY0BwADoAPXFkYv3Eeqh7wWDkMT9lfs38mqtCQZ+CPknPkVaWVoa6EWXR+cD6XjlH2VhZr7OA +82PnV/VD9Tvrd9L8InawWQnZjOxB9th12vva+3rYeWR6ZDITA3UvaqCre1j/TnVQjVmNN1UnLSgt +6I+tf5T9UYYAEOsQM2ixOhjs3/Gb8O/m3x2mF9YZ1un10UvNS232jCqSWTSwSNhj2Z+wPznHFRgS +GBLaHuoW6sZXxC/GLwZ1LxjsJ5zBxQ5XhBqLGoumy2UwZTAZv1zJspKFXu9S/btZ9JLwZ7oauoq7 +ik+lnnI95Zp1PLMysxLPj2fFs0LdCzbbcetyr+Ze7XH3uMZxDZsTNjU2NQh/MFD3gtKsu8L6JwFJ +AU0BzdAPoXKhctcw1yquVQiXi6wWWQ10Qd0MNvsgHMGYB1iMW4xnaWSPZHf9gfpj2x/b4FFFAl9h +/cAQzxDTENPV9VdRV1G3mW4l3kqcSJvYP7Ef6l4weiazTSZIJsgz5qTNSZulW5Z6LvUEThAD+w48 +sP5DrXQtZy2nP4dfs19zrm+eQJ4A5sCkyKQI1L1gNI4VDJ8Z31m+s64vXA1dDa0OW8dbxzNtAlfH +h7rczAUPrKnhA1NjU6NVo3Uu8NyOczueH8ipy6nD5+LD8eFQl4PREu5H3G+53+412Ptu7zu7t/ay +9rJscmCg7kUb4IH164hrchUyFRYUFlyoOB92PqzwTpFxkTGgAd+qh/0YWwxbHlvelrytWlu1dvvs +Ht49zD0IBupetAceWNMjBubV3ldar7SCZYLbgtsK7xTIF8jjKwnXXrlQl4NBiWMVx1WOq5uY7TbZ +bdq5c+eLnS8ElMC3d6DuRdvggUU+dmCK7YvFi8UjT0QoRSg9e5sTmBOIlcd8w3yDuhyMwobB/yek +LbRHaI/9oc2jm0ftMu0q7Cp4LvAk8SRBXY5+wAOLMoi7HNfvrF9bvza68U7yneT7Vfd97/sOMA48 +GngECEBdD0YW2eDH6ZWlVbapbNvqsvX11tfmHOb7zPcx+4CBuhx9ggcWlQy5D9kO2aZuT92fuj9O +J/Zp7NOPth+ZPjLhs+Db9rSEbQT8+Klptpmmmaadt91Lu5dqamruau4IT4QLwgXqdvQPHlhQUAdT +4VCxqWLTg8v3191f96gg/UD6gQ7jdp52HqAbyCMENgOgPqE50ZwaWRqVGpVWClY+Vj6rzMxczFw4 +y7kmuCbgD89THzywZgSMLkYWI1u4q9Cp0Cl9V7peut6zU0/nPJ3T7t6h1qGGUIL/YFADej3aFe2q +Gqz6QfWD2Z5VkasiTUtMDUwNRF+AgbodDAQPrBkKo4IRxgiXc5YLlws/2fYE9aT3uehz7HPe2tWf +0Z/ROF5cJ64THmK/aRh8aJNdi30L+xbtZdou2i4r4o15jHmWfVqGWoaakw8GUCIGNsPAA4tm4O+A +aTNqU2lTye/P/5z/uVS9BFGCKCwqOll0svFKw9mGs5hHGDeMG2GQjRACuw++uObewH2a+7RS5Pyo ++VG6K3QSdBI06jW5NbkXOi28tvAay30wUBeFTRU8sGgf8U5Kf0D/yf6T73e8132vW3a9LKgsqNwI +vDr7uPjD1w9fu2O6/bv9cQdxxjhjqOuSlTUYpm9MckxyMjdk4mTilN+ryKrIqsqp7lDdsdCLMJSc +ZONk+2X70XVgoK4Lmy54YNE54nVZWEdih1eHV+1bwmVZee3buvS69AbLet563kYNMM2GLfNb5nce +69jUsWm4aqRkpASLxaRj0gm/PcD9NjUBASo8iEHcZRJfBXQD3QjiClBMc5k2MW3i/ARGdOfc0Lmh +4sliWWJZEs6SlyUvS7dKt0i3yB2VDZENkdoqfVX6KqsJqzWrNfxSjr7BAwv2F8wGzGrM6r6zffv6 +9nUld1V1VXUc7NDr0Os/1b+tf1ufRd+2vm0DXAMEQ0TjN8ZcxlwmlCc3EPJ2IooQnUkLQlZOik2K +Ieci7ZH2jHoMMoScYzxDSAJDPEM8Yzzjc8bnrNZsu9l2c7Swv2Z/zb2IZwfPDm457kXci/hwfCx8 +LHPezGmd08qrx2vMa8xxHQzwihjYrAcPLBgMRjP+vwAAAP//HU4zzmkfpbMAAAAASUVORK5CYII= +" + style="image-rendering:optimizeQuality" + preserveAspectRatio="none" + height="64" + width="64" /> + </g> + <g + inkscape:groupmode="layer" + id="layer3" + inkscape:label="PLACE YOUR PICTOGRAM HERE" + style="display:inline" /> + <g + inkscape:groupmode="layer" + id="layer2" + inkscape:label="BADGE" + style="display:none" + sodipodi:insensitive="true"> + <g + style="display:inline" + transform="translate(-340.00001,-581)" + id="g4394" + clip-path="none"> + <g + id="g855"> + <g + inkscape:groupmode="maskhelper" + id="g870" + clip-path="url(#clipPath873)" + style="opacity:0.6;filter:url(#filter891)"> + <path + transform="matrix(1.4999992,0,0,1.4999992,-29.999795,-237.54282)" + d="m 264,552.36218 a 12,12 0 0 1 -12,12 12,12 0 0 1 -12,-12 12,12 0 0 1 12,-12 12,12 0 0 1 12,12 z" + sodipodi:ry="12" + sodipodi:rx="12" + sodipodi:cy="552.36218" + sodipodi:cx="252" + id="path844" + style="color:#000000;fill:#000000;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:4;marker:none;visibility:visible;display:inline;overflow:visible;enable-background:accumulate" + sodipodi:type="arc" /> + </g> + <g + id="g862"> + <path + sodipodi:type="arc" + style="color:#000000;fill:#f5f5f5;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:4;marker:none;visibility:visible;display:inline;overflow:visible;enable-background:accumulate" + id="path4398" + sodipodi:cx="252" + sodipodi:cy="552.36218" + sodipodi:rx="12" + sodipodi:ry="12" + d="m 264,552.36218 a 12,12 0 0 1 -12,12 12,12 0 0 1 -12,-12 12,12 0 0 1 12,-12 12,12 0 0 1 12,12 z" + transform="matrix(1.4999992,0,0,1.4999992,-29.999795,-238.54282)" /> + <path + transform="matrix(1.25,0,0,1.25,33,-100.45273)" + d="m 264,552.36218 a 12,12 0 0 1 -12,12 12,12 0 0 1 -12,-12 12,12 0 0 1 12,-12 12,12 0 0 1 12,12 z" + sodipodi:ry="12" + sodipodi:rx="12" + sodipodi:cy="552.36218" + sodipodi:cx="252" + id="path4400" + style="color:#000000;fill:#dd4814;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:4;marker:none;visibility:visible;display:inline;overflow:visible;enable-background:accumulate" + sodipodi:type="arc" /> + <path + sodipodi:type="star" + style="color:#000000;fill:#f5f5f5;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:3;marker:none;visibility:visible;display:inline;overflow:visible;enable-background:accumulate" + id="path4459" + sodipodi:sides="5" + sodipodi:cx="666.19574" + sodipodi:cy="589.50385" + sodipodi:r1="7.2431178" + sodipodi:r2="4.3458705" + sodipodi:arg1="1.0471976" + sodipodi:arg2="1.6755161" + inkscape:flatsided="false" + inkscape:rounded="0.1" + inkscape:randomized="0" + d="m 669.8173,595.77657 c -0.39132,0.22593 -3.62645,-1.90343 -4.07583,-1.95066 -0.44938,-0.0472 -4.05653,1.36297 -4.39232,1.06062 -0.3358,-0.30235 0.68963,-4.03715 0.59569,-4.47913 -0.0939,-0.44198 -2.5498,-3.43681 -2.36602,-3.8496 0.18379,-0.41279 4.05267,-0.59166 4.44398,-0.81759 0.39132,-0.22593 2.48067,-3.48704 2.93005,-3.4398 0.44938,0.0472 1.81505,3.67147 2.15084,3.97382 0.3358,0.30236 4.08294,1.2817 4.17689,1.72369 0.0939,0.44198 -2.9309,2.86076 -3.11469,3.27355 -0.18379,0.41279 0.0427,4.27917 -0.34859,4.5051 z" + transform="matrix(1.511423,-0.16366377,0.16366377,1.511423,-755.37346,-191.93651)" /> + </g> + </g> + </g> + </g> +</svg> diff --git a/sourcecode/JOID/charm-k8s-ovn/layers/ovn/layer.yaml b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/layer.yaml new file mode 100644 index 0000000..44b8acf --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/layer.yaml @@ -0,0 +1,2 @@ +includes: ['layer:basic', 'interface:kubernetes-cni', 'interface:master-config'] +repo: 'https://github.com/AakashKT/charm-ovn.git' diff --git a/sourcecode/JOID/charm-k8s-ovn/layers/ovn/metadata.yaml b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/metadata.yaml new file mode 100644 index 0000000..767bb13 --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/metadata.yaml @@ -0,0 +1,19 @@ +name: ovn +summary: SDN via OVN +maintainer: Aakash <aakashkt0@gmail.com> +description: | + Overlay networking via OVN and OVS +tags: + - networking +subordinate: true +requires: + cni: + interface: kubernetes-cni + scope: container +peers: + master-config: + interface: master-config + +series: + - xenial + - trusty diff --git a/sourcecode/JOID/charm-k8s-ovn/layers/ovn/reactive/ovn.py b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/reactive/ovn.py new file mode 100644 index 0000000..675a3dd --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/reactive/ovn.py @@ -0,0 +1,491 @@ +import os +import json +import re +import sys +import subprocess +import time +import urllib.request as urllib2 +import multiprocessing as mp + +from charmhelpers.core import host + +from charmhelpers.core.hookenv import ( + open_port, + open_ports, + status_set, + config, + unit_public_ip, + unit_private_ip, +) + +from charmhelpers.core.host import ( + service_start, + service_stop, + log, + mkdir, + write_file, +) + +from charmhelpers.fetch import ( + apt_install, + apt_update, + apt_upgrade +) + +from charms.reactive.helpers import ( + mark_invoked, + was_invoked, +) + +from charms.reactive import ( + when, + when_not, + when_file_changed, + hook, + RelationBase, + scopes, + set_state, + remove_state +) + + + +CONF_FILE = '/tmp'; + + +######################################################################### +# Common functions +######################################################################### + +def run_command(command=None): + + if command is None: + return False; + + log('Running Command "%s"' % command); + try: + return subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT).decode('utf-8').replace('\n', ''); + except subprocess.CalledProcessError as e: + log('Error running "%s" : %s' % (command, e.output)); + + return False; + +def get_config(key): + conf = config(key); + return conf; + +def retrieve(key): + try: + conf = open('/tmp/ovn_conf', 'r'); + except: + return ''; + + plain_text = conf.read(); + conf.close(); + if plain_text == '': + return ''; + else: + data = json.loads(plain_text); + return data[key]; + +def store(key, value): + conf = open('/tmp/ovn_conf', 'r'); + plain_text = conf.read(); + conf.close(); + + conf = open('/tmp/ovn_conf', 'w+'); + + data = {}; + if plain_text != '': + data = json.loads(plain_text); + data[key] = value; + + conf.truncate(0); + conf.seek(0, 0); + conf.write(json.dumps(data)); + conf.close(); + + +######################################################################### +# Hooks and reactive handlers +######################################################################### + +''' Common reactive handlers ''' + +@when_not('deps.installed') +def install_deps(): + status_set('maintenance', 'Installing dependencies'); + + conf = open('/tmp/ovn_conf', 'w+'); + conf.close(); + + run_command('sudo apt-get update ; sudo apt-get upgrade ; sudo apt-get install git -y'); + run_command('sudo apt-get install -y build-essential fakeroot debhelper \ + autoconf automake bzip2 libssl-dev docker.io \ + openssl graphviz python-all procps \ + python-dev python-setuptools python-pip python3 python3.4 \ + python-twisted-conch libtool git dh-autoreconf \ + linux-headers-$(uname -r) libcap-ng-dev'); + run_command('sudo pip2 install six'); + + status_set('maintenance', 'Configure and make openvswitch'); + run_command('git clone https://github.com/openvswitch/ovs.git /tmp/ovs'); + + os.chdir('/tmp/ovs'); + + run_command('./boot.sh'); + run_command('./configure --prefix=/usr --localstatedir=/var --sysconfdir=/etc \ + --enable-ssl --with-linux=/lib/modules/`uname -r`/build'); + run_command('make -j3 ; sudo make install ; sudo make modules_install'); + + status_set('maintenance', 'Replacing kernel module'); + + run_command('sudo mkdir /etc/depmod.d/'); + run_command('for module in datapath/linux/*.ko; \ + do modname="$(basename ${module})" ; \ + echo "override ${modname%.ko} * extra" >> "/etc/depmod.d/openvswitch.conf" ; \ + echo "override ${modname%.ko} * weak-updates" >> "/etc/depmod.d/openvswitch.conf" ; \ + done'); + run_command('/sbin/modprobe openvswitch'); + + run_command('/usr/share/openvswitch/scripts/ovs-ctl start --system-id=$(uuidgen)'); + status_set('maintenance', 'Open vSwitch Installed'); + + run_command('git clone https://github.com/openvswitch/ovn-kubernetes /tmp/ovn-kubernetes'); + os.chdir('/tmp/ovn-kubernetes'); + run_command('sudo -H pip2 install .'); + + set_state('deps.installed'); + + + +''' Master reactive handlers and functions ''' + +def get_worker_subnet(): + ip3 = int(retrieve('ip3')); + store('ip3', ip3+1); + + return '192.168.%s.0/24' % ip3; + +@when('master.initialised') +def restart_services(): + new_interface = retrieve('new_interface'); + old_interface = retrieve('old_interface'); + + run_command('sudo ovn-k8s-watcher --overlay --pidfile \ + --log-file -vfile:info -vconsole:emer --detach'); + run_command('sudo ovn-k8s-gateway-helper --physical-bridge=%s \ + --physical-interface=%s \ + --pidfile --detach' % (new_interface, old_interface)); + +@when('master.initialised', 'master-config.worker.cert.available') +def sign_and_send(mconfig): + data = mconfig.get_worker_data(); + central_ip = get_my_ip(); + master_hostname = run_command('hostname'); + + signed_certs = {}; + for unit in data: + worker_hostname = unit['worker_hostname']; + + if not was_invoked(worker_hostname): + mark_invoked(worker_hostname); + cert = unit['cert_to_sign']; + worker_subnet = get_worker_subnet(); + + os.chdir('/tmp/'); + cert_file = open('/tmp/ovncontroller-req.pem', 'w+'); + cert_file.truncate(0); + cert_file.seek(0, 0); + cert_file.write(cert); + cert_file.close(); + run_command('sudo ovs-pki -d /certs/pki -b sign ovncontroller switch --force'); + + cert_file = open('ovncontroller-cert.pem', 'r'); + signed_cert = cert_file.read(); + + signed_certs[worker_hostname] = { + "central_ip": central_ip, + "signed_cert": signed_cert, + "master_hostname": master_hostname, + "worker_hostname": worker_hostname, + "worker_subnet": worker_subnet, + }; + + mconfig.send_signed_certs(signed_certs); + +@when('cni.is-master', 'master.initialised') +@when_not('gateway.installed') +def install_gateway(cni): + status_set('maintenance', 'Initialising gateway'); + + run_command('sudo ovs-vsctl set Open_vSwitch . external_ids:k8s-api-server="0.0.0.0:8080"'); + + run_command('git clone https://github.com/openvswitch/ovn-kubernetes /tmp/ovn-kubernetes'); + os.chdir('/tmp/ovn-kubernetes'); + run_command('sudo pip2 install .'); + + old_interface = get_interface(old=True); + new_interface = get_interface(old=False); + + op = run_command('ifconfig %s | grep "inet addr:"' % (new_interface)); + br_ip = op.lstrip().split()[1].replace('addr:', ''); + + gateway_ip = run_command('ip route | grep default').split(' ')[2]; + hostname = run_command('hostname'); + + op = run_command('ovn-k8s-overlay gateway-init \ + --cluster-ip-subnet="192.168.0.0/16" \ + --bridge-interface %s \ + --physical-ip %s/32 \ + --node-name="%s-gateway" \ + --default-gw %s' % (new_interface, br_ip, hostname, gateway_ip)); + log('Gateway init output: %s' % (op)); + + op = run_command('ovn-k8s-gateway-helper --physical-bridge=%s \ + --physical-interface=%s --pidfile --detach' % (new_interface, old_interface)); + log('Gateway Helper start: %s' % (op)); + + status_set('active', 'Master subnet : 192.168.1.0/24'); + set_state('gateway.installed'); + +@when('cni.is-master', 'master.setup.done') +@when_not('master.initialised') +def initialise_master(cni): + status_set('maintenance', 'Initialising master network'); + + central_ip = get_my_ip(); + hostname = run_command('hostname'); + + run_command('sudo ovs-vsctl set Open_vSwitch . external_ids:k8s-api-server="0.0.0.0:8080"'); + run_command('sudo ovn-k8s-overlay master-init --cluster-ip-subnet="192.168.0.0/16" \ + --master-switch-subnet="192.168.1.0/24" \ + --node-name="%s"' % (hostname)); + + run_command('sudo ovn-k8s-watcher --overlay --pidfile --log-file -vfile:info \ + -vconsole:emer --detach'); + + status_set('maintenance', 'Waiting for gateway'); + set_state('master.initialised'); + +@when('cni.is-master', 'bridge.setup.done') +@when_not('master.setup.done') +def master_setup(cni): + status_set('maintenance', 'Setting up master'); + open_port(6641); + open_port(6642); + open_port(8080); + + central_ip = get_my_ip(); + run_command('sudo /usr/share/openvswitch/scripts/ovn-ctl start_northd'); + run_command('sudo ovn-nbctl set-connection pssl:6641'); + run_command('sudo ovn-sbctl set-connection pssl:6642'); + + os.chdir('/etc/openvswitch'); + run_command('sudo ovs-pki -d /certs/pki init --force'); + run_command('sudo cp /certs/pki/switchca/cacert.pem /etc/openvswitch/'); + + run_command('sudo ovs-pki req ovnnb --force && sudo ovs-pki self-sign ovnnb --force'); + run_command('sudo ovn-nbctl set-ssl /etc/openvswitch/ovnnb-privkey.pem \ + /etc/openvswitch/ovnnb-cert.pem /certs/pki/switchca/cacert.pem'); + + run_command('sudo ovs-pki req ovnsb --force && sudo ovs-pki self-sign ovnsb --force'); + run_command('sudo ovn-sbctl set-ssl /etc/openvswitch/ovnsb-privkey.pem \ + /etc/openvswitch/ovnsb-cert.pem /certs/pki/switchca/cacert.pem'); + + run_command('sudo ovs-pki req ovncontroller'); + run_command('sudo ovs-pki -b -d /certs/pki sign ovncontroller switch --force'); + + ovn_host_file = open('/etc/default/ovn-host', 'a'); + ovn_host_file.write('OVN_CTL_OPTS="--ovn-controller-ssl-key=/etc/openvswitch/ovncontroller-privkey.pem \ + --ovn-controller-ssl-cert=/etc/openvswitch/ovncontroller-cert.pem \ + --ovn-controller-ssl-bootstrap-ca-cert=/etc/openvswitch/ovnsb-ca.cert"'); + ovn_host_file.close(); + + run_command('sudo ovs-vsctl set Open_vSwitch . external_ids:ovn-remote="ssl:%s:6642" \ + external_ids:ovn-nb="ssl:%s:6641" \ + external_ids:ovn-encap-ip=%s \ + external_ids:ovn-encap-type="%s"' % (central_ip, central_ip, central_ip, 'geneve')); + run_command('sudo /usr/share/openvswitch/scripts/ovn-ctl \ + --ovn-controller-ssl-key="/etc/openvswitch/ovncontroller-privkey.pem" \ + --ovn-controller-ssl-cert="/etc/openvswitch/ovncontroller-cert.pem" \ + --ovn-controller-ssl-bootstrap-ca-cert="/etc/openvswitch/ovnsb-ca.cert" \ + restart_controller'); + + set_state('master.setup.done'); + +@when('cni.is-master', 'master.kv.setup') +@when_not('bridge.setup.done') +def bridge_setup(cni): + status_set('maintenance', 'Setting up new interface'); + + interface = get_config('gateway-physical-interface'); + if interface == 'none' or interface == None: + op = run_command('ip route | grep default').split(' '); + interface = op[4]; + + store('old_interface', interface); + store('new_interface', 'br%s' % (interface)); + + op = run_command('ovn-k8s-util nics-to-bridge %s' % (interface)); + log('Bridge create output: %s' % (op)); + + op = run_command('dhclient -r br%s' % (interface)); + op = run_command('dhclient br%s' % (interface)); + + status_set('maintenance', 'Waiting to initialise master'); + set_state('bridge.setup.done'); + +@when('cni.is-master', 'deps.installed') +@when_not('master.kv.setup') +def setup_master_kv(cni): + store('ip3', '2'); + + set_state('master.kv.setup'); + + +''' Worker reactive handlers and functions ''' + +@when('cni.is-worker', 'worker.data.registered') +@when_not('k8s.worker.certs.setup') +def setup_k8s_worker_certs(cni): + if os.path.isfile('/root/cdk/kubeconfig') and os.path.isfile('/root/cdk/ca.crt'): + set_state('k8s.worker.certs.setup'); + + master_hostname = retrieve('master_hostname'); + + k8s_api_ip = "%s:6443" % (master_hostname); + api_token = run_command('sudo awk \'$1=="token:" {print $2}\' /root/cdk/kubeconfig'); + + run_command('sudo cp /root/cdk/ca.crt /etc/openvswitch/k8s-ca.crt'); + run_command('sudo ovs-vsctl set Open_vSwitch . \ + external_ids:k8s-api-server="https://%s" \ + external_ids:k8s-api-token="%s"' % (k8s_api_ip, api_token)); + + +@when('cni.is-worker', 'worker.setup.done') +@when_not('worker.initialised') +def initialise_worker(cni): + status_set('maintenance', 'Initialising worker network'); + + local_ip = get_my_ip(); + worker_subnet = retrieve('worker_subnet'); + central_ip = retrieve('central_ip'); + hostname = run_command('hostname').replace('\n', ''); + + run_command('ovs-vsctl set Open_vSwitch . \ + external_ids:k8s-api-server="%s:8080"' % (central_ip)); + run_command('ovn-k8s-overlay minion-init --cluster-ip-subnet="192.168.0.0/16" \ + --minion-switch-subnet="%s" --node-name="%s"' % (worker_subnet, hostname)); + + os.chdir('/tmp/'); + run_command('wget https://github.com/containernetworking/cni/releases/download/v0.5.2/cni-amd64-v0.5.2.tgz'); + run_command('sudo mkdir -p /opt/cni/bin'); + run_command('sudo mkdir -p /etc/cni/net.d'); + os.chdir('/opt/cni/bin/'); + run_command('sudo tar xvzf /tmp/cni-amd64-v0.5.2.tgz'); + + status_set('active', 'Worker subnet : %s' % (worker_subnet)); + set_state('worker.initialised'); + +@when('cni.is-worker', 'worker.data.registered') +@when_not('worker.setup.done') +def worker_setup(cni): + status_set('maintenance', 'Setting up worker'); + open_port(8080); + + central_ip = retrieve('central_ip'); + local_ip = get_my_ip(); + + run_command('sudo ovs-vsctl set Open_vSwitch . \ + external_ids:ovn-remote="ssl:%s:6642" \ + external_ids:ovn-nb="ssl:%s:6641" \ + external_ids:ovn-encap-ip=%s \ + external_ids:ovn-encap-type=%s' % (central_ip, central_ip, local_ip, 'geneve')); + + ovn_host_file = open('/etc/default/ovn-host', 'a'); + ovn_host_file.write('OVN_CTL_OPTS="--ovn-controller-ssl-key=/etc/openvswitch/ovncontroller-privkey.pem \ + --ovn-controller-ssl-cert=/etc/openvswitch/ovncontroller-cert.pem \ + --ovn-controller-ssl-bootstrap-ca-cert=/etc/openvswitch/ovnsb-ca.cert"'); + ovn_host_file.close(); + + run_command('sudo /usr/share/openvswitch/scripts/ovn-ctl \ + --ovn-controller-ssl-key="/etc/openvswitch/ovncontroller-privkey.pem" \ + --ovn-controller-ssl-cert="/etc/openvswitch/ovncontroller-cert.pem" \ + --ovn-controller-ssl-bootstrap-ca-cert="/etc/openvswitch/ovnsb-ca.cert" \ + restart_controller'); + set_state('worker.setup.done'); + +@when('cni.is-worker', 'master-config.master.data.available', 'worker.cert.sent') +@when_not('worker.data.registered') +def receive_data(cni, mconfig): + status_set('maintenance', 'Certificate received') + worker_hostname = run_command('hostname'); + + data = mconfig.get_signed_cert(worker_hostname); + cert = data['signed_cert']; + worker_subnet = data['worker_subnet']; + master_ip = data['central_ip']; + master_hostname = data['master_hostname']; + + store('master_hostname', master_hostname); + store('worker_subnet', worker_subnet); + store('central_ip', master_ip); + cni.set_config(cidr='192.168.0.0/16'); + + os.chdir('/etc/openvswitch'); + cert_file = open('/etc/openvswitch/ovncontroller-cert.pem', 'a'); + cert_file.write(cert); + cert_file.close(); + + set_state('worker.data.registered'); + +@when('cni.is-worker', 'master-config.connected', 'worker.kv.setup') +@when_not('worker.cert.sent') +def send_cert(cni, mconfig): + worker_hostname = run_command('hostname'); + mconfig.set_worker_id(worker_hostname); + + os.chdir('/etc/openvswitch'); + run_command('sudo ovs-pki req ovncontroller'); + + req_file = open('ovncontroller-req.pem', 'r'); + cert = req_file.read(); + mconfig.send_worker_data({ + 'cert_to_sign': cert, + 'worker_hostname': worker_hostname + }); + + status_set('maintenance', 'Waiting for certificate'); + set_state('worker.cert.sent'); + +@when('cni.is-worker', 'deps.installed') +@when_not('worker.kv.setup') +def setup_worker_kv(cni): + hostname = run_command('hostname'); + interface = get_config('gateway-physical-interface'); + if interface == 'none' or interface == None: + op = run_command('ip route | grep default').split(' '); + interface = op[4]; + + store('new_interface', interface); + store('worker_hostname', hostname); + set_state('worker.kv.setup'); + + +######################################################################### +# Helper functions +######################################################################### + + +def get_my_ip(): + interface = get_interface(old=False); + + op = run_command('ifconfig %s | grep "inet addr:"' % (interface)); + br_ip = op.lstrip().split()[1].replace('addr:', ''); + + return br_ip; + +def get_interface(old): + key = 'old_interface' if old == True else 'new_interface'; + return retrieve(key); diff --git a/sourcecode/JOID/charm-k8s-ovn/layers/ovn/tests/00-setup b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/tests/00-setup new file mode 100755 index 0000000..f0616a5 --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/tests/00-setup @@ -0,0 +1,5 @@ +#!/bin/bash + +sudo add-apt-repository ppa:juju/stable -y +sudo apt-get update +sudo apt-get install amulet python-requests -y diff --git a/sourcecode/JOID/charm-k8s-ovn/layers/ovn/tests/10-deploy b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/tests/10-deploy new file mode 100755 index 0000000..1cd905f --- /dev/null +++ b/sourcecode/JOID/charm-k8s-ovn/layers/ovn/tests/10-deploy @@ -0,0 +1,35 @@ +#!/usr/bin/python3 + +import amulet +import requests +import unittest + + +class TestCharm(unittest.TestCase): + def setUp(self): + self.d = amulet.Deployment() + + self.d.add('OVN') + self.d.expose('OVN') + + self.d.setup(timeout=900) + self.d.sentry.wait() + + self.unit = self.d.sentry['OVN'][0] + + def test_service(self): + # test we can access over http + page = requests.get('http://{}'.format(self.unit.info['public-address'])) + self.assertEqual(page.status_code, 200) + # Now you can use self.d.sentry[SERVICE][UNIT] to address each of the units and perform + # more in-depth steps. Each self.d.sentry[SERVICE][UNIT] has the following methods: + # - .info - An array of the information of that unit from Juju + # - .file(PATH) - Get the details of a file on that unit + # - .file_contents(PATH) - Get plain text output of PATH file from that unit + # - .directory(PATH) - Get details of directory + # - .directory_contents(PATH) - List files and folders in PATH on that unit + # - .relation(relation, service:rel) - Get relation data from return service + + +if __name__ == '__main__': + unittest.main() @@ -0,0 +1,17 @@ +[tox] +minversion = 1.6 +envlist = + docs, + docs-linkcheck +skipsdist = true + +[testenv:docs] +deps = -rdocs/requirements.txt +commands = + sphinx-build -b html -n -d {envtmpdir}/doctrees ./docs/ {toxinidir}/docs/_build/html + echo "Generated docs available in {toxinidir}/docs/_build/html" +whitelist_externals = echo + +[testenv:docs-linkcheck] +deps = -rdocs/requirements.txt +commands = sphinx-build -b linkcheck -d {envtmpdir}/doctrees ./docs/ {toxinidir}/docs/_build/linkcheck |