summaryrefslogtreecommitdiffstats
path: root/deploy/environments/libvirt_environment.py
blob: c8a2ef529711db0cb907248ac83e55ac914fdbc6 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
###############################################################################
# Copyright (c) 2015 Ericsson AB and others.
# szilard.cserey@ericsson.com
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
###############################################################################


from lxml import etree
import glob
from execution_environment import ExecutionEnvironment
import tempfile

from common import (
    exec_cmd,
    log,
    check_dir_exists,
    check_file_exists,
    delete,
)


class LibvirtEnvironment(ExecutionEnvironment):

    def __init__(self, storage_dir, dha_file, dea, root_dir):
        super(LibvirtEnvironment, self).__init__(
            storage_dir, dha_file, root_dir)
        self.dea = dea
        self.network_dir = '%s/%s' % (self.root_dir,
                                      self.dha.get_virt_net_conf_dir())
        self.node_ids = self.dha.get_all_node_ids()
        self.net_names = self.collect_net_names()

    def create_storage(self, node_id, disk_path, disk_sizes):
        if node_id == self.fuel_node_id:
            disk_size = disk_sizes['fuel']
        else:
            roles = self.dea.get_node_role(node_id)
            role = 'controller' if 'controller' in roles else 'compute'
            disk_size = disk_sizes[role]
        exec_cmd('fallocate -l %s %s' % (disk_size, disk_path))

    def create_vms(self):
        temp_dir = tempfile.mkdtemp()
        disk_sizes = self.dha.get_disks()
        for node_id in self.node_ids:
            vm_name = self.dha.get_node_property(node_id, 'libvirtName')
            vm_template = '%s/%s' % (self.root_dir,
                                     self.dha.get_node_property(
                                         node_id, 'libvirtTemplate'))
            check_file_exists(vm_template)
            disk_path = '%s/%s.raw' % (self.storage_dir, vm_name)
            self.create_storage(node_id, disk_path, disk_sizes)
            temp_vm_file = '%s/%s' % (temp_dir, vm_name)
            exec_cmd('cp %s %s' % (vm_template, temp_vm_file))
            self.define_vm(vm_name, temp_vm_file, disk_path)
        delete(temp_dir)

    def start_vms(self):
        for node_id in self.node_ids:
            self.dha.node_power_on(node_id)

    def create_networks(self):
        for net_file in glob.glob('%s/*' % self.network_dir):
            exec_cmd('virsh net-define %s' % net_file)
        for net in self.net_names:
            log('Creating network %s' % net)
            exec_cmd('virsh net-autostart %s' % net)
            exec_cmd('virsh net-start %s' % net)

    def delete_networks(self):
        for net in self.net_names:
            log('Deleting network %s' % net)
            exec_cmd('virsh net-destroy %s' % net, False)
            exec_cmd('virsh net-undefine %s' % net, False)

    def get_net_name(self, net_file):
        with open(net_file) as f:
            net_xml = etree.parse(f)
            name_list = net_xml.xpath('/network/name')
            for name in name_list:
                net_name = name.text
        return net_name

    def collect_net_names(self):
        net_list = []
        for net_file in glob.glob('%s/*' % self.network_dir):
            name = self.get_net_name(net_file)
            net_list.append(name)
        return net_list

    def delete_vms(self):
        for node_id in self.node_ids:
            self.delete_vm(node_id)


    def setup_environment(self):
        check_dir_exists(self.network_dir)
        self.cleanup_environment()
        self.create_networks()
        self.create_vms()
        self.start_vms()

    def cleanup_environment(self):
        self.delete_vms()
        self.delete_networks()
ores and 16GB of memory. Google Kubernetes Engine (GKE) clusters are supported. * Installation of Istio in the k8s cluster. See :ref:`sdc_deploy_container`. * Clover CLI (**cloverctl**) has been downloaded and setup. Instructions to deploy can be found at :ref:`controller_services_controller`. Deploy with Clover CLI ---------------------- To deploy the visibility services into your k8s cluster use the **cloverctl** CLI command shown below:: $ cloverctl create system visibility Container images with the Gambia release tag will pulled if the tag is unspecified. The release tag is **opnfv-7.0.0** for the Gambia release. To deploy the latest containers from master, use the command shown below:: $ cloverctl create system visibility -t latest Using config file: /home/earrage/.cloverctl.yaml Creating visibility services Created clover-system namespace Created statefulset "cassandra". Created service "cassandra" Created pod "redis". Created service "redis" Created deployment "clover-collector". Image: opnfv/clover-collector:latest Created service "clover-collector" Created deployment "clover-controller". Image: opnfv/clover-controller:latest Created service "clover-controller-internal" Created serviceaccount "clover-spark". Created clusterrolebinding "clover-spark-default". Created clusterrolebinding "clover-spark". Created deployment "clover-spark-submit". Image: opnfv/clover-spark-submit:latest Verifying the deployment ------------------------ To verify the visibility services deployment, ensure the following pods have been deployed with the command below:: $ kubectl get pod --all-namespaces NAMESPACE NAME READY STATUS clover-system clover-collector-7dcc5d849f-6jc6m 1/1 Running clover-system clover-controller-74d8596bb5-qrr6b 1/1 Running clover-system cassandra-0 1/1 Running clover-system redis 2/2 Running clover-system clover-spark-submit-6c4d5bcdf8-kc6l9 1/1 Running Additionally, spark driver and executor pods will continuously be deployed as displayed below:: clover-system clover-spark-0fa43841362b3f27b35eaf6112965081-driver clover-system clover-spark-fast-d5135cdbdd8330f6b46431d9a7eb3c20-driver clover-system clover-spark-0fa43841362b3f27b35eaf6112965081-exec-3 clover-system clover-spark-0fa43841362b3f27b35eaf6112965081-exec-4 Initializing visibility services ================================ In order to setup visibility services, initialization and start commands must be invoked from the **cloverctl** CLI. There are sample yaml files in yaml directory from the **cloverctl** binary path. Navigate to this directory to execute the next sequence of commands. Initialize the visibility schemas in cassandra with the following command:: $ cloverctl init visibility Using config file: /home/earrage/.cloverctl.yaml clover-controller address: http://10.145.71.21:32044 Added visibility schemas in cassandra The initial configuration to the visibility services are the Jaeger tracing and Prometheus connection parameters and sample interval to **clover-collector**. To start visibility use the sample yaml provided and execute the command:: cloverctl start visibility -f start_visibility.yaml Started collector on pid: 44 The ``start_visibility.yaml`` has defaults for the tracing and monitoring modules packaged with Istio 1.0.0. Configure and control visibility ================================ The core requirement for Clover visibility services to function, is for your services to be added to the Istio service mesh. Istio deployment and usage instructions are in the :ref:`sdc_config_guide` and the Service Delivery Controller (SDC) sample can be used to evaluate the Clover visibility services initially. A user may inject their own web-based services into the service mesh and track separately. Connecting to visibility dashboard UI ------------------------------------- The **clover-controller** service comes packaged with a web-based UI with a visibility view. To access the dashboard, navigate to the **clover-controller** address for either a ``NodePort`` or ``LoadBalancer`` service * http://<node or CNI IP address>:<``NodePort`` port>/ * http://<``LoadBalancer`` IP address>/ See :ref:`exposing_clover_controller` to expose **clover-controller** externally with a k8s service. Set runtime parameters using Clover CLI --------------------------------------- The services visibility will track are based on the deployment/pod names specified in the k8s resources. Using some sample services from the SDC guide, the **proxy-access-control**, **clover-server1**, **clover-server2** and **clover-server3** services are specified in the ``set_visibility.yaml`` sample yaml referenced below. To modify the configuration of the services visibility will track, use the **cloverctl CLI**, executing the following command:: cloverctl set visibility -f set_visibility.yaml Use the ``services:`` section of the yaml to configure service names to track. .. code-block:: bash # set_visibility.yaml services: - name: proxy_access_control - name: clover_server1 - name: clover_server2 - name: clover_server3 metric_prefixes: - prefix: envoy_cluster_outbound_9180__ - prefix: envoy_cluster_inbound_9180__ metric_suffixes: - suffix: _default_svc_cluster_local_upstream_rq_2xx - suffix: _default_svc_cluster_local_upstream_cx_active custom_metrics: - metric: envoy_tracing_zipkin_spans_sent Set runtime parameters using dashboard UI ----------------------------------------- The services being tracked by visibility can also be configured by selecting from the boxes under **Discovered Services** within the dashboard, as shown in the graphic below. Services can be multi-selected by using by holding the ``Ctrl`` or ``command`` (Mac OS) keyboard button down while selecting or unselecting. The SDC services that were configured from the **cloverctl** CLI above are currently active, denoted as the boxes with blue backgrounds. .. image:: imgs/visibility_discovered_active.png :align: center :scale: 100% In order for any services to be discovered from Jaeger tracing and displayed within the dashboard, some traffic must target the services of interest. Using curl/wget to send HTTP requests to your services will cause services to be discovered. Using Clover JMeter validation services, as detailed :ref:`jmeter_config_guide` against SDC sample services will also generate a service listing. The **cloverctl** CLI commands below will generate traces through the SDC service chain with the JMeter master injected into the service mesh:: $ cloverctl create testplan –f yaml/jmeter_testplan.yaml # yaml located with cloverctl binary $ cloverctl start testplan Clearing visibility data ------------------------- To clear visibility data in cassandra and redis, which truncates **cassandra** tables and deletes or zeros out **redis** keys, use the following command:: $ cloverctl clear visibility This can be useful when analyzing or observing an issue during a particular time horizon. The same function can be performed from the dashboard UI using the ``Clear`` button under ``Visibility Controls``, as illustrated in the graphic from the previous section. Viewing visibility data ======================== The visibility dashboard can be used to view visibility data in real-time. The page will automatically refresh every 5 seconds. To disable continuous page refresh and freeze on a snapshot of the data, use the slider at the top of the page that defaults to ``On``. Toggling it will result in it displaying ``Off``. The visibility dashboard displays various metrics and graphs of analyzed data described in subsequent sections. System metrics -------------- System metrics provide aggregate counts of cassandra tables including total traces, spans and metrics, as depicted on the left side of the graphic below. .. image:: imgs/visibility_system_counts_response_times.png :align: center :scale: 100% The metrics counter will continually increase, as it is based on time series data from Prometheus. The trace count will correspond to the number of HTTP requests sent to services within the Istio service mesh. The span count ties to trace count, as it is a child object under Jaeger tracing data hierarchy and is based on the service graph (number of interactions between microservices for a given request). It will increase more rapidly when service graph depths are larger. Per service response times -------------------------- Per service response times are displayed on the right side of the graphic above and are calculated from tracing data when visibility is started. The minimum, maximum and average response times are output over the entire analysis period. Group by span field counts -------------------------- This category groups schema fields in various combinations to gain insight into the composition of HTTP data and can be used by CI scripts to perform various validations. Metrics include: * Per service * Distinct URL * Distinct URL / HTTP status code * Distinct user-agent (HTTP header) * Per service / distinct URL The dashboard displays bar/pie charts with counts and percentages, as depicted below. Each distinct key is displayed when hovering your mouse over a chart value. .. image:: imgs/visibility_distinct_counts.png :align: center :scale: 100% Distinct HTTP details --------------------- A listing of distinct HTTP user-agents, request URLs and status codes is shown below divided with tabs. .. image:: imgs/visibility_distinct_http.png :align: center :scale: 100% Monitoring Metrics ------------------ The Istio sidecars (Envoy) provide a lengthy set of metrics exposed through Prometheus. These metrics can be analyzed with the visibility service by setting up metrics, as outlined in section `Set runtime parameters using Clover CLI`_. Use ``metric_prefixes`` and ``metric_suffixes`` sections of the set visibility yaml for many Envoy metrics that have a key with the service straddled by a prefix/suffix. A row in the table and a graph will be displayed for each combination of service, prefix and suffix. The metrics are displayed in tabular and scatter plots over time formats from the dashboard, as shown in the graphic below: .. image:: imgs/visibility_monitoring_metrics.png :align: center :scale: 100% Uninstall from Kubernetes envionment ==================================== Delete with Clover CLI ---------------------- When you're finished working with Clover visibility services, you can uninstall them with the following command:: $ cloverctl delete system visibility The command above will remove the SDC sample services, Istio components and Jaeger/Prometheus tools from your Kubernetes environment. Uninstall from Docker environment ================================= The OPNFV docker images can be removed with the following commands: .. code-block:: bash $ docker rmi opnfv/clover-collector $ docker rmi opnfv/clover-spark $ docker rmi opnfv/clover-spark-submit $ docker rmi opnfv/clover-controller $ docker rmi k8s.gcr.io/redis $ docker rmi kubernetes/redis $ docker rmi cassandra:3