From 6b4f4a7389915f69d268f13d151bd32d39e8e23c Mon Sep 17 00:00:00 2001 From: Bryan Sullivan Date: Tue, 7 Nov 2017 15:54:47 -0800 Subject: Add cloudify API method for demo start/stop JIRA: MODELS-2 more bug fixes Change-Id: Ic617fd5ca030eed0beb850f97f54fa213fee87e3 Signed-off-by: Bryan Sullivan --- .../blueprints/k8s-hello-world/blueprint.yaml | 37 ++++- tools/cloudify/k8s-cloudify.sh | 184 ++++++++++++++++++--- tools/kubernetes/demo_deploy.sh | 4 +- 3 files changed, 192 insertions(+), 33 deletions(-) diff --git a/tools/cloudify/blueprints/k8s-hello-world/blueprint.yaml b/tools/cloudify/blueprints/k8s-hello-world/blueprint.yaml index bdfa80c..bdbba8c 100644 --- a/tools/cloudify/blueprints/k8s-hello-world/blueprint.yaml +++ b/tools/cloudify/blueprints/k8s-hello-world/blueprint.yaml @@ -1,5 +1,7 @@ tosca_definitions_version: cloudify_dsl_1_3 -# Following part based upon https://github.com/cloudify-incubator/cloudify-kubernetes-plugin/blob/master/examples/simple-blueprint-defined-resource.yaml +# Based upon +# https://github.com/cloudify-incubator/cloudify-kubernetes-plugin/blob/master/examples/simple-blueprint-defined-resource.yaml +# https://github.com/cloudify-incubator/cloudify-kubernetes-plugin/blob/1.2.0/examples/example-blueprint.yaml imports: - http://www.getcloudify.org/spec/cloudify/3.4/types.yaml @@ -7,6 +9,12 @@ imports: inputs: + spec_port: + default: 80 + + container_port: + default: 80 + kubernetes_configuration_file_content: default: kube.config @@ -55,12 +63,31 @@ inputs: default: { get_input: kubernetes_master_configuration } node_templates: - master: + kubernetes_master: type: cloudify.kubernetes.nodes.Master properties: configuration: file_content: { get_input: kubernetes_configuration_file_content } + nginx_service: + type: cloudify.kubernetes.resources.Service + properties: + definition: + apiVersion: v1 + metadata: + name: nginx-service + spec: + type: NodePort + ports: + - port: { get_input: spec_port } + selector: + app: nginx + relationships: + - type: cloudify.kubernetes.relationships.managed_by_master + target: kubernetes_master + - type: cloudify.relationships.depends_on + target: nginx_pod + # Following part based upon http://docs.getcloudify.org/4.1.0/plugins/kubernetes/ nginx_pod: type: cloudify.kubernetes.resources.Pod @@ -69,12 +96,14 @@ node_templates: apiVersion: v1 metadata: name: nginx + labels: + app: nginx spec: containers: - name: nginx image: nginx:1.7.9 ports: - - containerPort: 80 + - containerPort: { get_input: container_port } relationships: - type: cloudify.kubernetes.relationships.managed_by_master - target: master + target: kubernetes_master diff --git a/tools/cloudify/k8s-cloudify.sh b/tools/cloudify/k8s-cloudify.sh index d40f235..bdebf70 100644 --- a/tools/cloudify/k8s-cloudify.sh +++ b/tools/cloudify/k8s-cloudify.sh @@ -23,18 +23,28 @@ #. : IP or hostname of kubernetes master server #. $ ssh -x ubuntu@ cloudify/k8s-cloudify.sh prereqs #. prereqs: installs prerequisites and configures ubuntu user for kvm use -#. $ ssh -x ubuntu@ bash cloudify/k8s-cloudify.sh [setup|clean] +#. $ ssh -x ubuntu@ bash cloudify/k8s-cloudify.sh setup +#. setup: installs cloudify CLI and Manager +#. $ source ~/models/tools/cloudify/k8s-cloudify.sh demo +#. demo: control demo blueprint +#. start|stop: start or stop the demo +#. : IP or hostname of kubernetes master server +#. $ ssh -x ubuntu@ bash cloudify/k8s-cloudify.sh clean +#. clean: uninstalls cloudify CLI and Manager + #. Status: this is a work in progress, under test. function log() { f=$(caller 0 | awk '{print $2}') l=$(caller 0 | awk '{print $1}') + echo "" echo "$f:$l ($(date)) $1" } function prereqs() { log "Install prerequisites" - sudo apt-get install -y virtinst qemu-kvm libguestfs-tools virtualenv git python-pip + sudo apt-get install -y virtinst qemu-kvm libguestfs-tools virtualenv git \ + python-pip log "Setup $USER for kvm use" # Per http://libguestfs.org/guestfs-faq.1.html # workaround for virt-customize warning: libguestfs: warning: current user is not a member of the KVM group (group ID 121). This user cannot access /dev/kvm, so libguestfs may run very slowly. It is recommended that you 'chmod 0666 /dev/kvm' or add the current user to the KVM group (you might need to log out and log in again). @@ -61,7 +71,11 @@ function setup () { # sudo virsh destroy cloudify-manager; sudo virsh undefine cloudify-manager wget -q http://repository.cloudifysource.org/cloudify/17.9.21/community-release/cloudify-manager-community-17.9.21.qcow2 # nohup and redirection of output is a workaround for some issue with virt-install never outputting anything beyond "Creadint domain..." and thus not allowing the script to continue. - nohup virt-install --connect qemu:///system --virt-type kvm --name cloudify-manager --vcpus 4 --memory 16192 --disk cloudify-manager-community-17.9.21.qcow2 --import --network network=default --os-type=linux --os-variant=rhel7 > /dev/null 2>&1 & + nohup virt-install --connect qemu:///system --virt-type kvm \ + --name cloudify-manager --vcpus 4 --memory 16192 \ + --disk cloudify-manager-community-17.9.21.qcow2 --import \ + --network network=default --os-type=linux \ + --os-variant=rhel7 > /dev/null 2>&1 & VM_IP="" n=0 @@ -86,20 +100,45 @@ function setup () { # From https://github.com/cloudify-incubator/cloudify-kubernetes-plugin/releases wget -q https://github.com/cloudify-incubator/cloudify-kubernetes-plugin/releases/download/1.2.1/cloudify_kubernetes_plugin-1.2.1-py27-none-linux_x86_64-centos-Core.wgn # For Cloudify-CLI per http://docs.getcloudify.org/4.1.0/plugins/using-plugins/ - wagon install cloudify_kubernetes_plugin-1.2.1-py27-none-linux_x86_64-centos-Core.wgn + wagon install \ + cloudify_kubernetes_plugin-1.2.1-py27-none-linux_x86_64-centos-Core.wgn # For Cloudify-Manager per https://github.com/cloudify-incubator/cloudify-kubernetes-plugin/blob/master/examples/persistent-volumes-blueprint.yaml cfy plugins upload cloudify_kubernetes_plugin-1.2.1-py27-none-linux_x86_64-centos-Core.wgn log "Create secrets for kubernetes as referenced in blueprints" - cfy secrets create -s $(grep server ~/.kube/config | awk -F '/' '{print $3}' | awk -F ':' '{print $1}') kubernetes_master_ip - cfy secrets create -s $(grep server ~/.kube/config | awk -F '/' '{print $3}' | awk -F ':' '{print $2}') kubernetes_master_port - cfy secrets create -s $(grep 'certificate-authority-data: ' ~/.kube/config | awk -F ' ' '{print $2}') kubernetes_certificate_authority_data - cfy secrets create -s $(grep 'client-certificate-data: ' ~/.kube/config | awk -F ' ' '{print $2}') kubernetes-admin_client_certificate_data - cfy secrets create -s $(grep 'client-key-data: ' ~/.kube/config | awk -F ' ' '{print $2}') kubernetes-admin_client_key_data + cfy secrets create -s $(grep server ~/.kube/config | awk -F '/' '{print $3}' \ + | awk -F ':' '{print $1}') kubernetes_master_ip + cfy secrets create -s $(grep server ~/.kube/config | awk -F '/' '{print $3}' \ + | awk -F ':' '{print $2}') kubernetes_master_port + cfy secrets create -s \ + $(grep 'certificate-authority-data: ' ~/.kube/config | \ + awk -F ' ' '{print $2}') kubernetes_certificate_authority_data + cfy secrets create -s $(grep 'client-certificate-data: ' ~/.kube/config \ + | awk -F ' ' '{print $2}') kubernetes-admin_client_certificate_data + cfy secrets create -s $(grep 'client-key-data: ' ~/.kube/config \ + | awk -F ' ' '{print $2}') kubernetes-admin_client_key_data cfy secrets list + # get manager VM IP + VM_MAC=$(sudo virsh domiflist cloudify-manager | grep default | grep -Eo "[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+:[0-9a-f\]+") + VM_IP=$(/usr/sbin/arp -e | grep ${VM_MAC} | awk {'print $1'}) + + # get host IP + HOST_IP=$(ip route get 8.8.8.8 | awk '{print $NF; exit}') + + # Forward host port 80 to VM + sudo iptables -t nat -I PREROUTING -p tcp -d $HOST_IP --dport 80 -j DNAT --to-destination $VM_IP:80 + sudo iptables -I FORWARD -m state -d $VM_IP/32 --state NEW,RELATED,ESTABLISHED -j ACCEPT + sudo iptables -t nat -A POSTROUTING -j MASQUERADE + + while ! curl -u admin:admin --header 'Tenant: default_tenant' http://$HOST_IP/api/v3.1/status ; do + log "Cloudify API is not yet responding, waiting 10 seconds" + sleep 10 + done log "Cloudify CLI config is at ~/.cloudify/config.yaml" log "Cloudify CLI log is at ~/.cloudify/logs/cli.log" + log "Cloudify API access example: curl -u admin:admin --header 'Tenant: default_tenant' http://$HOST_IP/api/v3.1/status" + log "Cloudify setup is complete!" } function demo() { @@ -110,25 +149,116 @@ function demo() { # echo "master-port: $(grep server ~/.kube/config | awk -F '/' '{print $3}' | awk -F ':' '{print $2}')" >>~/cloudify/blueprints/k8s-hello-world/inputs.yaml # echo "file_content:" >>~/cloudify/blueprints/k8s-hello-world/inputs.yaml # sed 's/^/ /' ~/.kube/config | tee -a ~/cloudify/blueprints/k8s-hello-world/inputs.yaml - cp ~/.kube/config ~/cloudify/blueprints/k8s-hello-world/kube.config - - cfy blueprints package -o ~/cloudify/blueprints/k8s-hello-world ~/cloudify/blueprints/k8s-hello-world - cfy blueprints upload -t default_tenant -b k8s-hello-world ~/cloudify/blueprints/k8s-hello-world.tar.gz - cfy deployments create -t default_tenant -b k8s-hello-world k8s-hello-world - cfy workflows list -d k8s-hello-world - cfy executions start install -d k8s-hello-world - pod_ip=$(kubectl get pods --namespace default -o jsonpath='{.status.podIP}' nginx) - while [[ "x$pod_ip" == "x" ]]; do - log "nginx pod IP is not yet assigned, waiting 10 seconds" + manager_ip=$2 + cd ~/models/tools/cloudify/blueprints + + if [[ "$1" == "start" ]]; then + log "copy kube config from k8s master for insertion into blueprint" + scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + ubuntu@$manager_ip:/home/ubuntu/.kube/config k8s-hello-world/kube.config + + log "package the blueprint" + # CLI: cfy blueprints package -o ~/cloudify/blueprints/k8s-hello-world ~/cloudify/blueprints/k8s-hello-world + tar ckf /tmp/blueprint.tar k8s-hello-world + + log "upload the blueprint" + # CLI: cfy blueprints upload -t default_tenant -b k8s-hello-world ~/cloudify/blueprints/k8s-hello-world.tar.gz + curl -X PUT -u admin:admin --header 'Tenant: default_tenant' \ + --header "Content-Type: application/octet-stream" \ + http://$manager_ip/api/v3.1/blueprints/k8s-hello-world?application_file_name=blueprint.yaml \ + -T /tmp/blueprint.tar | jq + + log "create a deployment for the blueprint" + # CLI: cfy deployments create -t default_tenant -b k8s-hello-world k8s-hello-world + curl -X PUT -u admin:admin --header 'Tenant: default_tenant' \ + --header "Content-Type: application/json" \ + -d '{"blueprint_id": "k8s-hello-world", "inputs": {}}' \ + http://$manager_ip/api/v3.1/deployments/k8s-hello-world sleep 10 - pod_ip=$(kubectl get pods --namespace default -o jsonpath='{.status.podIP}' nginx) - done - while ! curl http://$pod_ip ; do - log "nginx pod is not yet responding at http://$pod_ip, waiting 10 seconds" + + # CLI: cfy workflows list -d k8s-hello-world + + log "install the deployment pod and service" + # CLI: cfy executions start install -d k8s-hello-world + curl -X POST -u admin:admin --header 'Tenant: default_tenant' \ + --header "Content-Type: application/json" \ + -d '{"deployment_id":"k8s-hello-world", "workflow_id":"install"}' \ + http://$manager_ip/api/v3.1/executions | jq + + log "get the service's assigned node_port" + port=$(curl -u admin:admin --header 'Tenant: default_tenant' \ + http://$manager_ip/api/v3.1/node-instances | \ + jq -r '.items[0].runtime_properties.kubernetes.spec.ports[0].node_port') + while [[ "$port" == "null" ]]; do + sleep 10 + port=$(curl -u admin:admin --header 'Tenant: default_tenant' \ + http://$manager_ip/api/v3.1/node-instances | \ + jq -r '.items[0].runtime_properties.kubernetes.spec.ports[0].node_port') + done + log "node_port = $port" + + log "verify service is responding" + while ! curl http://$manager_ip:$port ; do + log "nginx service is not yet responding at http://$manager_ip:$port, waiting 10 seconds" + sleep 10 + done + log "service is active at http://$manager_ip:$port" + else + log "uninstall the service" + curl -X POST -u admin:admin --header 'Tenant: default_tenant' \ + --header "Content-Type: application/json" \ + -d '{"deployment_id":"k8s-hello-world", "workflow_id":"uninstall", "force": "true"}' \ + http://$manager_ip/api/v3.1/executions + count=1 + state=$(curl -u admin:admin --header 'Tenant: default_tenant' \ + http://$manager_ip/api/v3.1/node-instances | jq -r '.items[0].state') + while [[ "$state" == "deleting" ]]; do + if [[ $count > 10 ]]; then + log "try to cancel all current executions" + exs=$(curl -u admin:admin --header 'Tenant: default_tenant' \ + http://$manager_ip/api/v3.1/executions | jq -r '.items[].status') + i=0 + for status in $exs; do + log "checking execution $i in state $status" + if [[ "$status" == "started" ]]; then + id=$(curl -u admin:admin --header 'Tenant: default_tenant' \ + http://$manager_ip/api/v3.1/executions | jq -r ".items[$i].id") + curl -X POST -u admin:admin --header 'Tenant: default_tenant' \ + --header "Content-Type: application/json" \ + -d '{"deployment_id": "k8s-hello-world", "action": "cancel"}' \ + http://$manager_ip/api/v3.1/executions/$id + fi + ((i++)) + done + log "force delete deployment via cfy CLI" + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + ubuntu@$manager_ip cfy deployment delete -f \ + -t default_tenant k8s-hello-world + fi + ((count ++)) + state=$(curl -u admin:admin --header 'Tenant: default_tenant' \ + http://$manager_ip/api/v3.1/node-instances | jq -r '.items[0].state') + done + + log "delete the deployment" + curl -X DELETE -u admin:admin --header 'Tenant: default_tenant' \ + http://$manager_ip/api/v3.1/deployments/k8s-hello-world sleep 10 - done - log "nginx pod is active at http://$pod_ip" - curl http://$pod_ip + log "delete the blueprint" + curl -X DELETE -u admin:admin --header 'Tenant: default_tenant' \ + http://$manager_ip/api/v3.1/blueprints/k8s-hello-world + sleep 10 + log "verify the blueprint is deleted" + curl -u admin:admin --header 'Tenant: default_tenant' \ + http://$manager_ip/api/v3.1/blueprints | jq + fi + +# API examples: use '| jq' to format JSON output +# curl -u admin:admin --header 'Tenant: default_tenant' http://$manager_ip/api/v3.1/blueprints | jq +# curl -u admin:admin --header 'Tenant: default_tenant' http://$manager_ip/api/v3.1/deployments | jq +# curl -u admin:admin --header 'Tenant: default_tenant' http://$manager_ip/api/v3.1/executions | jq +# curl -u admin:admin --header 'Tenant: default_tenant' http://$manager_ip/api/v3.1/deployments | jq -r '.items[0].blueprint_id' +# curl -u admin:admin --header 'Tenant: default_tenant' http://$manager_ip/api/v3.1/node-instances | jq } function clean () { @@ -145,7 +275,7 @@ case "$1" in setup ;; "demo") - demo + demo $2 $3 ;; "clean") clean diff --git a/tools/kubernetes/demo_deploy.sh b/tools/kubernetes/demo_deploy.sh index b91f828..b3ba479 100644 --- a/tools/kubernetes/demo_deploy.sh +++ b/tools/kubernetes/demo_deploy.sh @@ -85,8 +85,7 @@ ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$maste bash cloudify/k8s-cloudify.sh prereqs ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$master \ bash cloudify/k8s-cloudify.sh setup -ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$master \ - bash cloudify/k8s-cloudify.sh demo +source ~/models/tools/cloudify/k8s-cloudify.sh demo start $master echo "$0 $(date): All done!" export NODE_PORT=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$master kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services dw-dokuwiki) @@ -98,3 +97,4 @@ echo "Prometheus UI is available at http://$master:9090" echo "Grafana dashboards are available at http://$master:3000 (login as admin/admin)" echo "Grafana API is available at http://admin:admin@$master:3000/api/v1/query?query=" echo "Kubernetes API is available at https://$master:6443/api/v1/" +echo "Cloudify API access example: curl -u admin:admin --header 'Tenant: default_tenant' http://$master/api/v3.1/status" -- cgit 1.2.3-korg