diff options
author | earrage <eddie.arrage@huawei.com> | 2018-11-01 21:04:15 -0700 |
---|---|---|
committer | Eddie Arrage <eddie.arrage@huawei.com> | 2018-11-07 06:10:07 +0000 |
commit | bc300476e0e995df06a6e19a0f436a5ee2cb8f0b (patch) | |
tree | 24b9a2a6d3d6f22df7f6618fbb56286c63bcecbc | |
parent | d9bbeea6a6a0a37f60580a28c093aed96c033c26 (diff) |
Add create/delete of clover-system services from CLI
- Improve usability when deploying or deleting various
Clover components using cloverctl
- Add yaml for each of the clover-system services
including controller, collector, spark, redis, cassandra,
clovisor and jmeter separated by k8s resource and available
under a single yaml directory
- Deploy/delete yaml for services in various subsets including:
- visibility(controller, collector, spark, redis, cassandra)
- datastore(redis, cassandra)
- validation(jmeter master/slaves(4))
- clovisor (individually)
- collector (individually)
- controller (individually)
Ex. cloverctl create system controller
Ex. cloverctl create system visibility
- Ability to add nodeport for controller separately
(cloverctl create system controller nodeport)
- Use native client-go methods to create/delete all k8s resources
used by Clover including:
- pods, deployments, statefulsets, daemonsets, services
- serviceaccounts, clusterrolebindings
- Above allows yaml override of values such as image value specified
in deployment or daemonset. This is broken down by:
- tag (ex. latest, opnfv-7.0.0) or repo'(ex. opnfv, localhost:5000)
that can be added with CLI flags as shown below:
(cloverctl create system visibility -t latest -r localhost:5000)
- defaults to -r opnfv -t latest
- Creates/deletes clover-system namespace
- Also added ability to create/delete lb service for controller
external access when running on environment such as GKE
(cloverctl <create/delete> system controller lb)
Change-Id: I2a3c6c80035d4663fa38368b3ff13e9a14090a47
Signed-off-by: earrage <eddie.arrage@huawei.com>
26 files changed, 936 insertions, 244 deletions
diff --git a/clover/cloverctl/src/cloverctl/cmd/create_system.go b/clover/cloverctl/src/cloverctl/cmd/create_system.go index 68fa5af..402d0c0 100644 --- a/clover/cloverctl/src/cloverctl/cmd/create_system.go +++ b/clover/cloverctl/src/cloverctl/cmd/create_system.go @@ -9,29 +9,189 @@ package cmd import ( "fmt" - //"io/ioutil" - //"github.com/ghodss/yaml" "github.com/spf13/cobra" "cloverkube" ) - var systemCmd = &cobra.Command{ Use: "system", - Short: "Deploy clover-system in Kubernetes", + Short: "Deploy clover-system services in Kubernetes", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + cloverkube.ConfigNamespace("clover-system", "create") + }, +} + +var create_visibilityCmd = &cobra.Command{ + Use: "visibility", + Short: "Deploy visibility services in clover-system namespace", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Creating visibility services") + cloverkube.ConfigNamespace("clover-system", "create") + createCloverSystem("cassandra") + createCloverSystem("redis") + createCloverSystem("collector") + createCloverSystem("controller") + createCloverSystem("spark") + }, +} + +var create_collectorCmd = &cobra.Command{ + Use: "collector", + Short: "Deploy collector service in clover-system namespace", Long: ``, Run: func(cmd *cobra.Command, args []string) { - createCloverSystem() + fmt.Println("Creating collector service") + cloverkube.ConfigNamespace("clover-system", "create") + createCloverSystem("collector") }, } +var create_controllerCmd = &cobra.Command{ + Use: "controller", + Short: "Deploy controller service in clover-system namespace", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Creating controller service") + cloverkube.ConfigNamespace("clover-system", "create") + createCloverSystem("controller") + }, +} + +var create_controllernodeportCmd = &cobra.Command{ + Use: "nodeport", + Short: "Deploy nodeport service to expose controller externally", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Creating nodeport for controller service") + createCloverSystem("controller_nodeport") + }, +} + +var create_controllerlbCmd = &cobra.Command{ + Use: "lb", + Short: "Deploy lb service to expose controller externally", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Creating lb for controller service") + createCloverSystem("controller_lb") + }, +} + +var create_clovisorCmd = &cobra.Command{ + Use: "clovisor", + Short: "Deploy clovisor service in clovisor namespace", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Creating clovisor service") + cloverkube.ConfigNamespace("clovisor", "create") + createCloverSystem("clovisor") + }, +} + +var create_datastoreCmd = &cobra.Command{ + Use: "datastore", + Short: "Deploy redis/cassandra services in clover-system namespace", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Creating datastore services") + cloverkube.ConfigNamespace("clover-system", "create") + createCloverSystem("cassandra") + createCloverSystem("redis") + }, +} + +var create_validationCmd = &cobra.Command{ + Use: "validation", + Short: "Deploy jmeter master/slave services", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Creating validation services") + createCloverSystem("jmeter_master") + createCloverSystem("jmeter_slave") + }, +} + + +var repo string +var tag string func init() { createCmd.AddCommand(systemCmd) - //systemCmd.PersistentFlags().StringVarP(&cloverFile, "f", "f", "", "Input yaml file to create system") - + systemCmd.AddCommand(create_visibilityCmd) + systemCmd.AddCommand(create_collectorCmd) + systemCmd.AddCommand(create_controllerCmd) + create_controllerCmd.AddCommand(create_controllernodeportCmd) + create_controllerCmd.AddCommand(create_controllerlbCmd) + systemCmd.AddCommand(create_datastoreCmd) + systemCmd.AddCommand(create_validationCmd) + systemCmd.AddCommand(create_clovisorCmd) + systemCmd.PersistentFlags().StringVarP(&repo, "repo", "r", "opnfv", + "Image repo to use, ex. 'opnfv' or 'localhost:5000'") + systemCmd.PersistentFlags().StringVarP(&tag, "tag", "t", "opnfv-7.0.0", + "Image tag to use, ex. 'opnfv-7.0.0' or 'latest'") } -func createCloverSystem() { - cloverkube.DeployCloverSystem("create", "clover-system") - fmt.Println("Deployed clover-system successfully") +func createCloverSystem(clover_services string) { + image := "" + switch clover_services { + case "controller": + image = repo + "/clover-controller:" + tag + cloverkube.CreateResource("controller/deployment.yaml", + "deployment", image, "") + cloverkube.CreateResource("controller/service_internal.yaml", + "service", "", "") + case "controller_nodeport": + cloverkube.CreateResource("controller/service_nodeport.yaml", + "service", "", "") + case "controller_lb": + cloverkube.CreateResource("controller/service_lb.yaml", + "service", "", "") + case "collector": + image = repo + "/clover-collector:" + tag + cloverkube.CreateResource("collector/deployment.yaml", + "deployment", image, "") + cloverkube.CreateResource("collector/service.yaml", "service", + "", "") + case "spark": + image = repo + "/clover-spark-submit:" + tag + cloverkube.CreateResource("spark/serviceaccount.yaml", + "serviceaccount", "", "") + cloverkube.CreateResource("spark/clusterrolebinding.yaml", + "clusterrolebinding", "", "") + cloverkube.CreateResource("spark/clusterrolebinding_spark.yaml", + "clusterrolebinding", "", "") + cloverkube.CreateResource("spark/deployment.yaml", "deployment", + image, "") + case "clovisor": + image = repo + "/clover-clovisor:" + tag + cloverkube.CreateResource("clovisor/serviceaccount.yaml", + "serviceaccount", "", "clovisor") + cloverkube.CreateResource("clovisor/clusterrolebinding.yaml", + "clusterrolebinding", "", "clovisor") + cloverkube.CreateResource("clovisor/daemonset.yaml", "daemonset", + image, "clovisor") + case "redis": + cloverkube.CreateResource("datastore/redis_pod.yaml", "pod", + "", "") + cloverkube.CreateResource("datastore/redis_service.yaml", + "service", "", "") + case "cassandra": + cloverkube.CreateResource("datastore/cassandra_statefulset.yaml", + "statefulset", "", "") + cloverkube.CreateResource("datastore/cassandra_service.yaml", + "service", "", "") + case "jmeter_master": + image = repo + "/clover-jmeter-master:" + tag + cloverkube.CreateResource("jmeter/master_deployment.yaml", + "deployment", image, "default") + cloverkube.CreateResource("jmeter/master_service.yaml", + "service", "", "default") + case "jmeter_slave": + image = repo + "/clover-jmeter-slave:" + tag + cloverkube.CreateResource("jmeter/slave_deployment.yaml", + "deployment", image, "default") + cloverkube.CreateResource("jmeter/slave_service.yaml", + "service", "", "default") + } } diff --git a/clover/cloverctl/src/cloverctl/cmd/delete_system.go b/clover/cloverctl/src/cloverctl/cmd/delete_system.go index bc3f22b..ded0a68 100644 --- a/clover/cloverctl/src/cloverctl/cmd/delete_system.go +++ b/clover/cloverctl/src/cloverctl/cmd/delete_system.go @@ -13,21 +13,154 @@ import ( "cloverkube" ) - var delsystemCmd = &cobra.Command{ Use: "system", Short: "Delete clover-system in Kubernetes", Long: ``, Run: func(cmd *cobra.Command, args []string) { - delCloverSystem() + delCloverSystem("controller") + }, +} + +var del_controllerCmd = &cobra.Command{ + Use: "controller", + Short: "Delete controller service from clover-system namespace", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Deleting controller service") + delCloverSystem("controller") + }, +} + +var del_controllernodeportCmd = &cobra.Command{ + Use: "nodeport", + Short: "Delete controller nodeport service from clover-system namespace", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Deleting nodeport for controller service") + delCloverSystem("controller_nodeport") + }, +} + +var del_controllerlbCmd = &cobra.Command{ + Use: "lb", + Short: "Delete controller lb service from clover-system namespace", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Deleting lb for controller service") + delCloverSystem("controller_lb") + }, +} + +var del_collectorCmd = &cobra.Command{ + Use: "collector", + Short: "Delete collector service from clover-system namespace", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Deleting collector service") + delCloverSystem("collector") + }, +} + +var del_clovisorCmd = &cobra.Command{ + Use: "clovisor", + Short: "Delete clovisor service from clovisor namespace", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Deleting clovisor service") + delCloverSystem("clovisor") + cloverkube.ConfigNamespace("clovisor", "delete") + }, +} + +var del_visibilityCmd = &cobra.Command{ + Use: "visibility", + Short: "Delete visibility services from clover-system namespace", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Deleting visibility services") + delCloverSystem("spark") + delCloverSystem("controller") + delCloverSystem("collector") + delCloverSystem("cassandra") + delCloverSystem("redis") + cloverkube.ConfigNamespace("clover-system", "delete") + }, +} + +var del_datastoreCmd = &cobra.Command{ + Use: "datastore", + Short: "Delete datastore services from clover-system namespace", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Deleting datatore services") + delCloverSystem("cassandra") + delCloverSystem("redis") + }, +} + +var del_validationCmd = &cobra.Command{ + Use: "validation", + Short: "Delete jmeter master/slave services", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Deleting validation services") + delCloverSystem("jmeter_master") + delCloverSystem("jmeter_slave") }, } func init() { deleteCmd.AddCommand(delsystemCmd) + delsystemCmd.AddCommand(del_controllerCmd) + del_controllerCmd.AddCommand(del_controllernodeportCmd) + del_controllerCmd.AddCommand(del_controllerlbCmd) + delsystemCmd.AddCommand(del_collectorCmd) + delsystemCmd.AddCommand(del_visibilityCmd) + delsystemCmd.AddCommand(del_validationCmd) + delsystemCmd.AddCommand(del_datastoreCmd) + delsystemCmd.AddCommand(del_clovisorCmd) } -func delCloverSystem() { - cloverkube.DeployCloverSystem("delete", "clover-system") - fmt.Println("Deleted clover-system successfully") +func delCloverSystem(clover_services string) { + ns := "clover-system" + switch clover_services { + case "controller": + cloverkube.DeleteResource("clover-controller", "deployment", ns) + cloverkube.DeleteResource("clover-controller-internal", + "service", ns) + case "controller_nodeport", "controller_lb": + cloverkube.DeleteResource("clover-controller", + "service", ns) + case "collector": + cloverkube.DeleteResource("clover-collector", "deployment", ns) + cloverkube.DeleteResource("clover-collector", "service", ns) + case "spark": + cloverkube.DeleteResource("clover-spark", "serviceaccount", ns) + cloverkube.DeleteResource("clover-spark-default", + "clusterrolebinding", ns) + cloverkube.DeleteResource("clover-spark", "clusterrolebinding", ns) + cloverkube.DeleteResource("clover-spark-submit", "deployment", ns) + case "clovisor": + cloverkube.DeleteResource("clovisor", "serviceaccount", "clovisor") + cloverkube.DeleteResource("serv-account-rbac-clovisor", + "clusterrolebinding", "clovisor") + cloverkube.DeleteResource("clovisor", "daemonset", "clovisor") + case "redis": + cloverkube.DeleteResource("redis", "pod", ns) + cloverkube.DeleteResource("redis", "service", ns) + case "cassandra": + cloverkube.DeleteResource("cassandra", "statefulset", ns) + cloverkube.DeleteResource("cassandra", "service", ns) + case "jmeter_master": + cloverkube.DeleteResource("clover-jmeter-master", "deployment", + "default") + cloverkube.DeleteResource("clover-jmeter-master", + "service", "default") + case "jmeter_slave": + cloverkube.DeleteResource("clover-jmeter-slave", "deployment", + "default") + cloverkube.DeleteResource("clover-jmeter-slave", + "service", "default") + } } diff --git a/clover/cloverctl/src/cloverctl/cmd/set.go b/clover/cloverctl/src/cloverctl/cmd/set.go new file mode 100644 index 0000000..b20bfb5 --- /dev/null +++ b/clover/cloverctl/src/cloverctl/cmd/set.go @@ -0,0 +1,26 @@ +// Copyright (c) Authors of Clover +// +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Apache License, Version 2.0 +// which accompanies this distribution, and is available at +// http://www.apache.org/licenses/LICENSE-2.0 + +package cmd + +import ( + "fmt" + "github.com/spf13/cobra" +) + +var setCmd = &cobra.Command{ + Use: "set", + Short: "Set input to visibility or configurations for sample services", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Incomplete command") + }, +} + +func init() { + rootCmd.AddCommand(setCmd) +} diff --git a/clover/cloverctl/src/cloverctl/yaml/clovisor/clusterrolebinding.yaml b/clover/cloverctl/src/cloverctl/yaml/clovisor/clusterrolebinding.yaml new file mode 100644 index 0000000..b8a364a --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/clovisor/clusterrolebinding.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: serv-account-rbac-clovisor +subjects: + - kind: ServiceAccount + # Reference to upper's `metadata.name` + name: default + # Reference to upper's `metadata.namespace` + namespace: clovisor +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io diff --git a/clover/cloverctl/src/cloverctl/yaml/clovisor/daemonset.yaml b/clover/cloverctl/src/cloverctl/yaml/clovisor/daemonset.yaml new file mode 100644 index 0000000..4c845f6 --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/clovisor/daemonset.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: clovisor + namespace: clovisor +spec: + selector: + matchLabels: + app: clovisor + template: + metadata: + name: clovisor + labels: + app: clovisor + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: clovisor + image: opnfv/clovisor + securityContext: + privileged: true + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName diff --git a/clover/cloverctl/src/cloverctl/yaml/clovisor/serviceaccount.yaml b/clover/cloverctl/src/cloverctl/yaml/clovisor/serviceaccount.yaml new file mode 100644 index 0000000..1fbbebb --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/clovisor/serviceaccount.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: clovisor + namespace: clovisor diff --git a/clover/cloverctl/src/cloverctl/yaml/collector/deployment.yaml b/clover/cloverctl/src/cloverctl/yaml/collector/deployment.yaml new file mode 100644 index 0000000..490fbbd --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/collector/deployment.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: clover-collector + labels: + app: clover-collector + namespace: clover-system +spec: + template: + metadata: + labels: + app: clover-collector + spec: + containers: + - name: clover-collector + image: opnfv/clover-collector:latest + ports: + - containerPort: 50054 + - containerPort: 6379 + - containerPort: 9090 + - containerPort: 16686 + - containerPort: 9042 diff --git a/clover/cloverctl/src/cloverctl/yaml/collector/service.yaml b/clover/cloverctl/src/cloverctl/yaml/collector/service.yaml new file mode 100644 index 0000000..b84601b --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/collector/service.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: clover-collector + labels: + app: clover-collector + namespace: clover-system +spec: + ports: + - port: 50054 + name: grpc + - port: 6379 + name: redis + - port: 16686 + name: jaeger + - port: 9090 + name: prometheus + - port: 9042 + name: cassandra + selector: + app: clover-collector diff --git a/clover/cloverctl/src/cloverctl/yaml/controller/deployment.yaml b/clover/cloverctl/src/cloverctl/yaml/controller/deployment.yaml new file mode 100644 index 0000000..4f15b51 --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/controller/deployment.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: clover-controller + namespace: clover-system + labels: + app: "clover-controller" +spec: + template: + metadata: + labels: + app: "clover-controller" + spec: + containers: + - name: clover-controller + image: opnfv/clover-controller:latest + ports: + - containerPort: 50052 + - containerPort: 50054 + - containerPort: 6379 + - containerPort: 9042 + - containerPort: 80 diff --git a/clover/cloverctl/src/cloverctl/yaml/controller/service_internal.yaml b/clover/cloverctl/src/cloverctl/yaml/controller/service_internal.yaml new file mode 100644 index 0000000..8be95ee --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/controller/service_internal.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: clover-controller-internal + namespace: clover-system + labels: + app: clover-controller +spec: + ports: + - name: http + port: 80 + - name: grpc + port: 50052 + - name: grpc-server + port: 50054 + - name: redis + port: 6379 + - name: cassandra + port: 9042 + selector: + app: clover-controller diff --git a/clover/cloverctl/src/cloverctl/yaml/controller/service_lb.yaml b/clover/cloverctl/src/cloverctl/yaml/controller/service_lb.yaml new file mode 100644 index 0000000..2dcf5d1 --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/controller/service_lb.yaml @@ -0,0 +1,14 @@ +kind: Service +apiVersion: v1 +metadata: + name: clover-controller + namespace: clover-system +spec: + selector: + app: clover-controller + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 80 + type: LoadBalancer diff --git a/clover/cloverctl/src/cloverctl/yaml/controller/service_nodeport.yaml b/clover/cloverctl/src/cloverctl/yaml/controller/service_nodeport.yaml new file mode 100644 index 0000000..a719204 --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/controller/service_nodeport.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: clover-controller + labels: + app: clover-controller +spec: + type: NodePort + ports: + - name: http + port: 80 + targetPort: 80 + nodePort: 32044 + protocol: TCP + selector: + app: clover-controller diff --git a/clover/cloverctl/src/cloverctl/yaml/datastore/cassandra_service.yaml b/clover/cloverctl/src/cloverctl/yaml/datastore/cassandra_service.yaml new file mode 100644 index 0000000..12f7d14 --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/datastore/cassandra_service.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: cassandra + name: cassandra + namespace: clover-system +spec: + clusterIP: None + ports: + - port: 9042 + selector: + app: cassandra diff --git a/clover/cloverctl/src/cloverctl/yaml/datastore/cassandra_statefulset.yaml b/clover/cloverctl/src/cloverctl/yaml/datastore/cassandra_statefulset.yaml new file mode 100644 index 0000000..21ceb5f --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/datastore/cassandra_statefulset.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: cassandra + labels: + app: cassandra + namespace: clover-system +spec: + serviceName: cassandra + replicas: 1 + selector: + matchLabels: + app: cassandra + template: + metadata: + labels: + app: cassandra + spec: + terminationGracePeriodSeconds: 1800 + containers: + - name: cassandra + image: cassandra:3 + imagePullPolicy: Always + ports: + - containerPort: 7000 + name: intra-node + - containerPort: 7001 + name: tls-intra-node + - containerPort: 7199 + name: jmx + - containerPort: 9042 + name: cql + resources: + limits: + cpu: "4000m" + memory: 5Gi + requests: + cpu: "4000m" + memory: 5Gi + env: + - name: MAX_HEAP_SIZE + value: 512M + - name: HEAP_NEWSIZE + value: 100M + - name: CASSANDRA_SEEDS + value: "cassandra-0.cassandra.clover-system.svc.cluster.local" + - name: CASSANDRA_CLUSTER_NAME + value: "MyCassandraDemo" + - name: CASSANDRA_DC + value: "DC1-K8Demo" + - name: CASSANDRA_RACK + value: "Rack1-K8Demo" + diff --git a/clover/cloverctl/src/cloverctl/yaml/datastore/redis_pod.yaml b/clover/cloverctl/src/cloverctl/yaml/datastore/redis_pod.yaml new file mode 100644 index 0000000..976fd97 --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/datastore/redis_pod.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + name: redis + redis-sentinel: "true" + role: master + name: redis +spec: + containers: + - name: redis + image: k8s.gcr.io/redis:v1 + env: + - name: MASTER + value: "true" + ports: + - containerPort: 6379 + resources: + limits: + cpu: "0.1" + volumeMounts: + - mountPath: /redis-master-data + name: data + - name: sentinel + image: kubernetes/redis:v1 + env: + - name: SENTINEL + value: "true" + ports: + - containerPort: 26379 + volumes: + - name: data + emptyDir: {} diff --git a/clover/cloverctl/src/cloverctl/yaml/datastore/redis_service.yaml b/clover/cloverctl/src/cloverctl/yaml/datastore/redis_service.yaml new file mode 100644 index 0000000..7ecab14 --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/datastore/redis_service.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: redis +spec: + ports: + - port: 6379 + selector: + name: redis diff --git a/clover/cloverctl/src/cloverctl/yaml/jmeter/master_deployment.yaml b/clover/cloverctl/src/cloverctl/yaml/jmeter/master_deployment.yaml new file mode 100644 index 0000000..a2d0cbb --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/jmeter/master_deployment.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: clover-jmeter-master + labels: + app: clover-jmeter-master +spec: + template: + metadata: + labels: + app: clover-jmeter-master + spec: + containers: + - name: clover-jmeter-master + image: opnfv/clover-jmeter-master:latest + ports: + - containerPort: 50054 + - containerPort: 1099 + - containerPort: 80 + - containerPort: 443 diff --git a/clover/cloverctl/src/cloverctl/yaml/jmeter/master_service.yaml b/clover/cloverctl/src/cloverctl/yaml/jmeter/master_service.yaml new file mode 100644 index 0000000..da1e5ce --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/jmeter/master_service.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: clover-jmeter-master + labels: + app: clover-jmeter-master +spec: + ports: + - port: 50054 + name: grpc + - port: 1099 + name: rmi + - port: 80 + name: http + - port: 443 + name: https + selector: + app: clover-jmeter-master diff --git a/clover/cloverctl/src/cloverctl/yaml/jmeter/slave_deployment.yaml b/clover/cloverctl/src/cloverctl/yaml/jmeter/slave_deployment.yaml new file mode 100644 index 0000000..424828a --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/jmeter/slave_deployment.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: clover-jmeter-slave + labels: + app: clover-jmeter-slave +spec: + replicas: 3 + template: + metadata: + labels: + app: clover-jmeter-slave + spec: + containers: + - name: clover-jmeter-slave + image: opnfv/clover-jmeter-slave:latest + ports: + - containerPort: 1099 + - containerPort: 80 + - containerPort: 443 diff --git a/clover/cloverctl/src/cloverctl/yaml/jmeter/slave_service.yaml b/clover/cloverctl/src/cloverctl/yaml/jmeter/slave_service.yaml new file mode 100644 index 0000000..858191e --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/jmeter/slave_service.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: clover-jmeter-slave + labels: + app: clover-jmeter-slave +spec: + ports: + - port: 1099 + name: rmi + - port: 80 + name: http + - port: 443 + name: https + selector: + app: clover-jmeter-slave diff --git a/clover/cloverctl/src/cloverctl/yaml/spark/clusterrolebinding.yaml b/clover/cloverctl/src/cloverctl/yaml/spark/clusterrolebinding.yaml new file mode 100644 index 0000000..ec6b367 --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/spark/clusterrolebinding.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: clover-spark-default +subjects: + - kind: ServiceAccount + name: default + namespace: clover-system +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io diff --git a/clover/cloverctl/src/cloverctl/yaml/spark/clusterrolebinding_spark.yaml b/clover/cloverctl/src/cloverctl/yaml/spark/clusterrolebinding_spark.yaml new file mode 100644 index 0000000..be02e6f --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/spark/clusterrolebinding_spark.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: clover-spark +subjects: + - kind: ServiceAccount + name: clover-spark + namespace: clover-system +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +--- diff --git a/clover/cloverctl/src/cloverctl/yaml/spark/deployment.yaml b/clover/cloverctl/src/cloverctl/yaml/spark/deployment.yaml new file mode 100644 index 0000000..d6c2d59 --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/spark/deployment.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: clover-spark-submit + namespace: clover-system + labels: + app: clover-spark +spec: + template: + metadata: + labels: + app: clover-spark + spec: + containers: + - name: clover-spark + image: opnfv/clover-spark-submit:latest diff --git a/clover/cloverctl/src/cloverctl/yaml/spark/serviceaccount.yaml b/clover/cloverctl/src/cloverctl/yaml/spark/serviceaccount.yaml new file mode 100644 index 0000000..14bc316 --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/spark/serviceaccount.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: clover-spark + namespace: clover-system diff --git a/clover/cloverctl/src/cloverkube/main.go b/clover/cloverctl/src/cloverkube/main.go index 7710a13..bcda990 100644 --- a/clover/cloverctl/src/cloverkube/main.go +++ b/clover/cloverctl/src/cloverkube/main.go @@ -15,20 +15,22 @@ import ( "io/ioutil" "io" "bytes" + "github.com/ghodss/yaml" + "encoding/json" appsv1 "k8s.io/api/apps/v1" + v1beta1 "k8s.io/api/apps/v1beta1" + v1beta1ext "k8s.io/api/extensions/v1beta1" apiv1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/remotecommand" - ) func setClient() kubernetes.Interface { - - kubeconfig := filepath.Join( os.Getenv("HOME"), ".kube", "config", ) @@ -45,257 +47,222 @@ func setClient() kubernetes.Interface { return clientset } -func setControllerDeploy () (*appsv1.Deployment, *apiv1.Service) { - - deployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "clover-controller", - }, - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "clover-controller", - }, - }, - Template: apiv1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": "clover-controller", - }, - }, - Spec: apiv1.PodSpec{ - Containers: []apiv1.Container{ - { - Name: "clover-controller", - Image: "localhost:5000/clover-controller:latest", - Ports: []apiv1.ContainerPort{ - { - Name: "redis", - Protocol: apiv1.ProtocolTCP, - ContainerPort: 6379, - }, - { - Name: "grpc", - Protocol: apiv1.ProtocolTCP, - ContainerPort: 50054, - }, - { - Name: "gprcsecurity", - Protocol: apiv1.ProtocolTCP, - ContainerPort: 50052, - }, - { - Name: "cassandra", - Protocol: apiv1.ProtocolTCP, - ContainerPort: 9042, - }, - - - }, - }, - }, - }, - }, - }, +// Create various K8s resources +func CreateResource(input_yaml string, resource_type string, + image string, namespace string) { + if namespace == "" { + namespace = "clover-system" + } + // Check path from cloverctl first + exe_path, err := os.Executable() + abs_input_yaml := strings.Replace(exe_path, "cloverctl", "", -1) + + "/yaml/" + input_yaml + if _, err := os.Stat(abs_input_yaml); !os.IsNotExist(err) { + input_yaml = abs_input_yaml + } + in, err := ioutil.ReadFile(input_yaml) + if err != nil { + fmt.Println("Please specify a valid manifest yaml file") + return + } + out_json, err := yaml.YAMLToJSON(in) + if err != nil { + panic(err.Error()) } - service := &apiv1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "clover-controller", - Labels: map[string]string{ - "app": "clover-controller", - }, - - }, - Spec: apiv1.ServiceSpec{ - Selector: map[string]string{ - "app": "clover-controller", - }, - Type: "NodePort", - Ports: []apiv1.ServicePort{ - { - Name: "http", - Port: 80, - NodePort: 32044, - Protocol: "TCP", - }, - }, - }, + switch resource_type { + case "deployment": + deploy := v1beta1.Deployment{} + err = json.Unmarshal(out_json, &deploy) + deploy.Spec.Template.Spec.Containers[0].Image = image + CreateDeployment(&deploy, namespace) + fmt.Printf("Image: %s\n", + deploy.Spec.Template.Spec.Containers[0].Image) + case "service": + service := apiv1.Service{} + err = json.Unmarshal(out_json, &service) + CreateService(&service, namespace) + case "serviceaccount": + sa := apiv1.ServiceAccount{} + err = json.Unmarshal(out_json, &sa) + CreateServiceAccount(&sa, namespace) + case "clusterrolebinding": + clusterrolebinding := rbacv1.ClusterRoleBinding{} + err = json.Unmarshal(out_json, &clusterrolebinding) + CreateCRB(&clusterrolebinding) + case "statefulset": + statefulset := appsv1.StatefulSet{} + err = json.Unmarshal(out_json, &statefulset) + CreateStatefulSet(&statefulset, namespace) + case "pod": + pod := apiv1.Pod{} + err = json.Unmarshal(out_json, &pod) + CreatePod(&pod, namespace) + case "daemonset": + daemon := v1beta1ext.DaemonSet{} + err = json.Unmarshal(out_json, &daemon) + daemon.Spec.Template.Spec.Containers[0].Image = image + CreateDaemonSet(&daemon, namespace) + fmt.Printf("Image: %s\n", + daemon.Spec.Template.Spec.Containers[0].Image) + + default: + fmt.Println("No resource selected") } +} - return deployment, service +// Delete K8s resources +func DeleteResource(deploy_name string, resource_type string, + namespace string) { + clientset := setClient() + deletePolicy := metav1.DeletePropagationForeground + switch resource_type { + case "deployment": + deploymentsClient := clientset.AppsV1().Deployments(namespace) + if err := deploymentsClient.Delete(deploy_name, + &metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + fmt.Printf("Error deleting %v: %v\n", resource_type, err) + return + } + case "service": + servicesClient := clientset.CoreV1().Services(namespace) + if err := servicesClient.Delete(deploy_name, &metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + fmt.Printf("Error deleting %v: %v\n", resource_type, err) + return + } + case "serviceaccount": + saClient := clientset.CoreV1().ServiceAccounts(namespace) + if err := saClient.Delete(deploy_name, &metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + fmt.Printf("Error deleting %v: %v\n", resource_type, err) + return + } + case "clusterrolebinding": + crbClient := clientset.RbacV1().ClusterRoleBindings() + if err := crbClient.Delete(deploy_name, &metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + fmt.Printf("Error deleting %v: %v\n", resource_type, err) + return + } + case "statefulset": + statefulClient := clientset.AppsV1().StatefulSets(namespace) + if err := statefulClient.Delete(deploy_name, &metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + fmt.Printf("Error deleting %v: %v\n", resource_type, err) + return + } + case "pod": + podClient := clientset.CoreV1().Pods(namespace) + if err := podClient.Delete(deploy_name, &metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + fmt.Printf("Error deleting %v: %v\n", resource_type, err) + return + } + case "daemonset": + daemonsClient := clientset.AppsV1().DaemonSets(namespace) + if err := daemonsClient.Delete(deploy_name, + &metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + fmt.Printf("Error deleting %v: %v\n", resource_type, err) + return + } + } + fmt.Printf("Deleted %s %s\n", deploy_name, resource_type) } -func setCollectorDeploy () (*appsv1.Deployment, *apiv1.Service) { - - deployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "clover-collector", - }, - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "clover-collector", - }, - }, - Template: apiv1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": "clover-collector", - }, - }, - Spec: apiv1.PodSpec{ - Containers: []apiv1.Container{ - { - Name: "clover-collector", - Image: "localhost:5000/clover-collector:latest", - Ports: []apiv1.ContainerPort{ - { - Name: "redis", - Protocol: apiv1.ProtocolTCP, - ContainerPort: 6379, - }, - { - Name: "grpc", - Protocol: apiv1.ProtocolTCP, - ContainerPort: 50054, - }, - { - Name: "prometheus", - Protocol: apiv1.ProtocolTCP, - ContainerPort: 9090, - }, - { - Name: "jaeger", - Protocol: apiv1.ProtocolTCP, - ContainerPort: 16686, - }, - { - Name: "cassandra", - Protocol: apiv1.ProtocolTCP, - ContainerPort: 9042, - }, - - - }, - }, - }, - }, - }, - }, +// Create ServiceAccount +func CreateServiceAccount(sa *apiv1.ServiceAccount, namespace string) { + clientset := setClient() + saClient := clientset.CoreV1().ServiceAccounts(namespace) + result, err := saClient.Create(sa) + if err != nil { + fmt.Printf("Error creating serviceaccount: %v\n", err); return } + fmt.Printf("Created serviceaccount %q.\n", + result.GetObjectMeta().GetName()) +} - service := &apiv1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "clover-collector", - Labels: map[string]string{ - "app": "clover-collector", - }, - - }, - Spec: apiv1.ServiceSpec{ - Selector: map[string]string{ - "app": "clover-collector", - }, - Ports: []apiv1.ServicePort{ - { - Name: "grpc", - Port: 50054, - }, - { - Name: "redis", - Port: 6379, - }, - { - Name: "prometheus", - Port: 9090, - }, - { - Name: "jaeger", - Port: 16686, - }, - { - Name: "cassandra", - Port: 9042, - }, - - }, - }, +// Create ClusterRoleBinding +func CreateCRB(sa *rbacv1.ClusterRoleBinding) { + clientset := setClient() + crbClient := clientset.RbacV1().ClusterRoleBindings() + result, err := crbClient.Create(sa) + if err != nil { + fmt.Printf("Error creating clusterrolebinding: %v\n", err); return } - return deployment, service + fmt.Printf("Created clusterrolebinding %q.\n", + result.GetObjectMeta().GetName()) } -func DeployCloverSystem(action string, namespace string) { - if action == "create" { - // Create clover-system namespace - configNamespace("clover-system", "create") - // Controller - deployment, service := setControllerDeploy() - DeployService(deployment, service, namespace) - // Collector - deployment, service = setCollectorDeploy() - DeployService(deployment, service, namespace) - } else if action == "delete" { - fmt.Println("Deleting clover-system services...\n") - DeleteService("clover-controller", namespace) - DeleteService("clover-collector", namespace) - configNamespace("clover-system", "delete") +// Create DaemonSet +func CreateDaemonSet(daemonset *v1beta1ext.DaemonSet, namespace string) { + clientset := setClient() + daemonsClient := clientset.ExtensionsV1beta1().DaemonSets(namespace) + result, err := daemonsClient.Create(daemonset) + if err != nil { + fmt.Printf("Error creating daemonset: %v\n", err); return } - + fmt.Printf("Created daemonset %q.\n", result.GetObjectMeta().GetName()) } -func DeleteService(deploy_name string, namespace string) { - +// Create Deployment +func CreateDeployment(deployment *v1beta1.Deployment, namespace string) { clientset := setClient() - deploymentsClient := clientset.AppsV1().Deployments(namespace) - servicesClient := clientset.CoreV1().Services(namespace) - - // Delete Deployment - deletePolicy := metav1.DeletePropagationForeground - if err := deploymentsClient.Delete(deploy_name, &metav1.DeleteOptions{ - PropagationPolicy: &deletePolicy, - }); err != nil { - panic(err) + deploymentsClient := clientset.AppsV1beta1().Deployments(namespace) + result, err := deploymentsClient.Create(deployment) + if err != nil { + fmt.Printf("Error creating deployment: %v\n", err); return } - fmt.Printf("Deleted %s deployment\n", deploy_name) + fmt.Printf("Created deployment %q.\n", result.GetObjectMeta().GetName()) +} - // Delete Service - if err := servicesClient.Delete(deploy_name, &metav1.DeleteOptions{ - PropagationPolicy: &deletePolicy, - }); err != nil { - panic(err) +// Create StatefulSet +func CreateStatefulSet(statefulset *appsv1.StatefulSet, namespace string) { + clientset := setClient() + statefulsetClient := clientset.AppsV1().StatefulSets(namespace) + result, err := statefulsetClient.Create(statefulset) + if err != nil { + fmt.Printf("Error creating statefulset: %v\n", err); return } - fmt.Printf("Deleted %s service\n", deploy_name) + fmt.Printf("Created statefulset %q.\n", result.GetObjectMeta().GetName()) } -func DeployService(deployment *appsv1.Deployment, service *apiv1.Service, namespace string) { - +// Create Pod +func CreatePod(pod *apiv1.Pod, namespace string) { clientset := setClient() - deploymentsClient := clientset.AppsV1().Deployments(namespace) - - - // Create Deployment - fmt.Println("Creating deployment...") - result, err := deploymentsClient.Create(deployment) + podClient := clientset.CoreV1().Pods(namespace) + result, err := podClient.Create(pod) if err != nil { + fmt.Printf("Error creating pod: %v\n", err); return panic(err) } - fmt.Printf("Created deployment %q.\n", result.GetObjectMeta().GetName()) + fmt.Printf("Created pod %q.\n", result.GetObjectMeta().GetName()) +} - // Create Service - fmt.Println("Creating service...") +// Create Service +func CreateService(service *apiv1.Service, namespace string) { + clientset := setClient() servicesClient := clientset.CoreV1().Services(namespace) result1, err := servicesClient.Create(service) if err != nil { - panic(err) + fmt.Printf("Error creating service: %v\n", err); return } - fmt.Printf("Created service %q.\n", result1.GetObjectMeta().GetName()) - + fmt.Printf("Created service %q\n", result1.GetObjectMeta().GetName()) } -func configNamespace (name string, action string) { +// Create or delete namespace +func ConfigNamespace (name string, action string) { clientset := setClient() nameClient := clientset.CoreV1().Namespaces() @@ -312,7 +279,7 @@ func configNamespace (name string, action string) { if err := nameClient.Delete(name, &metav1.DeleteOptions{ PropagationPolicy: &deletePolicy, }); err != nil { - panic(err) + fmt.Printf("Error deleting namespace: %v\n", err); return } fmt.Printf("Deleted %s namespace\n", name) } @@ -330,13 +297,12 @@ func GetServices() *apiv1.ServiceList { fmt.Printf("Kind: %s\n", service.Kind) fmt.Printf("Labels: %s\n", service.GetLabels()) fmt.Printf("Type: %s\n", service.Spec.Type) - //fmt.Printf("External IP: %v\n", service.Spec.ExternalIPs) fmt.Printf("Cluster IP: %s\n", service.Spec.ClusterIP) for _, port := range service.Spec.Ports { - fmt.Printf("Port Name: %s, Port# %d, NodePort: %d\n", port.Name, port.Port, port.NodePort) + fmt.Printf("Port Name: %s, Port# %d, NodePort: %d\n", + port.Name, port.Port, port.NodePort) } - for _, ip := range service.Status.LoadBalancer.Ingress { fmt.Printf("LB IP: %s \n", ip.IP) } @@ -345,7 +311,6 @@ func GetServices() *apiv1.ServiceList { } func GetDeployments(namespace string) []appsv1.Deployment { - clientset := setClient() deploymentsClient := clientset.AppsV1().Deployments(namespace) @@ -360,7 +325,6 @@ func GetDeployments(namespace string) []appsv1.Deployment { } func GetServicesPortIP(service_name string) (int32, string) { - clientset := setClient() services, err := clientset.Core().Services("").List(metav1.ListOptions{}) var nodeport int32 @@ -382,7 +346,6 @@ func GetServicesPortIP(service_name string) (int32, string) { } } } - return nodeport, ipaddress } diff --git a/clover/tools/jmeter/yaml/manifest.template b/clover/tools/jmeter/yaml/manifest.template index 01a4595..369df31 100644 --- a/clover/tools/jmeter/yaml/manifest.template +++ b/clover/tools/jmeter/yaml/manifest.template @@ -7,7 +7,7 @@ metadata: app: {{ deploy_name }} spec: {%- if replica_count %} - replicas: 3{% endif %} + replicas: {{ replica_count }}{% endif %} template: metadata: labels: |