From c2d93ffefcd337c8125d3efb289dd488c9cbbaae Mon Sep 17 00:00:00 2001 From: earrage Date: Thu, 1 Nov 2018 21:04:15 -0700 Subject: Add create/delete of clover-system services from CLI - Improve usability when deploying or deleting various Clover components using cloverctl - Add yaml for each of the clover-system services including controller, collector, spark, redis, cassandra, clovisor and jmeter separated by k8s resource and available under a single yaml directory - Deploy/delete yaml for services in various subsets including: - visibility(controller, collector, spark, redis, cassandra) - datastore(redis, cassandra) - validation(jmeter master/slaves(4)) - clovisor (individually) - collector (individually) - controller (individually) Ex. cloverctl create system controller Ex. cloverctl create system visibility - Ability to add nodeport for controller separately (cloverctl create system controller nodeport) - Use native client-go methods to create/delete all k8s resources used by Clover including: - pods, deployments, statefulsets, daemonsets, services - serviceaccounts, clusterrolebindings - Above allows yaml override of values such as image value specified in deployment or daemonset. This is broken down by: - tag (ex. latest, opnfv-7.0.0) or repo'(ex. opnfv, localhost:5000) that can be added with CLI flags as shown below: (cloverctl create system visibility -t latest -r localhost:5000) - defaults to -r opnfv -t latest - Creates/deletes clover-system namespace - Also added ability to create/delete lb service for controller external access when running on environment such as GKE (cloverctl system controller lb) Change-Id: I2a3c6c80035d4663fa38368b3ff13e9a14090a47 Signed-off-by: earrage --- clover/cloverctl/src/cloverkube/main.go | 419 +++++++++++++++----------------- 1 file changed, 191 insertions(+), 228 deletions(-) (limited to 'clover/cloverctl/src/cloverkube/main.go') diff --git a/clover/cloverctl/src/cloverkube/main.go b/clover/cloverctl/src/cloverkube/main.go index 7710a13..bcda990 100644 --- a/clover/cloverctl/src/cloverkube/main.go +++ b/clover/cloverctl/src/cloverkube/main.go @@ -15,20 +15,22 @@ import ( "io/ioutil" "io" "bytes" + "github.com/ghodss/yaml" + "encoding/json" appsv1 "k8s.io/api/apps/v1" + v1beta1 "k8s.io/api/apps/v1beta1" + v1beta1ext "k8s.io/api/extensions/v1beta1" apiv1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/remotecommand" - ) func setClient() kubernetes.Interface { - - kubeconfig := filepath.Join( os.Getenv("HOME"), ".kube", "config", ) @@ -45,257 +47,222 @@ func setClient() kubernetes.Interface { return clientset } -func setControllerDeploy () (*appsv1.Deployment, *apiv1.Service) { - - deployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "clover-controller", - }, - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "clover-controller", - }, - }, - Template: apiv1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": "clover-controller", - }, - }, - Spec: apiv1.PodSpec{ - Containers: []apiv1.Container{ - { - Name: "clover-controller", - Image: "localhost:5000/clover-controller:latest", - Ports: []apiv1.ContainerPort{ - { - Name: "redis", - Protocol: apiv1.ProtocolTCP, - ContainerPort: 6379, - }, - { - Name: "grpc", - Protocol: apiv1.ProtocolTCP, - ContainerPort: 50054, - }, - { - Name: "gprcsecurity", - Protocol: apiv1.ProtocolTCP, - ContainerPort: 50052, - }, - { - Name: "cassandra", - Protocol: apiv1.ProtocolTCP, - ContainerPort: 9042, - }, - - - }, - }, - }, - }, - }, - }, +// Create various K8s resources +func CreateResource(input_yaml string, resource_type string, + image string, namespace string) { + if namespace == "" { + namespace = "clover-system" + } + // Check path from cloverctl first + exe_path, err := os.Executable() + abs_input_yaml := strings.Replace(exe_path, "cloverctl", "", -1) + + "/yaml/" + input_yaml + if _, err := os.Stat(abs_input_yaml); !os.IsNotExist(err) { + input_yaml = abs_input_yaml + } + in, err := ioutil.ReadFile(input_yaml) + if err != nil { + fmt.Println("Please specify a valid manifest yaml file") + return + } + out_json, err := yaml.YAMLToJSON(in) + if err != nil { + panic(err.Error()) } - service := &apiv1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "clover-controller", - Labels: map[string]string{ - "app": "clover-controller", - }, - - }, - Spec: apiv1.ServiceSpec{ - Selector: map[string]string{ - "app": "clover-controller", - }, - Type: "NodePort", - Ports: []apiv1.ServicePort{ - { - Name: "http", - Port: 80, - NodePort: 32044, - Protocol: "TCP", - }, - }, - }, + switch resource_type { + case "deployment": + deploy := v1beta1.Deployment{} + err = json.Unmarshal(out_json, &deploy) + deploy.Spec.Template.Spec.Containers[0].Image = image + CreateDeployment(&deploy, namespace) + fmt.Printf("Image: %s\n", + deploy.Spec.Template.Spec.Containers[0].Image) + case "service": + service := apiv1.Service{} + err = json.Unmarshal(out_json, &service) + CreateService(&service, namespace) + case "serviceaccount": + sa := apiv1.ServiceAccount{} + err = json.Unmarshal(out_json, &sa) + CreateServiceAccount(&sa, namespace) + case "clusterrolebinding": + clusterrolebinding := rbacv1.ClusterRoleBinding{} + err = json.Unmarshal(out_json, &clusterrolebinding) + CreateCRB(&clusterrolebinding) + case "statefulset": + statefulset := appsv1.StatefulSet{} + err = json.Unmarshal(out_json, &statefulset) + CreateStatefulSet(&statefulset, namespace) + case "pod": + pod := apiv1.Pod{} + err = json.Unmarshal(out_json, &pod) + CreatePod(&pod, namespace) + case "daemonset": + daemon := v1beta1ext.DaemonSet{} + err = json.Unmarshal(out_json, &daemon) + daemon.Spec.Template.Spec.Containers[0].Image = image + CreateDaemonSet(&daemon, namespace) + fmt.Printf("Image: %s\n", + daemon.Spec.Template.Spec.Containers[0].Image) + + default: + fmt.Println("No resource selected") } +} - return deployment, service +// Delete K8s resources +func DeleteResource(deploy_name string, resource_type string, + namespace string) { + clientset := setClient() + deletePolicy := metav1.DeletePropagationForeground + switch resource_type { + case "deployment": + deploymentsClient := clientset.AppsV1().Deployments(namespace) + if err := deploymentsClient.Delete(deploy_name, + &metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + fmt.Printf("Error deleting %v: %v\n", resource_type, err) + return + } + case "service": + servicesClient := clientset.CoreV1().Services(namespace) + if err := servicesClient.Delete(deploy_name, &metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + fmt.Printf("Error deleting %v: %v\n", resource_type, err) + return + } + case "serviceaccount": + saClient := clientset.CoreV1().ServiceAccounts(namespace) + if err := saClient.Delete(deploy_name, &metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + fmt.Printf("Error deleting %v: %v\n", resource_type, err) + return + } + case "clusterrolebinding": + crbClient := clientset.RbacV1().ClusterRoleBindings() + if err := crbClient.Delete(deploy_name, &metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + fmt.Printf("Error deleting %v: %v\n", resource_type, err) + return + } + case "statefulset": + statefulClient := clientset.AppsV1().StatefulSets(namespace) + if err := statefulClient.Delete(deploy_name, &metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + fmt.Printf("Error deleting %v: %v\n", resource_type, err) + return + } + case "pod": + podClient := clientset.CoreV1().Pods(namespace) + if err := podClient.Delete(deploy_name, &metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + fmt.Printf("Error deleting %v: %v\n", resource_type, err) + return + } + case "daemonset": + daemonsClient := clientset.AppsV1().DaemonSets(namespace) + if err := daemonsClient.Delete(deploy_name, + &metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + fmt.Printf("Error deleting %v: %v\n", resource_type, err) + return + } + } + fmt.Printf("Deleted %s %s\n", deploy_name, resource_type) } -func setCollectorDeploy () (*appsv1.Deployment, *apiv1.Service) { - - deployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "clover-collector", - }, - Spec: appsv1.DeploymentSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "app": "clover-collector", - }, - }, - Template: apiv1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": "clover-collector", - }, - }, - Spec: apiv1.PodSpec{ - Containers: []apiv1.Container{ - { - Name: "clover-collector", - Image: "localhost:5000/clover-collector:latest", - Ports: []apiv1.ContainerPort{ - { - Name: "redis", - Protocol: apiv1.ProtocolTCP, - ContainerPort: 6379, - }, - { - Name: "grpc", - Protocol: apiv1.ProtocolTCP, - ContainerPort: 50054, - }, - { - Name: "prometheus", - Protocol: apiv1.ProtocolTCP, - ContainerPort: 9090, - }, - { - Name: "jaeger", - Protocol: apiv1.ProtocolTCP, - ContainerPort: 16686, - }, - { - Name: "cassandra", - Protocol: apiv1.ProtocolTCP, - ContainerPort: 9042, - }, - - - }, - }, - }, - }, - }, - }, +// Create ServiceAccount +func CreateServiceAccount(sa *apiv1.ServiceAccount, namespace string) { + clientset := setClient() + saClient := clientset.CoreV1().ServiceAccounts(namespace) + result, err := saClient.Create(sa) + if err != nil { + fmt.Printf("Error creating serviceaccount: %v\n", err); return } + fmt.Printf("Created serviceaccount %q.\n", + result.GetObjectMeta().GetName()) +} - service := &apiv1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "clover-collector", - Labels: map[string]string{ - "app": "clover-collector", - }, - - }, - Spec: apiv1.ServiceSpec{ - Selector: map[string]string{ - "app": "clover-collector", - }, - Ports: []apiv1.ServicePort{ - { - Name: "grpc", - Port: 50054, - }, - { - Name: "redis", - Port: 6379, - }, - { - Name: "prometheus", - Port: 9090, - }, - { - Name: "jaeger", - Port: 16686, - }, - { - Name: "cassandra", - Port: 9042, - }, - - }, - }, +// Create ClusterRoleBinding +func CreateCRB(sa *rbacv1.ClusterRoleBinding) { + clientset := setClient() + crbClient := clientset.RbacV1().ClusterRoleBindings() + result, err := crbClient.Create(sa) + if err != nil { + fmt.Printf("Error creating clusterrolebinding: %v\n", err); return } - return deployment, service + fmt.Printf("Created clusterrolebinding %q.\n", + result.GetObjectMeta().GetName()) } -func DeployCloverSystem(action string, namespace string) { - if action == "create" { - // Create clover-system namespace - configNamespace("clover-system", "create") - // Controller - deployment, service := setControllerDeploy() - DeployService(deployment, service, namespace) - // Collector - deployment, service = setCollectorDeploy() - DeployService(deployment, service, namespace) - } else if action == "delete" { - fmt.Println("Deleting clover-system services...\n") - DeleteService("clover-controller", namespace) - DeleteService("clover-collector", namespace) - configNamespace("clover-system", "delete") +// Create DaemonSet +func CreateDaemonSet(daemonset *v1beta1ext.DaemonSet, namespace string) { + clientset := setClient() + daemonsClient := clientset.ExtensionsV1beta1().DaemonSets(namespace) + result, err := daemonsClient.Create(daemonset) + if err != nil { + fmt.Printf("Error creating daemonset: %v\n", err); return } - + fmt.Printf("Created daemonset %q.\n", result.GetObjectMeta().GetName()) } -func DeleteService(deploy_name string, namespace string) { - +// Create Deployment +func CreateDeployment(deployment *v1beta1.Deployment, namespace string) { clientset := setClient() - deploymentsClient := clientset.AppsV1().Deployments(namespace) - servicesClient := clientset.CoreV1().Services(namespace) - - // Delete Deployment - deletePolicy := metav1.DeletePropagationForeground - if err := deploymentsClient.Delete(deploy_name, &metav1.DeleteOptions{ - PropagationPolicy: &deletePolicy, - }); err != nil { - panic(err) + deploymentsClient := clientset.AppsV1beta1().Deployments(namespace) + result, err := deploymentsClient.Create(deployment) + if err != nil { + fmt.Printf("Error creating deployment: %v\n", err); return } - fmt.Printf("Deleted %s deployment\n", deploy_name) + fmt.Printf("Created deployment %q.\n", result.GetObjectMeta().GetName()) +} - // Delete Service - if err := servicesClient.Delete(deploy_name, &metav1.DeleteOptions{ - PropagationPolicy: &deletePolicy, - }); err != nil { - panic(err) +// Create StatefulSet +func CreateStatefulSet(statefulset *appsv1.StatefulSet, namespace string) { + clientset := setClient() + statefulsetClient := clientset.AppsV1().StatefulSets(namespace) + result, err := statefulsetClient.Create(statefulset) + if err != nil { + fmt.Printf("Error creating statefulset: %v\n", err); return } - fmt.Printf("Deleted %s service\n", deploy_name) + fmt.Printf("Created statefulset %q.\n", result.GetObjectMeta().GetName()) } -func DeployService(deployment *appsv1.Deployment, service *apiv1.Service, namespace string) { - +// Create Pod +func CreatePod(pod *apiv1.Pod, namespace string) { clientset := setClient() - deploymentsClient := clientset.AppsV1().Deployments(namespace) - - - // Create Deployment - fmt.Println("Creating deployment...") - result, err := deploymentsClient.Create(deployment) + podClient := clientset.CoreV1().Pods(namespace) + result, err := podClient.Create(pod) if err != nil { + fmt.Printf("Error creating pod: %v\n", err); return panic(err) } - fmt.Printf("Created deployment %q.\n", result.GetObjectMeta().GetName()) + fmt.Printf("Created pod %q.\n", result.GetObjectMeta().GetName()) +} - // Create Service - fmt.Println("Creating service...") +// Create Service +func CreateService(service *apiv1.Service, namespace string) { + clientset := setClient() servicesClient := clientset.CoreV1().Services(namespace) result1, err := servicesClient.Create(service) if err != nil { - panic(err) + fmt.Printf("Error creating service: %v\n", err); return } - fmt.Printf("Created service %q.\n", result1.GetObjectMeta().GetName()) - + fmt.Printf("Created service %q\n", result1.GetObjectMeta().GetName()) } -func configNamespace (name string, action string) { +// Create or delete namespace +func ConfigNamespace (name string, action string) { clientset := setClient() nameClient := clientset.CoreV1().Namespaces() @@ -312,7 +279,7 @@ func configNamespace (name string, action string) { if err := nameClient.Delete(name, &metav1.DeleteOptions{ PropagationPolicy: &deletePolicy, }); err != nil { - panic(err) + fmt.Printf("Error deleting namespace: %v\n", err); return } fmt.Printf("Deleted %s namespace\n", name) } @@ -330,13 +297,12 @@ func GetServices() *apiv1.ServiceList { fmt.Printf("Kind: %s\n", service.Kind) fmt.Printf("Labels: %s\n", service.GetLabels()) fmt.Printf("Type: %s\n", service.Spec.Type) - //fmt.Printf("External IP: %v\n", service.Spec.ExternalIPs) fmt.Printf("Cluster IP: %s\n", service.Spec.ClusterIP) for _, port := range service.Spec.Ports { - fmt.Printf("Port Name: %s, Port# %d, NodePort: %d\n", port.Name, port.Port, port.NodePort) + fmt.Printf("Port Name: %s, Port# %d, NodePort: %d\n", + port.Name, port.Port, port.NodePort) } - for _, ip := range service.Status.LoadBalancer.Ingress { fmt.Printf("LB IP: %s \n", ip.IP) } @@ -345,7 +311,6 @@ func GetServices() *apiv1.ServiceList { } func GetDeployments(namespace string) []appsv1.Deployment { - clientset := setClient() deploymentsClient := clientset.AppsV1().Deployments(namespace) @@ -360,7 +325,6 @@ func GetDeployments(namespace string) []appsv1.Deployment { } func GetServicesPortIP(service_name string) (int32, string) { - clientset := setClient() services, err := clientset.Core().Services("").List(metav1.ListOptions{}) var nodeport int32 @@ -382,7 +346,6 @@ func GetServicesPortIP(service_name string) (int32, string) { } } } - return nodeport, ipaddress } -- cgit 1.2.3-korg