From 5e213108dfade163a85cff9b9156de9bd2c18887 Mon Sep 17 00:00:00 2001 From: Eddie Arrage Date: Tue, 3 Apr 2018 02:50:36 +0000 Subject: Fix Nginx lb in k8s/istio - Provide workaround to make nginx lb work properly - nginx_client sample can modify default load balancing from three to two servers at runtime - Ensure port 9180 is used for default deploy for lb and servers - Modify render_yaml to specify deploy_name so that clover-server1, 2, 3 can be used for default lb config - Ensure proxy template is aligned to lb to allow the source IP from originating host to be propagated to final destination - Fix default nginx proxy server_name to 'proxy-access-control' and change default proxy destination to 'http-lb' - Split lb service_type to 'lbv1' and 'lbv2' to provide an example of how to modify the run-time configuration of the load balancer after deployment - modify http-lb-v2 to use clover-server4/5 instead of the defualt clover-server1/2/3 - modify http-lb-v1 to use clover-server1/2 instead of 1/2/3 - Aligned pod IP retrival method with nginx_client.py Change-Id: I73fa60a69c93ae1e82a477ef6283c00f67a21360 Signed-off-by: Eddie Arrage --- samples/services/nginx/docker/grpc/nginx_client.py | 38 +++++++++++++--------- .../nginx/docker/grpc/nginx_grpc_server.py | 7 ++-- .../nginx/docker/grpc/templates/lb.template | 15 ++++++++- .../nginx/docker/grpc/templates/proxy.template | 6 ++++ samples/services/nginx/yaml/render_yaml.py | 14 +++++--- 5 files changed, 56 insertions(+), 24 deletions(-) (limited to 'samples') diff --git a/samples/services/nginx/docker/grpc/nginx_client.py b/samples/services/nginx/docker/grpc/nginx_client.py index dfefb08..7a07464 100644 --- a/samples/services/nginx/docker/grpc/nginx_client.py +++ b/samples/services/nginx/docker/grpc/nginx_client.py @@ -16,18 +16,24 @@ import nginx_pb2 import nginx_pb2_grpc -def run(args): +def run(args, grpc_port='50054'): # get pod ip for grpc pod_ip = get_podip(args['service_name']) if pod_ip == '': - return "Cant find service with name: {}".format(args['service_name']) - nginx_grpc = pod_ip + ':50054' + return "Cant find service: {}".format(args['service_name']) + nginx_grpc = pod_ip + ':' + grpc_port channel = grpc.insecure_channel(nginx_grpc) stub = nginx_pb2_grpc.ControllerStub(channel) # modify config - if args['service_type'] == 'lb': - modify_lb(stub) + if args['service_type'] == 'lbv1': + slb_list = pickle.dumps( + ['clover-server1:9180', 'clover-server2:9180']) + modify_lb(stub, slb_list) + if args['service_type'] == 'lbv2': + slb_list = pickle.dumps( + ['clover-server4:9180', 'clover-server5:9180']) + modify_lb(stub, slb_list) elif args['service_type'] == 'proxy': modify_proxy(stub) elif args['service_type'] == 'server': @@ -38,14 +44,16 @@ def run(args): def get_podip(pod_name): - config.load_kube_config() - v1 = client.CoreV1Api() - ret = v1.list_pod_for_all_namespaces(watch=False) ip = '' - for i in ret.items: - if i.metadata.name.lower().find(pod_name.lower()) != -1: - print(i.status.pod_ip) - ip = i.status.pod_ip + if pod_name != '': + config.load_kube_config() + v1 = client.CoreV1Api() + ret = v1.list_pod_for_all_namespaces(watch=False) + for i in ret.items: + if i.metadata.name.lower().find(pod_name.lower()) != -1: + print("Pod IP: {}".format(i.status.pod_ip)) + ip = i.status.pod_ip + return str(ip) return str(ip) @@ -64,11 +72,9 @@ def modify_server(stub): print(response.message) -def modify_lb(stub): - slb_list = pickle.dumps( - ['clover-server1', 'clover-server2', 'clover-server3']) +def modify_lb(stub, slb_list): response = stub.ModifyLB(nginx_pb2.ConfigLB( - server_port='9188', server_name='http-lb', + server_port='9180', server_name='http-lb', slb_list=slb_list, slb_group='cloverlb', lb_path='/')) print(response.message) diff --git a/samples/services/nginx/docker/grpc/nginx_grpc_server.py b/samples/services/nginx/docker/grpc/nginx_grpc_server.py index 6f2de0f..1dfe708 100644 --- a/samples/services/nginx/docker/grpc/nginx_grpc_server.py +++ b/samples/services/nginx/docker/grpc/nginx_grpc_server.py @@ -33,8 +33,8 @@ class Controller(nginx_pb2_grpc.ControllerServicer): # self.template_file = 'templates/proxy.template' self.template_file = '/grpc/templates/proxy.template' self.ModifyProxy(nginx_pb2.ConfigProxy( - server_port='9180', server_name='http-proxy', - location_path='/', proxy_path='http://clover-server:9180', + server_port='9180', server_name='proxy-access-control', + location_path='/', proxy_path='http://http-lb:9180', mirror_path='http://snort-ids:80'), "") if service_type == "server": # self.template_file = 'templates/server.template' @@ -47,7 +47,8 @@ class Controller(nginx_pb2_grpc.ControllerServicer): # self.template_file = 'templates/lb.template' self.template_file = '/grpc/templates/lb.template' slb_list = pickle.dumps( - ['clover-server1', 'clover-server2', 'clover-server3']) + ['clover-server1:9180', 'clover-server2:9180', + 'clover-server3:9180']) self.ModifyLB(nginx_pb2.ConfigLB( server_port='9180', server_name='http-lb', slb_list=slb_list, diff --git a/samples/services/nginx/docker/grpc/templates/lb.template b/samples/services/nginx/docker/grpc/templates/lb.template index 4866408..37de1af 100644 --- a/samples/services/nginx/docker/grpc/templates/lb.template +++ b/samples/services/nginx/docker/grpc/templates/lb.template @@ -61,10 +61,20 @@ http { upstream {{ slb_group }} { {%- for item in slb_list %} - server {{ item }}; + server 127.0.0.1:800{{ loop.index }}; {%- endfor %} } + {%- for item in slb_list %} + server { + listen 800{{ loop.index }}; + location / { + proxy_http_version 1.1; + proxy_pass http://{{ item }}; + } + } + {%- endfor %} + server { listen {{ server_port }}; server_name {{ server_name }}; @@ -73,6 +83,9 @@ http { proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto http; + proxy_set_header X-Real-IP $remote_addr; proxy_pass http://{{ slb_group }}; } diff --git a/samples/services/nginx/docker/grpc/templates/proxy.template b/samples/services/nginx/docker/grpc/templates/proxy.template index 72d611e..e7305b3 100644 --- a/samples/services/nginx/docker/grpc/templates/proxy.template +++ b/samples/services/nginx/docker/grpc/templates/proxy.template @@ -67,6 +67,9 @@ http { proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto http; + proxy_set_header X-Real-IP $remote_addr; proxy_pass {{ proxy_path }}; post_action @post_ids; @@ -76,6 +79,9 @@ http { proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto http; + proxy_set_header X-Real-IP $remote_addr; proxy_pass {{ mirror_path }}; } diff --git a/samples/services/nginx/yaml/render_yaml.py b/samples/services/nginx/yaml/render_yaml.py index 527ba8d..850f715 100644 --- a/samples/services/nginx/yaml/render_yaml.py +++ b/samples/services/nginx/yaml/render_yaml.py @@ -16,13 +16,16 @@ def render_yaml(args): grpc_port = '50054' if args['service_type'] == 'lb': out_file = 'lb.yaml' - deploy_name = 'http-lb' + if args['deploy_name'] == 'default': + args['deploy_name'] = 'http-lb' elif args['service_type'] == 'proxy': out_file = 'proxy.yaml' - deploy_name = 'proxy-access-control' + if args['deploy_name'] == 'default': + args['deploy_name'] = 'proxy-access-control' elif args['service_type'] == 'server': out_file = 'server.yaml' - deploy_name = 'clover-server' + if args['deploy_name'] == 'default': + args['deploy_name'] = 'clover-server' else: return "Invalid service type: {}".format(args['service_type']) @@ -33,7 +36,7 @@ def render_yaml(args): image_path=args['image_path'], image_name=args['image_name'], image_tag=args['image_tag'], - deploy_name=deploy_name, + deploy_name=args['deploy_name'], server_port=server_port, grpc_port=grpc_port ) @@ -60,5 +63,8 @@ if __name__ == '__main__': parser.add_argument( '--image_tag', default='latest', help='The image tag to use') + parser.add_argument( + '--deploy_name', default='default', + help='The deployment name to use') args = parser.parse_args() print(render_yaml(vars(args))) -- cgit 1.2.3-korg