From 5e213108dfade163a85cff9b9156de9bd2c18887 Mon Sep 17 00:00:00 2001
From: Eddie Arrage <eddie.arrage@huawei.com>
Date: Tue, 3 Apr 2018 02:50:36 +0000
Subject: Fix Nginx lb in k8s/istio
- Provide workaround to make nginx lb work properly
- nginx_client sample can modify default load balancing
from three to two servers at runtime
- Ensure port 9180 is used for default deploy for lb and
servers
- Modify render_yaml to specify deploy_name so that
clover-server1, 2, 3 can be used for default lb config
- Ensure proxy template is aligned to lb to allow the
source IP from originating host to be propagated to final
destination
- Fix default nginx proxy server_name to 'proxy-access-control'
and change default proxy destination to 'http-lb'
- Split lb service_type to 'lbv1' and 'lbv2' to provide an example
of how to modify the run-time configuration of the load balancer
after deployment - modify http-lb-v2 to use clover-server4/5 instead
of the defualt clover-server1/2/3 - modify http-lb-v1 to use
clover-server1/2 instead of 1/2/3
- Aligned pod IP retrival method with nginx_client.py
Change-Id: I73fa60a69c93ae1e82a477ef6283c00f67a21360
Signed-off-by: Eddie Arrage <eddie.arrage@huawei.com>
---
samples/services/nginx/docker/grpc/nginx_client.py | 38 +++++++++++++---------
.../nginx/docker/grpc/nginx_grpc_server.py | 7 ++--
.../nginx/docker/grpc/templates/lb.template | 15 ++++++++-
.../nginx/docker/grpc/templates/proxy.template | 6 ++++
samples/services/nginx/yaml/render_yaml.py | 14 +++++---
5 files changed, 56 insertions(+), 24 deletions(-)
(limited to 'samples/services')
diff --git a/samples/services/nginx/docker/grpc/nginx_client.py b/samples/services/nginx/docker/grpc/nginx_client.py
index dfefb08..7a07464 100644
--- a/samples/services/nginx/docker/grpc/nginx_client.py
+++ b/samples/services/nginx/docker/grpc/nginx_client.py
@@ -16,18 +16,24 @@ import nginx_pb2
import nginx_pb2_grpc
-def run(args):
+def run(args, grpc_port='50054'):
# get pod ip for grpc
pod_ip = get_podip(args['service_name'])
if pod_ip == '':
- return "Cant find service with name: {}".format(args['service_name'])
- nginx_grpc = pod_ip + ':50054'
+ return "Cant find service: {}".format(args['service_name'])
+ nginx_grpc = pod_ip + ':' + grpc_port
channel = grpc.insecure_channel(nginx_grpc)
stub = nginx_pb2_grpc.ControllerStub(channel)
# modify config
- if args['service_type'] == 'lb':
- modify_lb(stub)
+ if args['service_type'] == 'lbv1':
+ slb_list = pickle.dumps(
+ ['clover-server1:9180', 'clover-server2:9180'])
+ modify_lb(stub, slb_list)
+ if args['service_type'] == 'lbv2':
+ slb_list = pickle.dumps(
+ ['clover-server4:9180', 'clover-server5:9180'])
+ modify_lb(stub, slb_list)
elif args['service_type'] == 'proxy':
modify_proxy(stub)
elif args['service_type'] == 'server':
@@ -38,14 +44,16 @@ def run(args):
def get_podip(pod_name):
- config.load_kube_config()
- v1 = client.CoreV1Api()
- ret = v1.list_pod_for_all_namespaces(watch=False)
ip = ''
- for i in ret.items:
- if i.metadata.name.lower().find(pod_name.lower()) != -1:
- print(i.status.pod_ip)
- ip = i.status.pod_ip
+ if pod_name != '':
+ config.load_kube_config()
+ v1 = client.CoreV1Api()
+ ret = v1.list_pod_for_all_namespaces(watch=False)
+ for i in ret.items:
+ if i.metadata.name.lower().find(pod_name.lower()) != -1:
+ print("Pod IP: {}".format(i.status.pod_ip))
+ ip = i.status.pod_ip
+ return str(ip)
return str(ip)
@@ -64,11 +72,9 @@ def modify_server(stub):
print(response.message)
-def modify_lb(stub):
- slb_list = pickle.dumps(
- ['clover-server1', 'clover-server2', 'clover-server3'])
+def modify_lb(stub, slb_list):
response = stub.ModifyLB(nginx_pb2.ConfigLB(
- server_port='9188', server_name='http-lb',
+ server_port='9180', server_name='http-lb',
slb_list=slb_list,
slb_group='cloverlb', lb_path='/'))
print(response.message)
diff --git a/samples/services/nginx/docker/grpc/nginx_grpc_server.py b/samples/services/nginx/docker/grpc/nginx_grpc_server.py
index 6f2de0f..1dfe708 100644
--- a/samples/services/nginx/docker/grpc/nginx_grpc_server.py
+++ b/samples/services/nginx/docker/grpc/nginx_grpc_server.py
@@ -33,8 +33,8 @@ class Controller(nginx_pb2_grpc.ControllerServicer):
# self.template_file = 'templates/proxy.template'
self.template_file = '/grpc/templates/proxy.template'
self.ModifyProxy(nginx_pb2.ConfigProxy(
- server_port='9180', server_name='http-proxy',
- location_path='/', proxy_path='http://clover-server:9180',
+ server_port='9180', server_name='proxy-access-control',
+ location_path='/', proxy_path='http://http-lb:9180',
mirror_path='http://snort-ids:80'), "")
if service_type == "server":
# self.template_file = 'templates/server.template'
@@ -47,7 +47,8 @@ class Controller(nginx_pb2_grpc.ControllerServicer):
# self.template_file = 'templates/lb.template'
self.template_file = '/grpc/templates/lb.template'
slb_list = pickle.dumps(
- ['clover-server1', 'clover-server2', 'clover-server3'])
+ ['clover-server1:9180', 'clover-server2:9180',
+ 'clover-server3:9180'])
self.ModifyLB(nginx_pb2.ConfigLB(
server_port='9180', server_name='http-lb',
slb_list=slb_list,
diff --git a/samples/services/nginx/docker/grpc/templates/lb.template b/samples/services/nginx/docker/grpc/templates/lb.template
index 4866408..37de1af 100644
--- a/samples/services/nginx/docker/grpc/templates/lb.template
+++ b/samples/services/nginx/docker/grpc/templates/lb.template
@@ -61,10 +61,20 @@ http {
upstream {{ slb_group }} {
{%- for item in slb_list %}
- server {{ item }};
+ server 127.0.0.1:800{{ loop.index }};
{%- endfor %}
}
+ {%- for item in slb_list %}
+ server {
+ listen 800{{ loop.index }};
+ location / {
+ proxy_http_version 1.1;
+ proxy_pass http://{{ item }};
+ }
+ }
+ {%- endfor %}
+
server {
listen {{ server_port }};
server_name {{ server_name }};
@@ -73,6 +83,9 @@ http {
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
+ proxy_set_header X-Forwarded-For $remote_addr;
+ proxy_set_header X-Forwarded-Proto http;
+ proxy_set_header X-Real-IP $remote_addr;
proxy_pass http://{{ slb_group }};
}
diff --git a/samples/services/nginx/docker/grpc/templates/proxy.template b/samples/services/nginx/docker/grpc/templates/proxy.template
index 72d611e..e7305b3 100644
--- a/samples/services/nginx/docker/grpc/templates/proxy.template
+++ b/samples/services/nginx/docker/grpc/templates/proxy.template
@@ -67,6 +67,9 @@ http {
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
+ proxy_set_header X-Forwarded-For $remote_addr;
+ proxy_set_header X-Forwarded-Proto http;
+ proxy_set_header X-Real-IP $remote_addr;
proxy_pass {{ proxy_path }};
post_action @post_ids;
@@ -76,6 +79,9 @@ http {
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
+ proxy_set_header X-Forwarded-For $remote_addr;
+ proxy_set_header X-Forwarded-Proto http;
+ proxy_set_header X-Real-IP $remote_addr;
proxy_pass {{ mirror_path }};
}
diff --git a/samples/services/nginx/yaml/render_yaml.py b/samples/services/nginx/yaml/render_yaml.py
index 527ba8d..850f715 100644
--- a/samples/services/nginx/yaml/render_yaml.py
+++ b/samples/services/nginx/yaml/render_yaml.py
@@ -16,13 +16,16 @@ def render_yaml(args):
grpc_port = '50054'
if args['service_type'] == 'lb':
out_file = 'lb.yaml'
- deploy_name = 'http-lb'
+ if args['deploy_name'] == 'default':
+ args['deploy_name'] = 'http-lb'
elif args['service_type'] == 'proxy':
out_file = 'proxy.yaml'
- deploy_name = 'proxy-access-control'
+ if args['deploy_name'] == 'default':
+ args['deploy_name'] = 'proxy-access-control'
elif args['service_type'] == 'server':
out_file = 'server.yaml'
- deploy_name = 'clover-server'
+ if args['deploy_name'] == 'default':
+ args['deploy_name'] = 'clover-server'
else:
return "Invalid service type: {}".format(args['service_type'])
@@ -33,7 +36,7 @@ def render_yaml(args):
image_path=args['image_path'],
image_name=args['image_name'],
image_tag=args['image_tag'],
- deploy_name=deploy_name,
+ deploy_name=args['deploy_name'],
server_port=server_port,
grpc_port=grpc_port
)
@@ -60,5 +63,8 @@ if __name__ == '__main__':
parser.add_argument(
'--image_tag', default='latest',
help='The image tag to use')
+ parser.add_argument(
+ '--deploy_name', default='default',
+ help='The deployment name to use')
args = parser.parse_args()
print(render_yaml(vars(args)))
--
cgit