From afc4beed0f62ea26b155e0ac6f7b1f84e10bd38b Mon Sep 17 00:00:00 2001 From: Eddie Arrage Date: Fri, 22 Jun 2018 21:25:54 +0000 Subject: Add file upload/download and configure URL paths - Compile nginx from source in order to employ additional modules - Add nginx-upload-module for high performance file upload that avoids the need for file copies with a web application. - File upload allows for placement of files for file download for performance benchmarking. - File upload can also be used directly for bi-directional throughput testing having emulated clients upload files while file downloads simultaneously occur. - Nginx file upload stores files with hash to avoid conflicting file names. Upload block in nginx config is configured to send REST message to clover-controller with file metadata (original filename, size, etc.) clover-controller will be responsible for modifying the hashed filename and placing in a target directory within an nginx server. - Build also adds nginx-rtmp module to act as streaming media server L7 loader will be extended to fetch streaming files from RTMP servers. - Add ability to create directories in server site root and create the location directive(s) in nginx configuration - Separated upload for configuration (download files in various paths) from upload for testing (upload to create bi-directional session throughput) - Upload for testing does not sent upload metadata to clover-controller - Added ability to move upload files to file folders in the nginx site root to use for download - Delete files in upload folder - Fixed issue with 426 Upgrade Required error message when upload module sends upload metadata to clover-controller - Added server name to metadata sent to clover-controller Change-Id: Ib4cf6240f92360b82f378c062675f4fdaa19ca93 Signed-off-by: Eddie Arrage --- samples/services/nginx/docker/grpc/nginx.proto | 4 + samples/services/nginx/docker/grpc/nginx_client.py | 16 +- .../nginx/docker/grpc/nginx_grpc_server.py | 113 ++++++++++++-- samples/services/nginx/docker/grpc/nginx_pb2.py | 46 ++++-- .../nginx/docker/grpc/templates/server.template | 165 +++++++++++++++------ .../docker/grpc/templates/upload_form.template | 19 +++ 6 files changed, 296 insertions(+), 67 deletions(-) create mode 100644 samples/services/nginx/docker/grpc/templates/upload_form.template (limited to 'samples/services/nginx/docker/grpc') diff --git a/samples/services/nginx/docker/grpc/nginx.proto b/samples/services/nginx/docker/grpc/nginx.proto index 3779a82..8255719 100644 --- a/samples/services/nginx/docker/grpc/nginx.proto +++ b/samples/services/nginx/docker/grpc/nginx.proto @@ -36,6 +36,10 @@ message ConfigServer { string server_name = 2; string site_root = 3; string site_index = 4; + string upload_path_config = 5; + string locations = 6; + string upload_path_test = 7; + string files = 8; } message ConfigLB { diff --git a/samples/services/nginx/docker/grpc/nginx_client.py b/samples/services/nginx/docker/grpc/nginx_client.py index 7a07464..e4ddb2a 100644 --- a/samples/services/nginx/docker/grpc/nginx_client.py +++ b/samples/services/nginx/docker/grpc/nginx_client.py @@ -66,9 +66,21 @@ def modify_proxy(stub): def modify_server(stub): + loc = [] + val = {} + val['uri_match'] = "/test" + val['directive'] = "try_files $uri @default1" + val['path'] = "/test" + loc.append(val) + locations = pickle.dumps(loc) + files = pickle.dumps([]) response = stub.ModifyServer(nginx_pb2.ConfigServer( - server_port='9180', server_name='clover-server', - site_root='/var/www/html', site_index='index.nginx-debian.html')) + server_port='9180', server_name='clover-server1', + upload_path_config='/upload', + upload_path_test='/upload_test', + locations=locations, + files=files, + site_root='/var/www/html', site_index='index.html')) print(response.message) diff --git a/samples/services/nginx/docker/grpc/nginx_grpc_server.py b/samples/services/nginx/docker/grpc/nginx_grpc_server.py index 1dfe708..5ad64b5 100644 --- a/samples/services/nginx/docker/grpc/nginx_grpc_server.py +++ b/samples/services/nginx/docker/grpc/nginx_grpc_server.py @@ -8,9 +8,12 @@ from concurrent import futures import time +import os import sys import grpc import subprocess +import psutil +import shutil import pickle import logging import nginx_pb2 @@ -28,23 +31,32 @@ class Controller(nginx_pb2_grpc.ControllerServicer): logging.basicConfig(filename='nginx.log', level=logging.DEBUG) self.service_type = service_type self.out_file = '/etc/nginx/nginx.conf' - # self.out_file = 'testfile' + self.nginx = 0 if service_type == "proxy": - # self.template_file = 'templates/proxy.template' self.template_file = '/grpc/templates/proxy.template' self.ModifyProxy(nginx_pb2.ConfigProxy( server_port='9180', server_name='proxy-access-control', location_path='/', proxy_path='http://http-lb:9180', mirror_path='http://snort-ids:80'), "") if service_type == "server": - # self.template_file = 'templates/server.template' self.template_file = '/grpc/templates/server.template' + loc = [] + val = {} + val['uri_match'] = "/test" + val['directive'] = "try_files $uri @default1" + val['path'] = "/test" + loc.append(val) + locations = pickle.dumps(loc) + files = pickle.dumps([]) self.ModifyServer(nginx_pb2.ConfigServer( server_port='9180', server_name='clover-server', site_root='/var/www/html', - site_index='index.nginx-debian.html'), "") + upload_path_config='/upload', + upload_path_test='/upload_test', + locations=locations, + files=files, + site_index='index.html'), "") if service_type == "lb": - # self.template_file = 'templates/lb.template' self.template_file = '/grpc/templates/lb.template' slb_list = pickle.dumps( ['clover-server1:9180', 'clover-server2:9180', @@ -76,18 +88,54 @@ class Controller(nginx_pb2_grpc.ControllerServicer): def ModifyServer(self, r, context): try: + locations = pickle.loads(r.locations) + # Generate nginx config with open(self.template_file) as f: tmpl = Template(f.read()) output = tmpl.render( server_port=r.server_port, server_name=r.server_name, site_root=r.site_root, - site_index=r.site_index + site_index=r.site_index, + upload_path_config=r.upload_path_config, + upload_path_test=r.upload_path_test, + locations=locations ) with open(self.out_file, "wb") as fh: fh.write(output) + + # Make dirs for locations + for l in locations: + self.MakeDirs(l['path']) + + # Generate upload form for config + template_upload = '/grpc/templates/upload_form.template' + out_file = r.site_root + '/' + r.site_index + with open(template_upload) as f: + tmpl = Template(f.read()) + output = tmpl.render( + upload_path=r.upload_path_config + ) + with open(out_file, "wb") as fh: + fh.write(output) + + # Generate upload form for test + template_upload = '/grpc/templates/upload_form.template' + out_file = r.site_root + '/' + 'upload_test.html' + with open(template_upload) as f: + tmpl = Template(f.read()) + output = tmpl.render( + upload_path=r.upload_path_test + ) + with open(out_file, "wb") as fh: + fh.write(output) + msg = "Modified nginx config" - self.RestartNginx() + self.RestartNginx('custom') + + # Move files that have been uploaded + file_ops = pickle.loads(r.files) + self.MoveServerFiles(file_ops) except Exception as e: logging.debug(e) msg = "Failed to modify nginx config" @@ -113,9 +161,54 @@ class Controller(nginx_pb2_grpc.ControllerServicer): msg = "Failed to modify nginx config" return nginx_pb2.NginxReply(message=msg) - def RestartNginx(self): - subprocess.Popen( - ["service nginx restart"], shell=True) + def DeleteServerFiles(self, upload_folder): + for f in os.listdir(upload_folder): + file_path = os.path.join(upload_folder, f) + try: + if os.path.isfile(file_path): + os.unlink(file_path) + except Exception as e: + logging.debug(e) + + def MoveServerFiles(self, file_ops): + response = 1 + try: + for fo in file_ops: + try: + shutil.move(fo['src_file'], fo['dest_file']) + except Exception as e: + logging.debug(e) + response = 0 + except Exception as e: + logging.debug(e) + return response + + def RestartNginx(self, package='default'): + if package == 'custom': + if self.nginx == 0: + p = subprocess.Popen(["nginx"], + stdout=subprocess.PIPE, + shell=True, + preexec_fn=os.setsid) + else: + for proc in psutil.process_iter(): + if proc.name() == "nginx": + proc.kill() + p = subprocess.Popen(["nginx"], + stdout=subprocess.PIPE, + shell=True, + preexec_fn=os.setsid) + self.nginx = p + else: + subprocess.Popen( + ["service nginx restart"], shell=True) + + def MakeDirs(self, path, prefix='/var/www/html/'): + try: + path = prefix + path.strip('/') + os.makedirs(path) + except Exception as e: + logging.debug(e) def ProcessAlerts(self, request, context): try: diff --git a/samples/services/nginx/docker/grpc/nginx_pb2.py b/samples/services/nginx/docker/grpc/nginx_pb2.py index 3600b6d..5665fc4 100644 --- a/samples/services/nginx/docker/grpc/nginx_pb2.py +++ b/samples/services/nginx/docker/grpc/nginx_pb2.py @@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor( name='nginx.proto', package='nginx', syntax='proto3', - serialized_pb=_b('\n\x0bnginx.proto\x12\x05nginx\"3\n\x0c\x41lertMessage\x12\x10\n\x08\x65vent_id\x18\x01 \x01(\t\x12\x11\n\tredis_key\x18\x02 \x01(\t\"w\n\x0b\x43onfigProxy\x12\x13\n\x0bserver_port\x18\x01 \x01(\t\x12\x13\n\x0bserver_name\x18\x02 \x01(\t\x12\x15\n\rlocation_path\x18\x03 \x01(\t\x12\x12\n\nproxy_path\x18\x04 \x01(\t\x12\x13\n\x0bmirror_path\x18\x05 \x01(\t\"_\n\x0c\x43onfigServer\x12\x13\n\x0bserver_port\x18\x01 \x01(\t\x12\x13\n\x0bserver_name\x18\x02 \x01(\t\x12\x11\n\tsite_root\x18\x03 \x01(\t\x12\x12\n\nsite_index\x18\x04 \x01(\t\"j\n\x08\x43onfigLB\x12\x13\n\x0bserver_port\x18\x01 \x01(\t\x12\x13\n\x0bserver_name\x18\x02 \x01(\t\x12\x10\n\x08slb_list\x18\x03 \x01(\t\x12\x11\n\tslb_group\x18\x04 \x01(\t\x12\x0f\n\x07lb_path\x18\x05 \x01(\t\"\x1d\n\nNginxReply\x12\x0f\n\x07message\x18\x01 \x01(\t2\xeb\x01\n\nController\x12\x36\n\x0bModifyProxy\x12\x12.nginx.ConfigProxy\x1a\x11.nginx.NginxReply\"\x00\x12\x38\n\x0cModifyServer\x12\x13.nginx.ConfigServer\x1a\x11.nginx.NginxReply\"\x00\x12\x30\n\x08ModifyLB\x12\x0f.nginx.ConfigLB\x1a\x11.nginx.NginxReply\"\x00\x12\x39\n\rProcessAlerts\x12\x13.nginx.AlertMessage\x1a\x11.nginx.NginxReply\"\x00\x62\x06proto3') + serialized_pb=_b('\n\x0bnginx.proto\x12\x05nginx\"3\n\x0c\x41lertMessage\x12\x10\n\x08\x65vent_id\x18\x01 \x01(\t\x12\x11\n\tredis_key\x18\x02 \x01(\t\"w\n\x0b\x43onfigProxy\x12\x13\n\x0bserver_port\x18\x01 \x01(\t\x12\x13\n\x0bserver_name\x18\x02 \x01(\t\x12\x15\n\rlocation_path\x18\x03 \x01(\t\x12\x12\n\nproxy_path\x18\x04 \x01(\t\x12\x13\n\x0bmirror_path\x18\x05 \x01(\t\"\xb7\x01\n\x0c\x43onfigServer\x12\x13\n\x0bserver_port\x18\x01 \x01(\t\x12\x13\n\x0bserver_name\x18\x02 \x01(\t\x12\x11\n\tsite_root\x18\x03 \x01(\t\x12\x12\n\nsite_index\x18\x04 \x01(\t\x12\x1a\n\x12upload_path_config\x18\x05 \x01(\t\x12\x11\n\tlocations\x18\x06 \x01(\t\x12\x18\n\x10upload_path_test\x18\x07 \x01(\t\x12\r\n\x05\x66iles\x18\x08 \x01(\t\"j\n\x08\x43onfigLB\x12\x13\n\x0bserver_port\x18\x01 \x01(\t\x12\x13\n\x0bserver_name\x18\x02 \x01(\t\x12\x10\n\x08slb_list\x18\x03 \x01(\t\x12\x11\n\tslb_group\x18\x04 \x01(\t\x12\x0f\n\x07lb_path\x18\x05 \x01(\t\"\x1d\n\nNginxReply\x12\x0f\n\x07message\x18\x01 \x01(\t2\xeb\x01\n\nController\x12\x36\n\x0bModifyProxy\x12\x12.nginx.ConfigProxy\x1a\x11.nginx.NginxReply\"\x00\x12\x38\n\x0cModifyServer\x12\x13.nginx.ConfigServer\x1a\x11.nginx.NginxReply\"\x00\x12\x30\n\x08ModifyLB\x12\x0f.nginx.ConfigLB\x1a\x11.nginx.NginxReply\"\x00\x12\x39\n\rProcessAlerts\x12\x13.nginx.AlertMessage\x1a\x11.nginx.NginxReply\"\x00\x62\x06proto3') ) @@ -157,6 +157,34 @@ _CONFIGSERVER = _descriptor.Descriptor( message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='upload_path_config', full_name='nginx.ConfigServer.upload_path_config', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='locations', full_name='nginx.ConfigServer.locations', index=5, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='upload_path_test', full_name='nginx.ConfigServer.upload_path_test', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='files', full_name='nginx.ConfigServer.files', index=7, + number=8, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -169,8 +197,8 @@ _CONFIGSERVER = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=196, - serialized_end=291, + serialized_start=197, + serialized_end=380, ) @@ -228,8 +256,8 @@ _CONFIGLB = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=293, - serialized_end=399, + serialized_start=382, + serialized_end=488, ) @@ -259,8 +287,8 @@ _NGINXREPLY = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=401, - serialized_end=430, + serialized_start=490, + serialized_end=519, ) DESCRIPTOR.message_types_by_name['AlertMessage'] = _ALERTMESSAGE @@ -313,8 +341,8 @@ _CONTROLLER = _descriptor.ServiceDescriptor( file=DESCRIPTOR, index=0, options=None, - serialized_start=433, - serialized_end=668, + serialized_start=522, + serialized_end=757, methods=[ _descriptor.MethodDescriptor( name='ModifyProxy', diff --git a/samples/services/nginx/docker/grpc/templates/server.template b/samples/services/nginx/docker/grpc/templates/server.template index b5f8f1f..c1582fa 100644 --- a/samples/services/nginx/docker/grpc/templates/server.template +++ b/samples/services/nginx/docker/grpc/templates/server.template @@ -3,69 +3,142 @@ worker_processes auto; pid /run/nginx.pid; events { - worker_connections 768; - # multi_accept on; + worker_connections 768; + # multi_accept on; } http { - ## - # Basic Settings - ## + ## + # Basic Settings + ## - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - # server_tokens off; + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + # server_tokens off; - include /etc/nginx/mime.types; - default_type application/octet-stream; + include /etc/nginx/mime.types; + default_type application/octet-stream; - ## - # SSL Settings - ## + ## + # SSL Settings + ## - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE - ssl_prefer_server_ciphers on; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE + ssl_prefer_server_ciphers on; - ## - # Logging Settings - ## + ## + # Logging Settings + ## - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; - ## - # Gzip Settings - ## + ## + # Gzip Settings + ## - gzip on; - gzip_disable "msie6"; + gzip on; + gzip_disable "msie6"; - # gzip_vary on; - # gzip_proxied any; - # gzip_comp_level 6; - # gzip_buffers 16 8k; - # gzip_http_version 1.1; - # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; + # gzip_vary on; + # gzip_proxied any; + # gzip_comp_level 6; + # gzip_buffers 16 8k; + # gzip_http_version 1.1; + # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; - ## - # Virtual Host Configs - ## + ## + # Virtual Host Configs + ## - include /etc/nginx/conf.d/*.conf; - #include /etc/nginx/sites-enabled/*; + include /etc/nginx/conf.d/*.conf; + #include /etc/nginx/sites-enabled/*; - server { - listen {{ server_port }}; - server_name {{ server_name }}; + server { + listen {{ server_port }}; + server_name {{ server_name }}; + client_max_body_size 500m; - root {{ site_root }}; - index {{ site_index }}; - } + root {{ site_root }}; + index {{ site_index }}; -} + {%- for l in locations %} + + location {{ l["uri_match"] }} { + {{ l["directive"] }}; + } + + {%- endfor %} + + location @default1 { + index ../index.html; + } + + location @default2 { + index ../../index.html; + } + + location @default3 { + index ../../../index.html; + } + + # Use to upload files for download + location {{ upload_path_config }} { + # Pass altered request body to this location + upload_pass @return_config; + + # Store files to this directory + upload_resumable on; + upload_state_store /tmp/state; + upload_store {{ site_root }}{{ upload_path_config }}; + + # Allow uploaded files to be read only by user + #upload_store_access user:r; + + # Set specified fields in request body + upload_set_form_field $upload_field_name.name "$upload_file_name"; + upload_set_form_field $upload_field_name.content_type "$upload_content_type"; + upload_set_form_field $upload_field_name.path "$upload_tmp_path"; + upload_set_form_field $upload_field_name.server "$server_name"; + + # Inform backend about hash and size of a file + upload_aggregate_form_field "$upload_field_name.md5" "$upload_file_md5"; + upload_aggregate_form_field "$upload_field_name.size" "$upload_file_size"; + upload_pass_form_field "^submit$|^description$"; + + upload_cleanup 400 404 499 500-505; + } + + location @return_config { + proxy_pass http://clover-controller:80; + proxy_http_version 1.1; + } + + # Use to upload files for performance testing + location {{ upload_path_test }} { + + upload_pass @return_path; + + # Store files to this directory + upload_resumable on; + upload_state_store /tmp/state; + upload_store {{ site_root }}{{ upload_path_test }}; + + upload_pass_form_field "^submit$|^description$"; + + upload_cleanup 400 404 499 500-505; + } + + location @return_path { + return 204; + } + + + } +} diff --git a/samples/services/nginx/docker/grpc/templates/upload_form.template b/samples/services/nginx/docker/grpc/templates/upload_form.template new file mode 100644 index 0000000..60f3967 --- /dev/null +++ b/samples/services/nginx/docker/grpc/templates/upload_form.template @@ -0,0 +1,19 @@ + + + + Test upload + + +

Select files to upload

+
+
+
+
+
+
+
+ + +
+ + -- cgit 1.2.3-korg