diff options
author | 2018-06-22 21:25:54 +0000 | |
---|---|---|
committer | 2018-08-28 17:23:09 +0000 | |
commit | afc4beed0f62ea26b155e0ac6f7b1f84e10bd38b (patch) | |
tree | b0dbe1a6c8b750c5b6bd5b9f392ec3189f38a7f7 /samples/services/nginx/docker/grpc/nginx_grpc_server.py | |
parent | 1f543c55dd426a34ab3cafa514fa446c22b6fa03 (diff) |
Add file upload/download and configure URL paths
- Compile nginx from source in order to employ additional modules
- Add nginx-upload-module for high performance file upload that
avoids the need for file copies with a web application.
- File upload allows for placement of files for file download
for performance benchmarking.
- File upload can also be used directly for bi-directional throughput
testing having emulated clients upload files while file downloads
simultaneously occur.
- Nginx file upload stores files with hash to avoid conflicting
file names. Upload block in nginx config is configured to send
REST message to clover-controller with file metadata (original
filename, size, etc.) clover-controller will be responsible for
modifying the hashed filename and placing in a target directory
within an nginx server.
- Build also adds nginx-rtmp module to act as streaming media server
L7 loader will be extended to fetch streaming files from RTMP
servers.
- Add ability to create directories in server site root
and create the location directive(s) in nginx configuration
- Separated upload for configuration (download
files in various paths) from upload for testing (upload
to create bi-directional session throughput)
- Upload for testing does not sent upload metadata to
clover-controller
- Added ability to move upload files to file folders in the nginx site
root to use for download
- Delete files in upload folder
- Fixed issue with 426 Upgrade Required error message
when upload module sends upload metadata to clover-controller
- Added server name to metadata sent to clover-controller
Change-Id: Ib4cf6240f92360b82f378c062675f4fdaa19ca93
Signed-off-by: Eddie Arrage <eddie.arrage@huawei.com>
Diffstat (limited to 'samples/services/nginx/docker/grpc/nginx_grpc_server.py')
-rw-r--r-- | samples/services/nginx/docker/grpc/nginx_grpc_server.py | 113 |
1 files changed, 103 insertions, 10 deletions
diff --git a/samples/services/nginx/docker/grpc/nginx_grpc_server.py b/samples/services/nginx/docker/grpc/nginx_grpc_server.py index 1dfe708..5ad64b5 100644 --- a/samples/services/nginx/docker/grpc/nginx_grpc_server.py +++ b/samples/services/nginx/docker/grpc/nginx_grpc_server.py @@ -8,9 +8,12 @@ from concurrent import futures import time +import os import sys import grpc import subprocess +import psutil +import shutil import pickle import logging import nginx_pb2 @@ -28,23 +31,32 @@ class Controller(nginx_pb2_grpc.ControllerServicer): logging.basicConfig(filename='nginx.log', level=logging.DEBUG) self.service_type = service_type self.out_file = '/etc/nginx/nginx.conf' - # self.out_file = 'testfile' + self.nginx = 0 if service_type == "proxy": - # self.template_file = 'templates/proxy.template' self.template_file = '/grpc/templates/proxy.template' self.ModifyProxy(nginx_pb2.ConfigProxy( server_port='9180', server_name='proxy-access-control', location_path='/', proxy_path='http://http-lb:9180', mirror_path='http://snort-ids:80'), "") if service_type == "server": - # self.template_file = 'templates/server.template' self.template_file = '/grpc/templates/server.template' + loc = [] + val = {} + val['uri_match'] = "/test" + val['directive'] = "try_files $uri @default1" + val['path'] = "/test" + loc.append(val) + locations = pickle.dumps(loc) + files = pickle.dumps([]) self.ModifyServer(nginx_pb2.ConfigServer( server_port='9180', server_name='clover-server', site_root='/var/www/html', - site_index='index.nginx-debian.html'), "") + upload_path_config='/upload', + upload_path_test='/upload_test', + locations=locations, + files=files, + site_index='index.html'), "") if service_type == "lb": - # self.template_file = 'templates/lb.template' self.template_file = '/grpc/templates/lb.template' slb_list = pickle.dumps( ['clover-server1:9180', 'clover-server2:9180', @@ -76,18 +88,54 @@ class Controller(nginx_pb2_grpc.ControllerServicer): def ModifyServer(self, r, context): try: + locations = pickle.loads(r.locations) + # Generate nginx config with open(self.template_file) as f: tmpl = Template(f.read()) output = tmpl.render( server_port=r.server_port, server_name=r.server_name, site_root=r.site_root, - site_index=r.site_index + site_index=r.site_index, + upload_path_config=r.upload_path_config, + upload_path_test=r.upload_path_test, + locations=locations ) with open(self.out_file, "wb") as fh: fh.write(output) + + # Make dirs for locations + for l in locations: + self.MakeDirs(l['path']) + + # Generate upload form for config + template_upload = '/grpc/templates/upload_form.template' + out_file = r.site_root + '/' + r.site_index + with open(template_upload) as f: + tmpl = Template(f.read()) + output = tmpl.render( + upload_path=r.upload_path_config + ) + with open(out_file, "wb") as fh: + fh.write(output) + + # Generate upload form for test + template_upload = '/grpc/templates/upload_form.template' + out_file = r.site_root + '/' + 'upload_test.html' + with open(template_upload) as f: + tmpl = Template(f.read()) + output = tmpl.render( + upload_path=r.upload_path_test + ) + with open(out_file, "wb") as fh: + fh.write(output) + msg = "Modified nginx config" - self.RestartNginx() + self.RestartNginx('custom') + + # Move files that have been uploaded + file_ops = pickle.loads(r.files) + self.MoveServerFiles(file_ops) except Exception as e: logging.debug(e) msg = "Failed to modify nginx config" @@ -113,9 +161,54 @@ class Controller(nginx_pb2_grpc.ControllerServicer): msg = "Failed to modify nginx config" return nginx_pb2.NginxReply(message=msg) - def RestartNginx(self): - subprocess.Popen( - ["service nginx restart"], shell=True) + def DeleteServerFiles(self, upload_folder): + for f in os.listdir(upload_folder): + file_path = os.path.join(upload_folder, f) + try: + if os.path.isfile(file_path): + os.unlink(file_path) + except Exception as e: + logging.debug(e) + + def MoveServerFiles(self, file_ops): + response = 1 + try: + for fo in file_ops: + try: + shutil.move(fo['src_file'], fo['dest_file']) + except Exception as e: + logging.debug(e) + response = 0 + except Exception as e: + logging.debug(e) + return response + + def RestartNginx(self, package='default'): + if package == 'custom': + if self.nginx == 0: + p = subprocess.Popen(["nginx"], + stdout=subprocess.PIPE, + shell=True, + preexec_fn=os.setsid) + else: + for proc in psutil.process_iter(): + if proc.name() == "nginx": + proc.kill() + p = subprocess.Popen(["nginx"], + stdout=subprocess.PIPE, + shell=True, + preexec_fn=os.setsid) + self.nginx = p + else: + subprocess.Popen( + ["service nginx restart"], shell=True) + + def MakeDirs(self, path, prefix='/var/www/html/'): + try: + path = prefix + path.strip('/') + os.makedirs(path) + except Exception as e: + logging.debug(e) def ProcessAlerts(self, request, context): try: |