Merge "Add file upload/download and configure URL paths"

6 files changed, 296 insertions, 67 deletions

diff --git a/samples/services/nginx/docker/grpc/nginx.proto b/samples/services/nginx/docker/grpc/nginx.proto
index 3779a82..8255719 100644
--- a/samples/services/nginx/docker/grpc/nginx.proto
+++ b/samples/services/nginx/docker/grpc/nginx.proto
@@ -36,6 +36,10 @@ message ConfigServer {
   string server_name = 2;
   string site_root = 3;
   string site_index = 4;
+  string upload_path_config = 5;
+  string locations = 6;
+  string upload_path_test = 7;
+  string files = 8;
 }
 
 message ConfigLB {
diff --git a/samples/services/nginx/docker/grpc/nginx_client.py b/samples/services/nginx/docker/grpc/nginx_client.py
index 7a07464..e4ddb2a 100644
--- a/samples/services/nginx/docker/grpc/nginx_client.py
+++ b/samples/services/nginx/docker/grpc/nginx_client.py
@@ -66,9 +66,21 @@ def modify_proxy(stub):
 
 
 def modify_server(stub):
+    loc = []
+    val = {}
+    val['uri_match'] = "/test"
+    val['directive'] = "try_files $uri @default1"
+    val['path'] = "/test"
+    loc.append(val)
+    locations = pickle.dumps(loc)
+    files = pickle.dumps([])
     response = stub.ModifyServer(nginx_pb2.ConfigServer(
-            server_port='9180', server_name='clover-server',
-            site_root='/var/www/html', site_index='index.nginx-debian.html'))
+            server_port='9180', server_name='clover-server1',
+            upload_path_config='/upload',
+            upload_path_test='/upload_test',
+            locations=locations,
+            files=files,
+            site_root='/var/www/html', site_index='index.html'))
     print(response.message)
 
 
diff --git a/samples/services/nginx/docker/grpc/nginx_grpc_server.py b/samples/services/nginx/docker/grpc/nginx_grpc_server.py
index 1dfe708..5ad64b5 100644
--- a/samples/services/nginx/docker/grpc/nginx_grpc_server.py
+++ b/samples/services/nginx/docker/grpc/nginx_grpc_server.py
@@ -8,9 +8,12 @@
 
 from concurrent import futures
 import time
+import os
 import sys
 import grpc
 import subprocess
+import psutil
+import shutil
 import pickle
 import logging
 import nginx_pb2
@@ -28,23 +31,32 @@ class Controller(nginx_pb2_grpc.ControllerServicer):
         logging.basicConfig(filename='nginx.log', level=logging.DEBUG)
         self.service_type = service_type
         self.out_file = '/etc/nginx/nginx.conf'
-        # self.out_file = 'testfile'
+        self.nginx = 0
         if service_type == "proxy":
-            # self.template_file = 'templates/proxy.template'
             self.template_file = '/grpc/templates/proxy.template'
             self.ModifyProxy(nginx_pb2.ConfigProxy(
                 server_port='9180', server_name='proxy-access-control',
                 location_path='/', proxy_path='http://http-lb:9180',
                 mirror_path='http://snort-ids:80'), "")
         if service_type == "server":
-            # self.template_file = 'templates/server.template'
             self.template_file = '/grpc/templates/server.template'
+            loc = []
+            val = {}
+            val['uri_match'] = "/test"
+            val['directive'] = "try_files $uri @default1"
+            val['path'] = "/test"
+            loc.append(val)
+            locations = pickle.dumps(loc)
+            files = pickle.dumps([])
             self.ModifyServer(nginx_pb2.ConfigServer(
                 server_port='9180', server_name='clover-server',
                 site_root='/var/www/html',
-                site_index='index.nginx-debian.html'), "")
+                upload_path_config='/upload',
+                upload_path_test='/upload_test',
+                locations=locations,
+                files=files,
+                site_index='index.html'), "")
         if service_type == "lb":
-            # self.template_file = 'templates/lb.template'
             self.template_file = '/grpc/templates/lb.template'
             slb_list = pickle.dumps(
                     ['clover-server1:9180', 'clover-server2:9180',
@@ -76,18 +88,54 @@ class Controller(nginx_pb2_grpc.ControllerServicer):
 
     def ModifyServer(self, r, context):
         try:
+            locations = pickle.loads(r.locations)
+            # Generate nginx config
             with open(self.template_file) as f:
                 tmpl = Template(f.read())
             output = tmpl.render(
                 server_port=r.server_port,
                 server_name=r.server_name,
                 site_root=r.site_root,
-                site_index=r.site_index
+                site_index=r.site_index,
+                upload_path_config=r.upload_path_config,
+                upload_path_test=r.upload_path_test,
+                locations=locations
             )
             with open(self.out_file, "wb") as fh:
                 fh.write(output)
+
+            # Make dirs for locations
+            for l in locations:
+                self.MakeDirs(l['path'])
+
+            # Generate upload form for config
+            template_upload = '/grpc/templates/upload_form.template'
+            out_file = r.site_root + '/' + r.site_index
+            with open(template_upload) as f:
+                tmpl = Template(f.read())
+            output = tmpl.render(
+                upload_path=r.upload_path_config
+            )
+            with open(out_file, "wb") as fh:
+                fh.write(output)
+
+            # Generate upload form for test
+            template_upload = '/grpc/templates/upload_form.template'
+            out_file = r.site_root + '/' + 'upload_test.html'
+            with open(template_upload) as f:
+                tmpl = Template(f.read())
+            output = tmpl.render(
+                upload_path=r.upload_path_test
+            )
+            with open(out_file, "wb") as fh:
+                fh.write(output)
+
             msg = "Modified nginx config"
-            self.RestartNginx()
+            self.RestartNginx('custom')
+
+            # Move files that have been uploaded
+            file_ops = pickle.loads(r.files)
+            self.MoveServerFiles(file_ops)
         except Exception as e:
             logging.debug(e)
             msg = "Failed to modify nginx config"
@@ -113,9 +161,54 @@ class Controller(nginx_pb2_grpc.ControllerServicer):
             msg = "Failed to modify nginx config"
         return nginx_pb2.NginxReply(message=msg)
 
-    def RestartNginx(self):
-        subprocess.Popen(
-                  ["service nginx restart"], shell=True)
+    def DeleteServerFiles(self, upload_folder):
+        for f in os.listdir(upload_folder):
+            file_path = os.path.join(upload_folder, f)
+            try:
+                if os.path.isfile(file_path):
+                    os.unlink(file_path)
+            except Exception as e:
+                logging.debug(e)
+
+    def MoveServerFiles(self, file_ops):
+        response = 1
+        try:
+            for fo in file_ops:
+                try:
+                    shutil.move(fo['src_file'], fo['dest_file'])
+                except Exception as e:
+                    logging.debug(e)
+                    response = 0
+        except Exception as e:
+            logging.debug(e)
+        return response
+
+    def RestartNginx(self, package='default'):
+        if package == 'custom':
+            if self.nginx == 0:
+                p = subprocess.Popen(["nginx"],
+                                     stdout=subprocess.PIPE,
+                                     shell=True,
+                                     preexec_fn=os.setsid)
+            else:
+                for proc in psutil.process_iter():
+                    if proc.name() == "nginx":
+                        proc.kill()
+                p = subprocess.Popen(["nginx"],
+                                     stdout=subprocess.PIPE,
+                                     shell=True,
+                                     preexec_fn=os.setsid)
+            self.nginx = p
+        else:
+            subprocess.Popen(
+                      ["service nginx restart"], shell=True)
+
+    def MakeDirs(self, path, prefix='/var/www/html/'):
+        try:
+            path = prefix + path.strip('/')
+            os.makedirs(path)
+        except Exception as e:
+            logging.debug(e)
 
     def ProcessAlerts(self, request, context):
         try:
diff --git a/samples/services/nginx/docker/grpc/nginx_pb2.py b/samples/services/nginx/docker/grpc/nginx_pb2.py
index 3600b6d..5665fc4 100644
--- a/samples/services/nginx/docker/grpc/nginx_pb2.py
+++ b/samples/services/nginx/docker/grpc/nginx_pb2.py
@@ -19,7 +19,7 @@ DESCRIPTOR = _descriptor.FileDescriptor(
   name='nginx.proto',
   package='nginx',
   syntax='proto3',
-  serialized_pb=_b('\n\x0bnginx.proto\x12\x05nginx\"3\n\x0c\x41lertMessage\x12\x10\n\x08\x65vent_id\x18\x01 \x01(\t\x12\x11\n\tredis_key\x18\x02 \x01(\t\"w\n\x0b\x43onfigProxy\x12\x13\n\x0bserver_port\x18\x01 \x01(\t\x12\x13\n\x0bserver_name\x18\x02 \x01(\t\x12\x15\n\rlocation_path\x18\x03 \x01(\t\x12\x12\n\nproxy_path\x18\x04 \x01(\t\x12\x13\n\x0bmirror_path\x18\x05 \x01(\t\"_\n\x0c\x43onfigServer\x12\x13\n\x0bserver_port\x18\x01 \x01(\t\x12\x13\n\x0bserver_name\x18\x02 \x01(\t\x12\x11\n\tsite_root\x18\x03 \x01(\t\x12\x12\n\nsite_index\x18\x04 \x01(\t\"j\n\x08\x43onfigLB\x12\x13\n\x0bserver_port\x18\x01 \x01(\t\x12\x13\n\x0bserver_name\x18\x02 \x01(\t\x12\x10\n\x08slb_list\x18\x03 \x01(\t\x12\x11\n\tslb_group\x18\x04 \x01(\t\x12\x0f\n\x07lb_path\x18\x05 \x01(\t\"\x1d\n\nNginxReply\x12\x0f\n\x07message\x18\x01 \x01(\t2\xeb\x01\n\nController\x12\x36\n\x0bModifyProxy\x12\x12.nginx.ConfigProxy\x1a\x11.nginx.NginxReply\"\x00\x12\x38\n\x0cModifyServer\x12\x13.nginx.ConfigServer\x1a\x11.nginx.NginxReply\"\x00\x12\x30\n\x08ModifyLB\x12\x0f.nginx.ConfigLB\x1a\x11.nginx.NginxReply\"\x00\x12\x39\n\rProcessAlerts\x12\x13.nginx.AlertMessage\x1a\x11.nginx.NginxReply\"\x00\x62\x06proto3')
+  serialized_pb=_b('\n\x0bnginx.proto\x12\x05nginx\"3\n\x0c\x41lertMessage\x12\x10\n\x08\x65vent_id\x18\x01 \x01(\t\x12\x11\n\tredis_key\x18\x02 \x01(\t\"w\n\x0b\x43onfigProxy\x12\x13\n\x0bserver_port\x18\x01 \x01(\t\x12\x13\n\x0bserver_name\x18\x02 \x01(\t\x12\x15\n\rlocation_path\x18\x03 \x01(\t\x12\x12\n\nproxy_path\x18\x04 \x01(\t\x12\x13\n\x0bmirror_path\x18\x05 \x01(\t\"\xb7\x01\n\x0c\x43onfigServer\x12\x13\n\x0bserver_port\x18\x01 \x01(\t\x12\x13\n\x0bserver_name\x18\x02 \x01(\t\x12\x11\n\tsite_root\x18\x03 \x01(\t\x12\x12\n\nsite_index\x18\x04 \x01(\t\x12\x1a\n\x12upload_path_config\x18\x05 \x01(\t\x12\x11\n\tlocations\x18\x06 \x01(\t\x12\x18\n\x10upload_path_test\x18\x07 \x01(\t\x12\r\n\x05\x66iles\x18\x08 \x01(\t\"j\n\x08\x43onfigLB\x12\x13\n\x0bserver_port\x18\x01 \x01(\t\x12\x13\n\x0bserver_name\x18\x02 \x01(\t\x12\x10\n\x08slb_list\x18\x03 \x01(\t\x12\x11\n\tslb_group\x18\x04 \x01(\t\x12\x0f\n\x07lb_path\x18\x05 \x01(\t\"\x1d\n\nNginxReply\x12\x0f\n\x07message\x18\x01 \x01(\t2\xeb\x01\n\nController\x12\x36\n\x0bModifyProxy\x12\x12.nginx.ConfigProxy\x1a\x11.nginx.NginxReply\"\x00\x12\x38\n\x0cModifyServer\x12\x13.nginx.ConfigServer\x1a\x11.nginx.NginxReply\"\x00\x12\x30\n\x08ModifyLB\x12\x0f.nginx.ConfigLB\x1a\x11.nginx.NginxReply\"\x00\x12\x39\n\rProcessAlerts\x12\x13.nginx.AlertMessage\x1a\x11.nginx.NginxReply\"\x00\x62\x06proto3')
 )
 
 
@@ -157,6 +157,34 @@ _CONFIGSERVER = _descriptor.Descriptor(
       message_type=None, enum_type=None, containing_type=None,
       is_extension=False, extension_scope=None,
       options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='upload_path_config', full_name='nginx.ConfigServer.upload_path_config', index=4,
+      number=5, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='locations', full_name='nginx.ConfigServer.locations', index=5,
+      number=6, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='upload_path_test', full_name='nginx.ConfigServer.upload_path_test', index=6,
+      number=7, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None, file=DESCRIPTOR),
+    _descriptor.FieldDescriptor(
+      name='files', full_name='nginx.ConfigServer.files', index=7,
+      number=8, type=9, cpp_type=9, label=1,
+      has_default_value=False, default_value=_b("").decode('utf-8'),
+      message_type=None, enum_type=None, containing_type=None,
+      is_extension=False, extension_scope=None,
+      options=None, file=DESCRIPTOR),
   ],
   extensions=[
   ],
@@ -169,8 +197,8 @@ _CONFIGSERVER = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=196,
-  serialized_end=291,
+  serialized_start=197,
+  serialized_end=380,
 )
 
 
@@ -228,8 +256,8 @@ _CONFIGLB = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=293,
-  serialized_end=399,
+  serialized_start=382,
+  serialized_end=488,
 )
 
 
@@ -259,8 +287,8 @@ _NGINXREPLY = _descriptor.Descriptor(
   extension_ranges=[],
   oneofs=[
   ],
-  serialized_start=401,
-  serialized_end=430,
+  serialized_start=490,
+  serialized_end=519,
 )
 
 DESCRIPTOR.message_types_by_name['AlertMessage'] = _ALERTMESSAGE
@@ -313,8 +341,8 @@ _CONTROLLER = _descriptor.ServiceDescriptor(
   file=DESCRIPTOR,
   index=0,
   options=None,
-  serialized_start=433,
-  serialized_end=668,
+  serialized_start=522,
+  serialized_end=757,
   methods=[
   _descriptor.MethodDescriptor(
     name='ModifyProxy',
diff --git a/samples/services/nginx/docker/grpc/templates/server.template b/samples/services/nginx/docker/grpc/templates/server.template
index b5f8f1f..c1582fa 100644
--- a/samples/services/nginx/docker/grpc/templates/server.template
+++ b/samples/services/nginx/docker/grpc/templates/server.template
@@ -3,69 +3,142 @@ worker_processes auto;
 pid /run/nginx.pid;
 
 events {
-	worker_connections 768;
-	# multi_accept on;
+    worker_connections 768;
+    # multi_accept on;
 }
 
 http {
 
-	##
-	# Basic Settings
-	##
+    ##
+    # Basic Settings
+    ##
 
-	sendfile on;
-	tcp_nopush on;
-	tcp_nodelay on;
-	keepalive_timeout 65;
-	types_hash_max_size 2048;
-	# server_tokens off;
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    keepalive_timeout 65;
+    types_hash_max_size 2048;
+    # server_tokens off;
 
 
-	include /etc/nginx/mime.types;
-	default_type application/octet-stream;
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
 
-	##
-	# SSL Settings
-	##
+    ##
+    # SSL Settings
+    ##
 
-	ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
-	ssl_prefer_server_ciphers on;
+    ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
+    ssl_prefer_server_ciphers on;
 
-	##
-	# Logging Settings
-	##
+    ##
+    # Logging Settings
+    ##
 
-	access_log /var/log/nginx/access.log;
-	error_log /var/log/nginx/error.log;
+    access_log /var/log/nginx/access.log;
+    error_log /var/log/nginx/error.log;
 
-	##
-	# Gzip Settings
-	##
+    ##
+    # Gzip Settings
+    ##
 
-	gzip on;
-	gzip_disable "msie6";
+    gzip on;
+    gzip_disable "msie6";
 
-	# gzip_vary on;
-	# gzip_proxied any;
-	# gzip_comp_level 6;
-	# gzip_buffers 16 8k;
-	# gzip_http_version 1.1;
-	# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
+    # gzip_vary on;
+    # gzip_proxied any;
+    # gzip_comp_level 6;
+    # gzip_buffers 16 8k;
+    # gzip_http_version 1.1;
+    # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
 
-	##
-	# Virtual Host Configs
-	##
+    ##
+    # Virtual Host Configs
+    ##
 
-	include /etc/nginx/conf.d/*.conf;
-	#include /etc/nginx/sites-enabled/*;
+    include /etc/nginx/conf.d/*.conf;
+    #include /etc/nginx/sites-enabled/*;
 
-    server {
-        listen {{ server_port }};
-        server_name {{ server_name }};
+        server {
+            listen {{ server_port }};
+            server_name {{ server_name }};
+            client_max_body_size 500m;
 
-        root {{ site_root }};
-        index {{ site_index }};
-    }
+            root {{ site_root }};
+            index {{ site_index }};
 
-}
+            {%- for l in locations %}
+
+            location {{ l["uri_match"] }} {
+                {{ l["directive"] }};
+            }
+
+            {%- endfor %}
+
+            location @default1 {
+                 index ../index.html;
+            }
+
+            location @default2 {
+                 index ../../index.html;
+            }
+
+            location @default3 {
+                 index ../../../index.html;
+            }
+
+            # Use to upload files for download
+            location {{ upload_path_config }} {
+                # Pass altered request body to this location
+                upload_pass   @return_config;
+
+                # Store files to this directory
+                upload_resumable on;
+                upload_state_store /tmp/state;
+                upload_store {{ site_root }}{{ upload_path_config }};
+
+                # Allow uploaded files to be read only by user
+                #upload_store_access user:r;
+
+                # Set specified fields in request body
+                upload_set_form_field $upload_field_name.name "$upload_file_name";
+                upload_set_form_field $upload_field_name.content_type "$upload_content_type";
+                upload_set_form_field $upload_field_name.path "$upload_tmp_path";
+                upload_set_form_field $upload_field_name.server "$server_name";
+
+                # Inform backend about hash and size of a file
+                upload_aggregate_form_field "$upload_field_name.md5" "$upload_file_md5";
+                upload_aggregate_form_field "$upload_field_name.size" "$upload_file_size";
 
+                upload_pass_form_field "^submit$|^description$";
+
+                upload_cleanup 400 404 499 500-505;
+            }
+
+            location @return_config {
+                proxy_pass http://clover-controller:80;
+                proxy_http_version 1.1;
+            }
+
+            # Use to upload files for performance testing
+            location {{ upload_path_test }} {
+
+                upload_pass   @return_path;
+
+                # Store files to this directory
+                upload_resumable on;
+                upload_state_store /tmp/state;
+                upload_store {{ site_root }}{{ upload_path_test }};
+
+                upload_pass_form_field "^submit$|^description$";
+
+                upload_cleanup 400 404 499 500-505;
+            }
+
+            location @return_path {
+                return 204;
+            }
+
+
+        }
+}
diff --git a/samples/services/nginx/docker/grpc/templates/upload_form.template b/samples/services/nginx/docker/grpc/templates/upload_form.template
new file mode 100644
index 0000000..60f3967
--- /dev/null
+++ b/samples/services/nginx/docker/grpc/templates/upload_form.template
@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Test upload</title>
+  </head>
+  <body>
+    <h2>Select files to upload</h2>
+    <form name="upload" method="POST" enctype="multipart/form-data" action="{{ upload_path }}">
+      <input type="file" name="file1"><br>
+      <input type="file" name="file2"><br>
+      <input type="file" name="file3"><br>
+      <input type="file" name="file4"><br>
+      <input type="file" name="file5"><br>
+      <input type="file" name="file6"><br>
+      <input type="submit" name="submit" value="Upload">
+      <input type="hidden" name="test" value="value">
+    </form>
+  </body>
+</html>