Add file upload/download and configure URL paths

- Compile nginx from source in order to employ additional modules
- Add nginx-upload-module for high performance file upload that
avoids the need for file copies with a web application.
- File upload allows for placement of files for file download
for performance benchmarking.
- File upload can also be used directly for bi-directional throughput
testing having emulated clients upload files while file downloads
simultaneously occur.
- Nginx file upload stores files with hash to avoid conflicting
file names. Upload block in nginx config is configured to send
REST message to clover-controller with file metadata (original
filename, size, etc.) clover-controller will be responsible for
modifying the hashed filename and placing in a target directory
within an nginx server.
- Build also adds nginx-rtmp module to act as streaming media server
L7 loader will be extended to fetch streaming files from RTMP
servers.

- Add ability to create directories in server site root
and create the location directive(s) in nginx configuration
- Separated upload for configuration (download
files in various paths) from upload for testing (upload
to create bi-directional session throughput)
- Upload for testing does not sent upload metadata to
clover-controller
- Added ability to move upload files to file folders in the nginx site
root to use for download
- Delete files in upload folder
- Fixed issue with 426 Upgrade Required error message
when upload module sends upload metadata to clover-controller
- Added server name to metadata sent to clover-controller

Change-Id: Ib4cf6240f92360b82f378c062675f4fdaa19ca93
Signed-off-by: Eddie Arrage <eddie.arrage@huawei.com>

1 files changed, 119 insertions, 46 deletions

diff --git a/samples/services/nginx/docker/grpc/templates/server.template b/samples/services/nginx/docker/grpc/templates/server.template
index b5f8f1f..c1582fa 100644
--- a/samples/services/nginx/docker/grpc/templates/server.template
+++ b/samples/services/nginx/docker/grpc/templates/server.template
@@ -3,69 +3,142 @@ worker_processes auto;
 pid /run/nginx.pid;
 
 events {
-	worker_connections 768;
-	# multi_accept on;
+    worker_connections 768;
+    # multi_accept on;
 }
 
 http {
 
-	##
-	# Basic Settings
-	##
+    ##
+    # Basic Settings
+    ##
 
-	sendfile on;
-	tcp_nopush on;
-	tcp_nodelay on;
-	keepalive_timeout 65;
-	types_hash_max_size 2048;
-	# server_tokens off;
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    keepalive_timeout 65;
+    types_hash_max_size 2048;
+    # server_tokens off;
 
 
-	include /etc/nginx/mime.types;
-	default_type application/octet-stream;
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
 
-	##
-	# SSL Settings
-	##
+    ##
+    # SSL Settings
+    ##
 
-	ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
-	ssl_prefer_server_ciphers on;
+    ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
+    ssl_prefer_server_ciphers on;
 
-	##
-	# Logging Settings
-	##
+    ##
+    # Logging Settings
+    ##
 
-	access_log /var/log/nginx/access.log;
-	error_log /var/log/nginx/error.log;
+    access_log /var/log/nginx/access.log;
+    error_log /var/log/nginx/error.log;
 
-	##
-	# Gzip Settings
-	##
+    ##
+    # Gzip Settings
+    ##
 
-	gzip on;
-	gzip_disable "msie6";
+    gzip on;
+    gzip_disable "msie6";
 
-	# gzip_vary on;
-	# gzip_proxied any;
-	# gzip_comp_level 6;
-	# gzip_buffers 16 8k;
-	# gzip_http_version 1.1;
-	# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
+    # gzip_vary on;
+    # gzip_proxied any;
+    # gzip_comp_level 6;
+    # gzip_buffers 16 8k;
+    # gzip_http_version 1.1;
+    # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
 
-	##
-	# Virtual Host Configs
-	##
+    ##
+    # Virtual Host Configs
+    ##
 
-	include /etc/nginx/conf.d/*.conf;
-	#include /etc/nginx/sites-enabled/*;
+    include /etc/nginx/conf.d/*.conf;
+    #include /etc/nginx/sites-enabled/*;
 
-    server {
-        listen {{ server_port }};
-        server_name {{ server_name }};
+        server {
+            listen {{ server_port }};
+            server_name {{ server_name }};
+            client_max_body_size 500m;
 
-        root {{ site_root }};
-        index {{ site_index }};
-    }
+            root {{ site_root }};
+            index {{ site_index }};
 
-}
+            {%- for l in locations %}
+
+            location {{ l["uri_match"] }} {
+                {{ l["directive"] }};
+            }
+
+            {%- endfor %}
+
+            location @default1 {
+                 index ../index.html;
+            }
+
+            location @default2 {
+                 index ../../index.html;
+            }
+
+            location @default3 {
+                 index ../../../index.html;
+            }
+
+            # Use to upload files for download
+            location {{ upload_path_config }} {
+                # Pass altered request body to this location
+                upload_pass   @return_config;
+
+                # Store files to this directory
+                upload_resumable on;
+                upload_state_store /tmp/state;
+                upload_store {{ site_root }}{{ upload_path_config }};
+
+                # Allow uploaded files to be read only by user
+                #upload_store_access user:r;
+
+                # Set specified fields in request body
+                upload_set_form_field $upload_field_name.name "$upload_file_name";
+                upload_set_form_field $upload_field_name.content_type "$upload_content_type";
+                upload_set_form_field $upload_field_name.path "$upload_tmp_path";
+                upload_set_form_field $upload_field_name.server "$server_name";
+
+                # Inform backend about hash and size of a file
+                upload_aggregate_form_field "$upload_field_name.md5" "$upload_file_md5";
+                upload_aggregate_form_field "$upload_field_name.size" "$upload_file_size";
 
+                upload_pass_form_field "^submit$|^description$";
+
+                upload_cleanup 400 404 499 500-505;
+            }
+
+            location @return_config {
+                proxy_pass http://clover-controller:80;
+                proxy_http_version 1.1;
+            }
+
+            # Use to upload files for performance testing
+            location {{ upload_path_test }} {
+
+                upload_pass   @return_path;
+
+                # Store files to this directory
+                upload_resumable on;
+                upload_state_store /tmp/state;
+                upload_store {{ site_root }}{{ upload_path_test }};
+
+                upload_pass_form_field "^submit$|^description$";
+
+                upload_cleanup 400 404 499 500-505;
+            }
+
+            location @return_path {
+                return 204;
+            }
+
+
+        }
+}