Merge "update verigraph"

1 files changed, 113 insertions, 60 deletions

diff --git a/verigraph/tester/test.py b/verigraph/tester/test.py
index ad0b82a..ee5f6a2 100644
--- a/verigraph/tester/test.py
+++ b/verigraph/tester/test.py
@@ -1,5 +1,4 @@
 #!/usr/bin/python
-
 ##############################################################################
 # Copyright (c) 2017 Politecnico di Torino and others.
 #
@@ -20,17 +19,25 @@ import json
 import getopt
 import os
 import subprocess
+import csv
+import datetime
+import time
+import re
+import argparse
+
 
 # Constants (change them, if appropriate)
 VERIGRAPH_PORT = "8080"
 TEST_CASES_DIR = "testcases"
-BASE_URL = "http://localhost:"+VERIGRAPH_PORT+"/verify/api/graphs/"
+BASE_URL = "http://localhost:"+VERIGRAPH_PORT+"/verigraph/api/graphs/"
 SCHEMA_FILE = "testcase_schema.json"
 
 # Variables
-success = 0
+fail = 0
 run = 0
 
+tests_number = 1
+
 # Utils
 def eprint(toPrint):
     sys.stdout.flush()
@@ -46,70 +53,116 @@ try:
 except ValueError:
     eprint("Invalid json schema (check your "+SCHEMA_FILE+")!\nExiting.")
     exit(-1)
+with open('result.csv', 'w') as file:
+    writer=csv.writer(file, delimiter=',', quoting=csv.QUOTE_MINIMAL, lineterminator='\n')
+    writer.writerow(['SRC','DST','GRAPH_ID','TEST_ID','RESULT','TIMES (ms)'])
+    # Iterate over .json files contained in the TEST_CASES_DIR
+    for i in os.listdir(TEST_CASES_DIR):
+        if i.endswith(".json"):
+            with open(TEST_CASES_DIR+os.path.sep+i) as data_file:
+                try:
+                    # Load json file (raise exception if malformed)
+                    data = json.load(data_file)
+
+                    # Validate input json against schema (raise exception if invalid)
+                    validate(data, schema)
+
+
+                    parser = argparse.ArgumentParser(description='iteration number')
+                    parser.add_argument('-iteration')
+                    args = vars(parser.parse_args())
+                    tests_number=args['iteration'][0]
+
+                    print("Test case ID: "+str(data["id"]))
+                    print("\tFILE NAME: "+i)
+                    print("\tTEST NAME: "+data["name"])
+                    print("\tTEST DESCRIPTION: "+data["description"])
+
+                    requested=data["policy_url_parameters"]
+                    results=data["results"]
+
+                    if(len(requested)==len(results)):
+
+                        run += 1
+                        # POST the graph
+                        r = requests.post(BASE_URL, json=data["graph"])
+                        if r.status_code == 201:
+                            graph_id = r.json()["id"]
+                            print("\tCreated Graph has ID " + str(graph_id) + " on VeriGraph")
+
+                            for i in range(len(requested)):
+                                print("request #: "+str(i))
+                                output=[]
+                                total_time=[]
+                                result=[]
+                                temp=requested[i]
+                                temp=re.split(r'[&=]', temp)
+                                output.append(temp[3])
+                                output.append(temp[5])
+
+                                #range(0, n)) where n is the number of tests to execute for every request (1 is the default)
+                                for n in range(0, int(tests_number)):
+                                    start_time=datetime.datetime.now()
+                                    # GET the policy verification result
+                                    policy = requests.get(BASE_URL+str(graph_id)+"/policy"+data["policy_url_parameters"][i])
+                                    total_time.append(int((datetime.datetime.now()-start_time).total_seconds() * 1000))
+
+                                    # Check the response
+                                    if policy.status_code == 200:
+                                        print("\tVerification result is " + policy.json()["result"])
+
+                                        # Check the result with the expected one
+                                        if policy.json()["result"] == data["results"][i]:
+                                            # SUCCESS
+                                            print("\t+++ Test passed +++")
+                                            if n==0:
+                                                result.append(policy.json()["result"])
 
-# Iterate over .json files contained in the TEST_CASES_DIR
-for i in os.listdir(TEST_CASES_DIR):
-    if i.endswith(".json"): 
-        with open(TEST_CASES_DIR+os.path.sep+i) as data_file:
-            try:
-                # Load json file (raise exception if malformed)
-                data = json.load(data_file)
-
-                # Validate input json against schema (raise exception if invalid)
-                validate(data, schema)
-
-                run += 1
-                print("Test case ID: "+str(data["id"]))
-                print("\tFILE NAME: "+i)
-                print("\tTEST NAME: "+data["name"])
-                print("\tTEST DESCRIPTION: "+data["description"])
-
-                # POST the graph
-                r = requests.post(BASE_URL, json=data["graph"])
-                if r.status_code == 201:
-                    graph_id = r.json()["id"]
-                    print("\tCreated Graph has ID " + str(graph_id) + " on VeriGraph")
-
-                    # GET the policy verification result
-                    policy = requests.get(BASE_URL+str(graph_id)+"/policy"+data["policy_url_parameters"])
-
-                    # Check the response
-                    if policy.status_code == 200:
-                        print("\tVerification result is " + policy.json()["result"])
-
-                        # Check the result with the expected one
-                        if policy.json()["result"] == data["result"]:
-                            # SUCCESS
-                            print("\t+++ Test passed +++")
-                            success += 1
-                        else:
-                            # FAIL
-                            eprint("\t[ERROR] Expected result was " + data["result"] + " but VeriGraph returned " + policy.json()["result"])
-                            print("\t--- Test failed ---")
+                                        else:
+                                            # FAIL
+                                            #eprint("\t[ERROR] Expected result was " + data["result"][i] + " but VeriGraph returned " + policy.json()["result"])
+                                            result.append("FAIL")
+                                            fail+=1
+                                            print("\t--- Test failed ---")
+                                    else:
+                                        print("\tVeriGraph returned an unexpected response -> " + str(policy.status_code), policy.reason)
+                                        fail+=1
+                                        print("\t--- Test failed ---")
+                                if(policy.status_code == 200):
+                                    output.append(graph_id)
+                                    output.append(data["id"])
+                                    output.append(result[0])
+                                    for j in range(len(total_time)):
+                                        output.append(total_time[j])
+                                    writer.writerow(output)
+                                else:
+                                    output.append("Error")
+                                    output.append("Policy code="+str(policy.status_code))
+                                    writer.writerow(output)
+                        print()
                     else:
-                        print("\tVeriGraph returned an unexpected response -> " + str(policy.status_code), policy.reason)
-                        print("\t--- Test failed ---")
-                print()
-            except ValueError:
-                print("Malformed json!\nSkipping "+i+" file")
-                print("\t--- Test failed ---")
-            except ValidationError:
-                print("Invalid json (see Schema file)!\nSkipping "+i+" file")
-                print("\t--- Test failed ---")
-            except ConnectionError:
-                print("Connection refused!")
-                print("\t--- Test failed ---")
-            except HTTPError:
-                print("HTTP error!")
-                print("\t--- Test failed ---")
+                        print("\tThe number of requests ("+str(len(requested))+")is not equal to the numer of results ("+str(len(results))+")")
+                        print("\tPlease check the testcases file")
+                except ValueError:
+                    print("Malformed json!\nSkipping "+i+" file")
+                    print("\t--- Test failed ---")
+                except ValidationError:
+                    print("Invalid json (see Schema file)!\nSkipping "+i+" file")
+                    print("\t--- Test failed ---")
+                except ConnectionError:
+                    print("Connection refused!")
+                    print("\t--- Test failed ---")
+                except HTTPError:
+                    print("HTTP error!")
+                    print("\t--- Test failed ---")
 
 # Final output
 print("\nTest run = "+str(run))
-print("Test succeded = "+str(success))
 if run != 0:
-    if success != run:
+    if fail != 0:
         print("\n --- Some tests failed. See the output. ---")
     else:
         print("\n +++ All tests passed +++")
 else:
-    print("\n\n +++ 0 tests executed +++")
\ No newline at end of file
+    print("\n\n +++ 0 tests executed +++")
+    
\ No newline at end of file