blob: ad0b82a29026ecb7ddd3e3a1b68f74f4c3e3e5aa (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
|
#!/usr/bin/python
##############################################################################
# Copyright (c) 2017 Politecnico di Torino and others.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
from __future__ import print_function
from jsonschema import validate
from pprint import pprint
import sys
import requests
from requests.exceptions import *
from jsonschema.exceptions import *
import json
import getopt
import os
import subprocess
# Constants (change them, if appropriate)
VERIGRAPH_PORT = "8080"
TEST_CASES_DIR = "testcases"
BASE_URL = "http://localhost:"+VERIGRAPH_PORT+"/verify/api/graphs/"
SCHEMA_FILE = "testcase_schema.json"
# Variables
success = 0
run = 0
# Utils
def eprint(toPrint):
sys.stdout.flush()
print(toPrint, file=sys.stderr)
sys.stderr.flush()
# Print PYTHON ver
print("PYTHON " + sys.version)
# Loading schema file
try:
schema = json.load(open(SCHEMA_FILE))
except ValueError:
eprint("Invalid json schema (check your "+SCHEMA_FILE+")!\nExiting.")
exit(-1)
# Iterate over .json files contained in the TEST_CASES_DIR
for i in os.listdir(TEST_CASES_DIR):
if i.endswith(".json"):
with open(TEST_CASES_DIR+os.path.sep+i) as data_file:
try:
# Load json file (raise exception if malformed)
data = json.load(data_file)
# Validate input json against schema (raise exception if invalid)
validate(data, schema)
run += 1
print("Test case ID: "+str(data["id"]))
print("\tFILE NAME: "+i)
print("\tTEST NAME: "+data["name"])
print("\tTEST DESCRIPTION: "+data["description"])
# POST the graph
r = requests.post(BASE_URL, json=data["graph"])
if r.status_code == 201:
graph_id = r.json()["id"]
print("\tCreated Graph has ID " + str(graph_id) + " on VeriGraph")
# GET the policy verification result
policy = requests.get(BASE_URL+str(graph_id)+"/policy"+data["policy_url_parameters"])
# Check the response
if policy.status_code == 200:
print("\tVerification result is " + policy.json()["result"])
# Check the result with the expected one
if policy.json()["result"] == data["result"]:
# SUCCESS
print("\t+++ Test passed +++")
success += 1
else:
# FAIL
eprint("\t[ERROR] Expected result was " + data["result"] + " but VeriGraph returned " + policy.json()["result"])
print("\t--- Test failed ---")
else:
print("\tVeriGraph returned an unexpected response -> " + str(policy.status_code), policy.reason)
print("\t--- Test failed ---")
print()
except ValueError:
print("Malformed json!\nSkipping "+i+" file")
print("\t--- Test failed ---")
except ValidationError:
print("Invalid json (see Schema file)!\nSkipping "+i+" file")
print("\t--- Test failed ---")
except ConnectionError:
print("Connection refused!")
print("\t--- Test failed ---")
except HTTPError:
print("HTTP error!")
print("\t--- Test failed ---")
# Final output
print("\nTest run = "+str(run))
print("Test succeded = "+str(success))
if run != 0:
if success != run:
print("\n --- Some tests failed. See the output. ---")
else:
print("\n +++ All tests passed +++")
else:
print("\n\n +++ 0 tests executed +++")
|
