diff options
Diffstat (limited to 'testing-scheduler/server/src')
16 files changed, 1138 insertions, 0 deletions
diff --git a/testing-scheduler/server/src/__init__.py b/testing-scheduler/server/src/__init__.py new file mode 100644 index 00000000..e8198009 --- /dev/null +++ b/testing-scheduler/server/src/__init__.py @@ -0,0 +1,8 @@ +##############################################################################
+# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
diff --git a/testing-scheduler/server/src/conductor_processor/__init__.py b/testing-scheduler/server/src/conductor_processor/__init__.py new file mode 100644 index 00000000..bb02be17 --- /dev/null +++ b/testing-scheduler/server/src/conductor_processor/__init__.py @@ -0,0 +1,8 @@ +##############################################################################
+# Copyright (c) 2018 Huawei Technologies Co.,Ltd. and others
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
diff --git a/testing-scheduler/server/src/conductor_processor/defaultTaskFile.json b/testing-scheduler/server/src/conductor_processor/defaultTaskFile.json new file mode 100644 index 00000000..a98a5819 --- /dev/null +++ b/testing-scheduler/server/src/conductor_processor/defaultTaskFile.json @@ -0,0 +1,9 @@ +{
+ "name": "",
+ "retryCount": 6,
+ "timeOutSeconds": 1200,
+ "timeOutPolicy": "TIME_OUT_WF",
+ "retryLogic": "FIXED",
+ "retryDelaySeconds": 3,
+ "responseTimeOutSeconds": 3600
+}
\ No newline at end of file diff --git a/testing-scheduler/server/src/conductor_processor/defaultWorkflowFile.json b/testing-scheduler/server/src/conductor_processor/defaultWorkflowFile.json new file mode 100644 index 00000000..8f6251c0 --- /dev/null +++ b/testing-scheduler/server/src/conductor_processor/defaultWorkflowFile.json @@ -0,0 +1,24 @@ +{
+ "name": "workflow_demo_05",
+ "description": "run a workflow of yardstick test service",
+ "version": 1,
+ "tasks": [
+ {
+ "name": "http_yardstick_test",
+ "taskReferenceName": "ping_test",
+ "inputParameters": {
+ "http_request": {
+ "uri": "http://192.168.199.105:8080/greet",
+ "method": "GET"
+ }
+ },
+ "type": "HTTP"
+ }
+ ],
+ "outputParameters": {
+ "header": "${ping_test.output.response.headers}",
+ "response": "${ping_test.output.response.body}",
+ "status": "${ping_test.output.response.statusCode}"
+ },
+ "schemaVersion": 2
+}
\ No newline at end of file diff --git a/testing-scheduler/server/src/conductor_processor/task.py b/testing-scheduler/server/src/conductor_processor/task.py new file mode 100644 index 00000000..6f25aef8 --- /dev/null +++ b/testing-scheduler/server/src/conductor_processor/task.py @@ -0,0 +1,28 @@ +##############################################################################
+# Copyright (c) 2018 Huawei Technologies Co.,Ltd. and others
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import json
+import os
+
+
+class TaskFile(object):
+ def __init__(self, taskName='task_0'):
+ self._defaultConfFile = self._getFilePath("defaultTaskFile.json")
+ with open(self._defaultConfFile) as defaultConf:
+ self._jsonObj = json.load(defaultConf)
+ self._jsonObj['name'] = taskName
+
+ def generateFromStep(self, stepObject):
+ self._jsonObj['name'] = stepObject.getName()
+ print "taskFile:", self._jsonObj['name']
+ return self._jsonObj
+
+ def _getFilePath(self, fileName):
+ dirPath = os.path.dirname(os.path.realpath(__file__))
+ return os.path.join(dirPath, fileName)
diff --git a/testing-scheduler/server/src/conductor_processor/workflow.py b/testing-scheduler/server/src/conductor_processor/workflow.py new file mode 100644 index 00000000..19f0896c --- /dev/null +++ b/testing-scheduler/server/src/conductor_processor/workflow.py @@ -0,0 +1,243 @@ +##############################################################################
+# Copyright (c) 2018 Huawei Technologies Co.,Ltd. and others
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import random
+import collections
+import re
+from src.conductor_processor.task import TaskFile
+
+
+class WorkflowFile(object):
+ def __init__(self, name):
+ self._name = "workflow_" + name + "(%s)" % getRandString(10)
+ self._description = ''
+ self._version = 1
+ self._schemaVersion = 2
+ self._tasks = []
+ self._outputParameters = {}
+
+ def getDict(self):
+ d = collections.OrderedDict()
+ d['name'] = self._name
+ d['description'] = self._description
+ d['version'] = self._version
+ d['schemaVersion'] = self._schemaVersion
+ d['tasks'] = self._tasks
+ d['outputParameters'] = self._outputParameters
+
+ return d
+
+ def generateMetaData(self, flowList, stepObjArr):
+ flowParser = FlowParser(flowList, stepObjArr)
+ self._tasks, taskMetaList = flowParser.parseMainFlow()
+ normalTasks = flowParser.getNormalTaskList()
+ for normalTask in normalTasks:
+ taskName = normalTask['name']
+ referenceName = normalTask['taskReferenceName']
+ self._outputParameters["%s(%s)" % (taskName, referenceName)] = \
+ "${%s.output.response.body}" % referenceName
+ return self.getDict(), taskMetaList
+
+
+class FlowParser(object):
+ def __init__(self, flowList, stepObjArr):
+ self._mainFlow = {}
+ self._subFlowDict = {}
+ self._stepObjArr = stepObjArr
+ self._normalTasks = []
+ for flow in flowList:
+ if flow['name'] == "main":
+ self._mainFlow = flow
+ else:
+ self._subFlowDict[flow['name']] = flow
+
+ def parseMainFlow(self):
+ return self.parseOrderList(self._mainFlow['orders'], self._stepObjArr)
+
+ def parse(self, obj, stepObjArr):
+ if isinstance(obj, str):
+ return self.parseFlow(obj, stepObjArr)
+ else:
+ return self.parseOrderList(obj, stepObjArr)
+
+ def parseFlow(self, flowName, stepObjArr):
+ orderList = self._subFlowDict[flowName]['orders']
+ return self.parseOrderList(orderList, stepObjArr)
+
+ def parseOrderList(self, orderList, stepObjArr):
+ tasks = []
+ taskMetaAllList = []
+ for order in orderList:
+ if order['type'] == "normal":
+ genTask = NormalTask(order, stepObjArr, self)
+ self._normalTasks.append(genTask)
+ elif order['type'] == "switch":
+ genTask = SwitchTask(order, stepObjArr, self)
+ elif order['type'] == "parallel":
+ genTask = ParallelTask(order, stepObjArr, self)
+ tasks.append(genTask.getDict())
+
+ if order['type'] == "parallel":
+ joinTask = genTask.getJoinTask()
+ tasks.append(joinTask.getDict())
+
+ taskMetaList = genTask.getTaskMetaList()
+ if taskMetaList is not None:
+ taskMetaAllList.extend(taskMetaList)
+ return tasks, taskMetaAllList
+
+ def getNormalTaskList(self):
+ normalTasksDict = []
+ for normalTask in self._normalTasks:
+ normalTasksDict.append(normalTask.getDict())
+ return normalTasksDict
+
+ def getNormalTask(self, stepId):
+ for normalTask in self._normalTasks:
+ if normalTask.getStepId() == stepId:
+ return normalTask
+ return None
+
+
+class BaseWorkflowTask(object):
+ def __init__(self, name):
+ self._name = name
+ self._taskReferenceName = self._name + "_task_%s" % getRandString(10)
+ self._type = ''
+ self._args = {}
+
+ def __str__(self):
+ dictObj = self.getDict()
+ return str(dictObj)
+
+ def getDict(self):
+ d1 = {
+ "name": self._name,
+ "taskReferenceName": self._taskReferenceName,
+ "type": self._type
+ }
+ return dict(d1, **self._args)
+
+ def getName(self):
+ return self._name
+
+ def getReferenceName(self):
+ return self._taskReferenceName
+
+ def getTaskMetaList(self):
+ taskFile = TaskFile()
+ return [taskFile.generateFromStep(self)]
+
+
+class NormalTask(BaseWorkflowTask):
+ def __init__(self, order, stepObjArr, flowParser):
+ relatedStepObj = stepObjArr[order['step'] - 1]
+ super(NormalTask, self).__init__(relatedStepObj.getName())
+ self._taskReferenceName = "task_%s" % getRandString(10)
+ self._stepId = relatedStepObj.getId()
+ self._type = "HTTP"
+ self._args['inputParameters'] = relatedStepObj.getArgs()
+ self._paramTransform(self._args['inputParameters'], flowParser)
+ print "NormalTask:----------------------\n", relatedStepObj.getArgs()
+
+ def _paramTransform(self, argsDict, flowParser):
+ for (k, v) in argsDict.items():
+ if isinstance(v, str):
+ if re.match("^\(\(\d+\..*\)\)", v):
+ v = v[2:-2]
+ stepId, outputParam = v.split(".")
+ stepId = int(stepId)
+ normalTask = flowParser.getNormalTask(stepId)
+ if normalTask is None:
+ continue
+ argsDict[k] = "${%s.output.response.body.%s}" % \
+ (normalTask.getReferenceName(), outputParam)
+ elif isinstance(v, dict):
+ self._paramTransform(v, flowParser)
+
+ def getStepId(self):
+ return self._stepId
+
+
+class SwitchTask(BaseWorkflowTask):
+ seqNumber = 0
+
+ def __init__(self, order, stepObjArr, flowParser):
+ super(SwitchTask, self).__init__("switch_" + str(SwitchTask.seqNumber))
+ SwitchTask.seqNumber = SwitchTask.seqNumber + 1
+ if 'name' in order:
+ self._name = order['name']
+ self._type = "DECISION"
+ caseValueParam = 'value'
+ order['value'] = order['value'][2:-2]
+ stepId, outputParam = order['value'].split(".")
+ stepId = int(stepId)
+ normalTask = flowParser.getNormalTask(stepId)
+ caseValue = "${%s.output.response.body.%s}" % \
+ (normalTask.getReferenceName(), outputParam)
+ self._args['inputParameters'] = {caseValueParam: caseValue}
+ self._args['caseValueParam'] = caseValueParam
+ self._args['decisionCases'] = {}
+ self._childTaskMetaList = []
+ for case, caseOrders in order['cases'].items():
+ self._args['decisionCases'][case], taskMetaList = \
+ flowParser.parse(caseOrders, stepObjArr)
+ if taskMetaList is not None:
+ self._childTaskMetaList.extend(taskMetaList)
+
+ def getTaskMetaList(self):
+ selfTaskMetaList = super(SwitchTask, self).getTaskMetaList()
+ selfTaskMetaList.extend(self._childTaskMetaList)
+ return selfTaskMetaList
+
+
+class ParallelTask(BaseWorkflowTask):
+ seqNumber = 0
+
+ def __init__(self, order, stepObjArr, flowParser):
+ InstSeqNumber = ParallelTask.seqNumber
+ super(ParallelTask, self).__init__("parallel_" + str(InstSeqNumber))
+ ParallelTask.seqNumber = ParallelTask.seqNumber + 1
+ if 'name' in order:
+ self._name = order['name']
+ self._type = "FORK_JOIN"
+ self._args['forkTasks'] = []
+ self._childTaskMetaList = []
+ lastTasksNameList = []
+ parallelList = order['parallel'].items()
+ parallelList.sort()
+ for key, orderList in parallelList:
+ print orderList
+ taskList, taskMetaList = flowParser.parse(orderList, stepObjArr)
+ self._args['forkTasks'].append(taskList)
+ lastTasksNameList.append(taskList[-1]['taskReferenceName'])
+ if taskMetaList is not None:
+ self._childTaskMetaList.extend(taskMetaList)
+ self._joinTaskObj = ParallelJoinTask(InstSeqNumber, lastTasksNameList)
+
+ def getTaskMetaList(self):
+ selfTaskMetaList = super(ParallelTask, self).getTaskMetaList()
+ selfTaskMetaList.extend(self._childTaskMetaList)
+ selfTaskMetaList.extend(self._joinTaskObj.getTaskMetaList())
+ return selfTaskMetaList
+
+ def getJoinTask(self):
+ return self._joinTaskObj
+
+
+class ParallelJoinTask(BaseWorkflowTask):
+ def __init__(self, seqNumber, joinOnList):
+ super(ParallelJoinTask, self).__init__(
+ "paralleljoin_" + str(seqNumber))
+ self._type = "JOIN"
+ self._args['joinOn'] = joinOnList
+
+
+def getRandString(length):
+ return "".join(random.choice(str("0123456789")) for i in range(length))
diff --git a/testing-scheduler/server/src/env/context/context.yaml b/testing-scheduler/server/src/env/context/context.yaml new file mode 100644 index 00000000..58ed4614 --- /dev/null +++ b/testing-scheduler/server/src/env/context/context.yaml @@ -0,0 +1,75 @@ +##############################################################################
+# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+---
+
+hosts:
+ node26:
+ name: node26
+ ip: 192.168.1.26
+ port: 22
+ user: root
+ password: 123456
+
+ node27:
+ name: node27
+ ip: 192.168.1.27
+ port: 22
+ user: root
+ password: 123456
+
+ node28:
+ name: node28
+ ip: 192.168.1.28
+ port: 22
+ user: root
+ password: 123456
+
+ node29:
+ name: node29
+ ip: 192.168.1.29
+ port: 22
+ user: root
+ password: 123456
+
+ node30:
+ name: node30
+ ip: 192.168.1.30
+ port: 22
+ user: root
+ password: 123456
+
+ node31:
+ name: node31
+ ip: 192.168.1.31
+ port: 22
+ user: root
+ password: 123456
+
+ node32:
+ name: node32
+ ip: 192.168.1.32
+ port: 22
+ user: root
+ password: 123456
+
+
+cassandra:
+ name: node24
+ ip: 192.168.1.24
+ port: 22
+ user: root
+ password: 123456
+
+sprout:
+ name: node32
+ ip: 192.168.1.32
+ port: 22
+ user: root
+ password: 123456
\ No newline at end of file diff --git a/testing-scheduler/server/src/env/service/ansible.yaml b/testing-scheduler/server/src/env/service/ansible.yaml new file mode 100644 index 00000000..7b97602d --- /dev/null +++ b/testing-scheduler/server/src/env/service/ansible.yaml @@ -0,0 +1,65 @@ +##############################################################################
+# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+---
+
+ansible:
+ apis:
+ - method: POST
+ name: faultload
+ params:
+ - description: call user count
+ name: call_user
+ - description: workload time
+ name: duration
+ - description: faultload time
+ name: fault_duration
+ - description: register speed
+ name: initial_reg_rate
+ - description: multiplier
+ name: multiplier
+ - description: register user count
+ name: reg_user
+ baseuri: faultload
+ template:
+ uri: ((baseuri))
+ body:
+ call_user: ((call_user))
+ duration: ((duration))
+ fault_duration: ((fault_duration))
+ initial_reg_rate: ((initial_reg_rate))
+ multiplier: ((multiplier))
+ reg_user: ((reg_user))
+
+ - method: POST
+ name: workload
+ params:
+ - description: call user count
+ name: call_user
+ - description: workload time
+ name: duration
+ - description: register speed
+ name: initial_reg_rate
+ - description: multiplier
+ name: multiplier
+ - description: register user count
+ name: reg_user
+ baseuri: workload
+ template:
+ uri: ((baseuri))
+ body:
+ call_user: ((call_user))
+ duration: ((duration))
+ fault_duration: ((fault_duration))
+ initial_reg_rate: ((initial_reg_rate))
+ multiplier: ((multiplier))
+ reg_user: ((reg_user))
+
+ ip: 100.64.227.222
+ port: 9006
diff --git a/testing-scheduler/server/src/env/service/greet.yaml b/testing-scheduler/server/src/env/service/greet.yaml new file mode 100644 index 00000000..1b21f413 --- /dev/null +++ b/testing-scheduler/server/src/env/service/greet.yaml @@ -0,0 +1,68 @@ +##############################################################################
+# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+---
+
+greet:
+ ip: t-scheduler-server
+ port: 5312
+ apis:
+ -
+ name: greet
+ method: GET
+ baseuri: greet
+ params:
+ - name: name
+ description: user name
+ template:
+ uri: ((baseuri))?name=((name))
+ response:
+ result: int
+ next:
+ aa: str
+ bb: int
+ -
+ name: answer
+ method: POST
+ baseuri: answer
+ params:
+ - name: ping
+ description: param ping
+ template:
+ uri: ((baseuri))
+ body:
+ ping: ((ping))
+
+ -
+ name: ten
+ method: GET
+ baseuri: ten
+ template:
+ uri: ((baseuri))
+
+
+ -
+ name: switch
+ method: GET
+ baseuri: switch
+ template:
+ uri: ((baseuri))
+ return:
+ -
+ result
+
+ -
+ name: switch_2
+ method: GET
+ baseuri: switch_2
+ template:
+ uri: ((baseuri))
+ return:
+ -
+ result
diff --git a/testing-scheduler/server/src/step/__init__.py b/testing-scheduler/server/src/step/__init__.py new file mode 100644 index 00000000..e8198009 --- /dev/null +++ b/testing-scheduler/server/src/step/__init__.py @@ -0,0 +1,8 @@ +##############################################################################
+# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
diff --git a/testing-scheduler/server/src/step/general_test_step.py b/testing-scheduler/server/src/step/general_test_step.py new file mode 100644 index 00000000..2f9e8bcc --- /dev/null +++ b/testing-scheduler/server/src/step/general_test_step.py @@ -0,0 +1,87 @@ +##############################################################################
+# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from src.step.test_step import TestStep
+import os
+import yaml
+import re
+
+
+class GeneralTestStep(TestStep):
+ __step_type__ = "test"
+
+ def __init__(self, id, name, service, action, args, context):
+ super(GeneralTestStep, self).__init__(
+ self.__step_type__, id, name, service, action, args, context)
+ self._stepParse()
+ self.action()
+
+ def _contextTransform(self, argsDict):
+ for (k, v) in argsDict.items():
+ if isinstance(v, str):
+ if re.match('^\(\(context\..*\)\)', v):
+ v = v[10:-2]
+ layers = v.split(".")
+ contextData = self._context
+ for layer in layers:
+ contextData = contextData[layer]
+ argsDict[k] = contextData
+ elif isinstance(v, dict):
+ self._contextTransform(v)
+
+ def _stepParse(self):
+ self._args_temp = self._args
+ self._args = {}
+
+ # transform the service config
+ envFilePath = os.path.join(
+ self._getCurrentDir(), "..", "env",
+ "service", self._serviceName + ".yaml")
+ requestParam = {}
+ with open(envFilePath, 'r') as f:
+ conf = yaml.load(f)
+ conf = conf[self._serviceName]
+ for apiItem in conf["apis"]:
+ if apiItem['name'] == self._serviceInterface:
+ interfaceConf = apiItem
+ if interfaceConf is None:
+ return
+
+ # transform the args config
+ self._contextTransform(self._args_temp)
+
+ interfaceUri = interfaceConf['baseuri'] + \
+ interfaceConf['template']['uri'][11:]
+ interfaceUri = "http://%s:%s/%s" % (
+ conf['ip'], conf['port'], interfaceUri)
+ requestParam['uri'] = self._uriTransform(interfaceUri)
+
+ requestParam['method'] = interfaceConf['method']
+ if requestParam["method"] == "POST":
+ requestParam['body'] = interfaceConf['template']['body']
+ self._paramTransform(requestParam['body'], self._args_temp)
+ self._args['http_request'] = requestParam
+
+ def _uriTransform(self, uri):
+ return re.sub("\(\(.*?\)\)", self._uriResReplace, uri)
+
+ def _uriResReplace(self, match):
+ matchTrim = match.group()[2:-2]
+ return self._args_temp[matchTrim]
+
+ def _paramTransform(self, argsTemplate, argsDict):
+ for (k, v) in argsTemplate.items():
+ if isinstance(v, str):
+ if re.match('^\(\(.*\)\)', v):
+ argsTemplate[k] = argsDict[v[2:-2]]
+ elif isinstance(v, dict):
+ self._paramTransform(v, argsDict)
+
+ def start(self):
+ pass
diff --git a/testing-scheduler/server/src/step/monitor.py b/testing-scheduler/server/src/step/monitor.py new file mode 100644 index 00000000..6deb9e2e --- /dev/null +++ b/testing-scheduler/server/src/step/monitor.py @@ -0,0 +1,57 @@ +##############################################################################
+# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import json
+import os
+from src.step.test_step import TestStep
+
+
+class MonitorStep(TestStep):
+ __step_type__ = 'monitor'
+
+ def __init__(self, name, service, action, args):
+ super(MonitorStep, self).__init__(name, service, action, args)
+ self._argsParse()
+ self.action()
+
+ def _argsParse(self):
+ if self._call == "REST":
+ currentDirPath = os.path.dirname(os.path.abspath(__file__))
+ envDirPath = os.path.abspath(os.path.join(
+ currentDirPath, os.pardir, os.pardir, 'env'))
+ envFilePath = os.path.join(
+ envDirPath, "%s.json" % self._service['name'])
+ with open(envFilePath) as f:
+ propDict = json.load(f)
+ self._args['ip'] = propDict['ip']
+ self._args['port'] = propDict['port']
+ self._args['api'] = "%s/%s" % (
+ propDict['api_map']['workload'], self._args['command'])
+ exclude = {'ip', 'port', 'api', 'command', 'method'}
+ self._args['req_body'] = {
+ key: value for key, value in
+ self._args.items() if key not in exclude}
+
+ def setUp(self):
+ print "monitor setUp"
+
+ def uninstall(self):
+ print "monitor uninstall"
+
+ def start(self):
+ print "monitor start...."
+
+ def stop(self):
+ print "monitor stop"
+
+
+if __name__ == "__main__":
+ service = {"name": "ansible", "call": "REST"}
+ monitor = MonitorStep(
+ "monitor_cpu", service, "start", **{"target": "abc:qq"})
diff --git a/testing-scheduler/server/src/step/step_manager.py b/testing-scheduler/server/src/step/step_manager.py new file mode 100644 index 00000000..8d76c67c --- /dev/null +++ b/testing-scheduler/server/src/step/step_manager.py @@ -0,0 +1,41 @@ +##############################################################################
+# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from src.step.test_step import TestStep
+import os
+import sys
+
+
+class TestStepManager(object):
+ def __init__(self, context):
+ self._context = context
+
+ currentDirPath = os.path.dirname(os.path.abspath(__file__))
+ sys.path.append(currentDirPath)
+
+ excludeFiles = ('__init__.py', 'step_manager.py', 'test_step.py')
+ for fileName in os.listdir(currentDirPath):
+ if os.path.isfile(os.path.join(currentDirPath, fileName)) and \
+ os.path.splitext(fileName)[1] == '.py' and \
+ fileName not in excludeFiles:
+ __import__(os.path.splitext(fileName)[0])
+
+ def getStepObj(self, type, id, name, service, action, args):
+ for subclass in TestStep.__subclasses__():
+ if type == subclass.__step_type__:
+ return subclass(id, name, service, action, args, self._context)
+
+
+if __name__ == "__main__":
+ tsMgr = TestStepManager()
+ args = {'command': 'greet', 'method': 'POST', 'args': {'name': 'leo'}}
+ stepObj = tsMgr.getStepObj('test', 1, 'test_cpu', {
+ 'name': 'greet', 'call': 'REST'}, 'start', args)
+ print stepObj
+ print stepObj.__class__.__mro__
diff --git a/testing-scheduler/server/src/step/test_step.py b/testing-scheduler/server/src/step/test_step.py new file mode 100644 index 00000000..363c4800 --- /dev/null +++ b/testing-scheduler/server/src/step/test_step.py @@ -0,0 +1,56 @@ +##############################################################################
+# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import os
+
+
+class TestStep(object):
+ def __init__(self, type, id, name, service, action, args, context):
+ self._type = type
+ self._id = id
+ self._name = name
+ self._serviceName = service['name']
+ self._serviceInterface = service['interface']
+ self._action = action
+ self._args = args
+ self._context = context
+
+ def getId(self):
+ return self._id
+
+ def getName(self):
+ return self._name
+
+ def getServiceName(self):
+ return self._serviceName
+
+ def getCallFunction(self):
+ return self._callType
+
+ def getArgs(self):
+ return self._args
+
+ def action(self):
+ f = getattr(self, self._action)
+ f()
+
+ def _argsParse(self):
+ pass
+
+ def _getCurrentDir(self):
+ return os.path.dirname(__file__)
+
+ def __str__(self):
+ return str(self.__dict__)
+
+
+if __name__ == "__main__":
+ args = {'command': 'start'}
+ stepObj = TestStep('test_cpu', 'ansible', 'REST', **args)
+ print stepObj
diff --git a/testing-scheduler/server/src/step/workload.py b/testing-scheduler/server/src/step/workload.py new file mode 100644 index 00000000..265b9a6b --- /dev/null +++ b/testing-scheduler/server/src/step/workload.py @@ -0,0 +1,46 @@ +##############################################################################
+# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import json
+import os
+from src.step.test_step import TestStep
+
+
+class WorkloadStep(TestStep):
+ __step_type__ = 'workload'
+
+ def __init__(self, id, name, service, action, args):
+ super(WorkloadStep, self).__init__(
+ self.__step_type__, id, name, service, action, args)
+ self._argsParse()
+ self._action()
+
+ def _argsParse(self):
+ if self._callType == "REST":
+ currentDirPath = os.path.dirname(os.path.abspath(__file__))
+ envDirPath = os.path.abspath(
+ os.path.join(currentDirPath, os.pardir, os.pardir, 'env'))
+ envFilePath = os.path.join(
+ envDirPath, "%s.json" % self._service['name'])
+ with open(envFilePath) as f:
+ propDict = json.load(f)
+ self._args['ip'] = propDict['ip']
+ self._args['port'] = propDict['port']
+ self._args['api'] = "%s/%s" % (
+ propDict['api_map']['workload'], self._args['command'])
+ exclude = {'ip', 'port', 'api', 'command', 'method'}
+ self._args['req_body'] = {
+ key: value for key, value in
+ self._args.items() if key not in exclude}
+
+ def _start(self):
+ print "workload start"
+
+ def _stop(self):
+ print "workload stop"
diff --git a/testing-scheduler/server/src/test_parser.py b/testing-scheduler/server/src/test_parser.py new file mode 100644 index 00000000..7b471517 --- /dev/null +++ b/testing-scheduler/server/src/test_parser.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python
+
+##############################################################################
+# Copyright (c) 2018 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import click
+import os
+import yaml
+import json
+import collections
+from src.step.step_manager import TestStepManager
+from src.conductor_processor.workflow import WorkflowFile
+from conductorclient.run_new_workflow import WorkflowMgr
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+CONDUCTOR_SERVER_ADDR = "http://conductor_conductor-server_1:8080"
+STORE_TASK_PATH = "/tmp/generate_task.json"
+STORE_WF_PATH = "/tmp/generate_workflow.json"
+
+
+@click.command()
+@click.option("--filepath", help="file path of test case")
+def cmdParse(filepath):
+ parse(filepath)
+
+
+def parse(filepath):
+ filePrefix, fileName = os.path.split(filepath)
+ print '------------ start to parse the test case:' + \
+ '%s ----------------' % fileName
+ with open(filepath) as f:
+ yaml_file = yaml.load(f)
+ parseTestcase(yaml_file['schema'], fileName)
+
+ workflowId = runWorkFlow()
+ print '------------------- parse executes end -------------------------'
+
+ return workflowId
+
+
+def parseTestcase(schema, tcName='testcase0'):
+ if schema is None:
+ return parseLog(False, reason='schema not found.')
+ steps = schema['steps']
+ if steps is None:
+ return parseLog(False, reason='steps is invalid.')
+ flows = schema['flows']
+ if flows is None:
+ return parseLog(False, reasion='flows is invalid.')
+ # steps is a list, step is dict. no json here.
+ # steps = sorted(steps, sortById)
+
+ # load context
+ contextDict = {}
+ contextDir = os.path.join(BASE_DIR, "env", "context", "context.yaml")
+ with open(contextDir, "r") as f:
+ contextDict = yaml.load(f)
+ #
+ testStepMgr = TestStepManager(contextDict)
+
+ stepObjArr = []
+ for step in steps:
+ if 'args' not in step:
+ step['args'] = {}
+ # type and action can be extended, default couple is 'test' & 'start'.
+ if 'type' not in step:
+ step['type'] = 'test'
+ step['action'] = 'start'
+
+ stepObj = testStepMgr.getStepObj(
+ step['type'], step['id'], step['name'], step['service'],
+ step['action'], step['args'])
+ stepObjArr.append(stepObj)
+
+ # generate workflow by 'flow' and 'step'
+ tcName = os.path.splitext(tcName)[0]
+ wfFileObj = WorkflowFile(tcName)
+ workflowDict, taskMetaList = wfFileObj.generateMetaData(flows, stepObjArr)
+
+ with open(STORE_TASK_PATH, 'w') as f:
+ f.write(json.dumps({'task_group': taskMetaList}, indent=True))
+ with open(STORE_WF_PATH, 'w') as f:
+ f.write(json.dumps(workflowDict, indent=True))
+
+
+def parseWebTestcase(webTestcase):
+ print 'parseWebTestcase----------------------------'
+
+ stepList = webTestcase['stepList']
+ mainOrdersList = webTestcase['mainOrdersList']
+ subflowList = webTestcase['subflowList']
+
+ parseData = collections.OrderedDict()
+ parseData['schema'] = collections.OrderedDict()
+ parseData['schema']['steps'] = []
+ parseData['schema']['flows'] = []
+
+ parseStepList = parseData['schema']['steps']
+ parseFlowList = parseData['schema']['flows']
+ stepIndexDict = {}
+ # parse stepList
+ for index in range(len(stepList)):
+ stepItem = stepList[index]
+ parseStep = collections.OrderedDict()
+
+ parseStep['id'] = index + 1
+ parseStep['name'] = stepItem['name']
+ parseStep['service'] = collections.OrderedDict()
+ parseStep['service']['name'] = stepItem['service']
+ parseStep['service']['interface'] = stepItem['action']
+ parseStep['action'] = 'start'
+ parseStep['args'] = {}
+ for paramItem in stepItem['params']:
+ parseStep['args'][paramItem['key']] = transParamString(
+ paramItem['value'])
+
+ parseStepList.append(parseStep)
+ stepIndexDict[parseStep['name']] = parseStep['id']
+ # parse flows
+ # parse mainflow
+ print stepIndexDict
+ typeDict = {1: 'normal', 2: 'switch', 3: 'parallel'}
+ mainFlow = collections.OrderedDict()
+ mainFlow['name'] = 'main'
+ mainFlow['orders'] = []
+ mainFlow['orders'] = parseOrderList(
+ mainOrdersList, stepIndexDict, typeDict)
+ parseFlowList.append(mainFlow)
+
+ # parse subflow
+ for subflowItem in subflowList:
+ replaceSubflow = collections.OrderedDict()
+ replaceSubflow['name'] = subflowItem['name']
+ replaceSubflow['orders'] = parseOrderList(
+ subflowItem['orderList'], stepIndexDict, typeDict)
+ parseFlowList.append(replaceSubflow)
+
+ print 'END parseWebTestcase----------------------------'
+ return parseData
+
+
+# parse orderlist from web edition to server edition
+def parseOrderList(orderList, stepIndexDict, typeDict):
+ replaceList = []
+ for orderItem in orderList:
+ replaceOrder = collections.OrderedDict()
+ orderType = typeDict[orderItem['type']]
+ replaceOrder['type'] = orderType
+ if orderType == 'normal':
+ stepId = stepIndexDict[orderItem['step']]
+ replaceOrder['step'] = stepId
+ elif orderType == 'switch':
+ replaceOrder['value'] = orderItem['value']
+ replaceOrder['cases'] = collections.OrderedDict()
+ for caseItem in orderItem['cases']:
+ caseValue = caseItem['value']
+ caseOrderType = caseItem['orderType']
+ caseOrderValue = caseItem['orderValue']
+ if caseOrderType == "step":
+ orderInCase = collections.OrderedDict()
+ orderInCase['type'] = 'normal'
+ orderInCase['step'] = stepIndexDict[caseOrderValue]
+ replaceOrder['cases'][caseValue] = [orderInCase]
+ else:
+ replaceOrder['cases'][caseValue] = caseOrderValue
+ else:
+ replaceOrder['parallel'] = collections.OrderedDict()
+ pIndex = 1
+ for branchItem in orderItem['branches']:
+ pKey = 'p' + str(pIndex)
+ branchOrderType = branchItem['orderType']
+ branchOrderValue = branchItem['orderValue']
+ if branchOrderType == "step":
+ replaceBranchItem = collections.OrderedDict()
+ replaceBranchItem['type'] = 'normal'
+ replaceBranchItem['step'] = stepIndexDict[branchOrderValue]
+ replaceOrder['parallel'][pKey] = [replaceBranchItem]
+ else:
+ replaceOrder['parallel'][pKey] = branchOrderValue
+ pIndex += 1
+ replaceList.append(replaceOrder)
+ return replaceList
+
+
+def transParamString(val):
+ if type(val) != str:
+ return val
+ if '.' not in val:
+ if val.isdigit():
+ return int(val)
+ try:
+ f = float(val)
+ return f
+ except ValueError:
+ return val
+
+
+def getWebTestcase(originTcDict):
+ print "getWebTestcase----------------------------------"
+ webTcDict = {
+ "stepList": [],
+ "mainOrdersList": [],
+ "subflowList": []
+ }
+ stepList = webTcDict['stepList']
+ subflowList = webTcDict['subflowList']
+ if originTcDict is None:
+ return webTcDict
+ originContent = originTcDict['schema']
+ originSteps = originContent['steps']
+ stepIndexDict = {}
+ # transform steps to stepList
+ for stepItem in originSteps:
+ replaceStep = {}
+ replaceStep['name'] = stepItem['name']
+ replaceStep['service'] = stepItem['service']['name']
+ replaceStep['action'] = stepItem['service']['interface']
+ replaceStep['params'] = []
+ if 'args' in stepItem:
+ for (key, value) in stepItem['args'].items():
+ replaceParam = {}
+ replaceParam['key'] = key
+ replaceParam['value'] = value
+ replaceStep['params'].append(replaceParam)
+ stepList.append(replaceStep)
+ stepIndexDict[stepItem['id']] = stepItem['name']
+
+ # transform main flow
+ originFlows = originContent['flows']
+ originMainflow = {}
+ for flowIndex in range(len(originFlows)):
+ flowItem = originFlows[flowIndex]
+ if flowItem['name'] == 'main':
+ originMainflow = flowItem
+ originFlows.pop(flowIndex)
+ break
+ typeDict = {'normal': 1, 'switch': 2, 'parallel': 3}
+ webTcDict['mainOrdersList'] = getOrderList(
+ originMainflow['orders'], stepIndexDict, typeDict)
+
+ # transform subflows
+ for originSubflow in originFlows:
+ replaceSubflow = {}
+ replaceSubflow['name'] = originSubflow['name']
+ replaceSubflow['orderList'] = getOrderList(
+ originSubflow['orders'], stepIndexDict, typeDict)
+ subflowList.append(replaceSubflow)
+
+ # return web edition of testcase
+ print "END getWebTestcase----------------------------------"
+ return webTcDict
+
+
+def getOrderList(originOrderList, stepIndexDict, typeDict):
+ replaceOrderList = []
+ for orderItem in originOrderList:
+ replaceOrderItem = {}
+ orderType = orderItem['type']
+ replaceOrderItem['type'] = typeDict[orderType]
+ if orderType == 'normal':
+ stepName = stepIndexDict[orderItem['step']]
+ replaceOrderItem['step'] = stepName
+ elif orderType == 'switch':
+ replaceOrderItem['value'] = orderItem['value']
+ replaceOrderItem['cases'] = []
+ for (caseValue, ordersInCase) in orderItem['cases'].items():
+ replaceCase = {}
+ replaceCase['value'] = caseValue
+ if type(ordersInCase) == list:
+ replaceCase['orderType'] = 'step'
+ caseStepName = stepIndexDict[ordersInCase[0]['step']]
+ replaceCase['orderValue'] = caseStepName
+ else:
+ replaceCase['orderType'] = 'flow'
+ replaceCase['orderValue'] = ordersInCase
+ replaceOrderItem['cases'].append(replaceCase)
+ else:
+ replaceOrderItem['branches'] = []
+ for paraIndex in orderItem['parallel']:
+ paraItem = orderItem['parallel'][paraIndex]
+ replaceBranch = {}
+ if type(paraItem) == list:
+ replaceBranch['orderType'] = 'step'
+ branchStepName = stepIndexDict[paraItem[0]['step']]
+ replaceBranch['orderValue'] = branchStepName
+ else:
+ replaceBranch['orderType'] = 'flow'
+ replaceBranch['orderValue'] = paraItem
+ replaceOrderItem['branches'].append(replaceBranch)
+ replaceOrderList.append(replaceOrderItem)
+
+ return replaceOrderList
+
+
+def runWorkFlow():
+ wfMgr = WorkflowMgr(CONDUCTOR_SERVER_ADDR)
+ wfMgr.setTaskDefFromFile(STORE_TASK_PATH)
+ wfMgr.setWorkflowFromFile(STORE_WF_PATH)
+ inputParam = {'input': 'fake'}
+ workflowId = wfMgr.startWorkflow(inputParam)
+ return workflowId
+
+
+def parseLog(flag, **msg):
+ return {'result': flag, 'message': msg}
+
+
+if __name__ == "__main__":
+ cmdParse()
|