diff options
-rw-r--r-- | jjb/fastpathmetrics/fastpathmetrics.yml | 2 | ||||
-rw-r--r-- | jjb/fuel/fuel-plugin-verify-jobs.yml | 226 | ||||
-rw-r--r-- | utils/test/dashboard/dashboard/common/elastic_access.py | 33 | ||||
-rw-r--r-- | utils/test/dashboard/dashboard/conf/config.py | 12 | ||||
-rw-r--r-- | utils/test/dashboard/dashboard/elastic2kibana/main.py | 5 | ||||
-rw-r--r-- | utils/test/dashboard/dashboard/mongo2elastic/main.py | 49 | ||||
-rw-r--r-- | utils/test/dashboard/etc/config.ini | 4 |
7 files changed, 272 insertions, 59 deletions
diff --git a/jjb/fastpathmetrics/fastpathmetrics.yml b/jjb/fastpathmetrics/fastpathmetrics.yml index 504e07f25..40df385d8 100644 --- a/jjb/fastpathmetrics/fastpathmetrics.yml +++ b/jjb/fastpathmetrics/fastpathmetrics.yml @@ -18,7 +18,7 @@ gs-pathname: '' disabled: false - colorado: - branch: '{stream}' + branch: 'stable/{stream}' gs-pathname: '/{stream}' disabled: false diff --git a/jjb/fuel/fuel-plugin-verify-jobs.yml b/jjb/fuel/fuel-plugin-verify-jobs.yml new file mode 100644 index 000000000..affc705f3 --- /dev/null +++ b/jjb/fuel/fuel-plugin-verify-jobs.yml @@ -0,0 +1,226 @@ +- project: + name: 'fuel-plugin-verify-jobs' + + project: 'fuel-plugin' + + installer: 'fuel' +##################################### +# branch definitions +##################################### + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false +##################################### +# patch verification phases +##################################### + phase: + - 'build': + slave-label: 'opnfv-build-ubuntu' + - 'test': + slave-label: 'opnfv-build-ubuntu' +##################################### +# jobs +##################################### + jobs: + - 'fuel-verify-plugin-{stream}' + - 'fuel-verify-plugin-{phase}-{stream}' +##################################### +# job templates +##################################### +- job-template: + name: 'fuel-verify-plugin-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 4 + option: 'project' + + # the url to plugin repo can essentially become a variable if + # the plugin name is injected to env by gerrit plugin + scm: + - git: + url: 'https://git.openstack.org/openstack/fuel-plugin-bgpvpn' + refspec: '$GERRIT_REFSPEC' + branches: + - 'origin/$GERRIT_BRANCH' + skip-tag: true + choosing-strategy: 'gerrit' + timeout: 10 + wipe-workspace: true + + wrappers: + - ssh-agent-credentials: + users: + - '{ssh-credentials}' + - timeout: + timeout: 360 + fail: true + + triggers: + - gerrit: + server-name: 'review.openstack.org' + silent-start: false + skip-vote: + successful: true + failed: true + unstable: true + notbuilt: true + escape-quotes: true + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'PLAIN' + project-pattern: 'openstack/fuel-plugin-bgpvpn' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + forbidden-file-paths: + - compare-type: ANT + pattern: 'README.md|.gitignore|.gitreview' + readable-message: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - 'opnfv-build-defaults' + - 'fuel-verify-plugin-defaults': + gs-pathname: '{gs-pathname}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - multijob: + name: build + condition: SUCCESSFUL + projects: + - name: 'fuel-verify-plugin-build-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: test + condition: SUCCESSFUL + projects: + - name: 'fuel-verify-plugin-test-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + +- job-template: + name: 'fuel-verify-plugin-{phase}-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 6 + option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'fuel-verify-plugin-test-.*' + block-level: 'NODE' + + # the url to plugin repo can essentially become a variable if + # the plugin name is injected to env by gerrit plugin + scm: + - git: + url: 'https://git.openstack.org/openstack/fuel-plugin-bgpvpn' + refspec: '$GERRIT_REFSPEC' + branches: + - 'origin/$GERRIT_BRANCH' + skip-tag: true + choosing-strategy: 'gerrit' + timeout: 10 + wipe-workspace: true + + wrappers: + - ssh-agent-credentials: + users: + - '{ssh-credentials}' + - timeout: + timeout: 360 + fail: true + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - '{slave-label}-defaults' + - '{installer}-defaults' + - 'fuel-verify-plugin-defaults': + gs-pathname: '{gs-pathname}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - 'fuel-verify-plugin-{phase}-macro' +##################################### +# builder macros +##################################### +- builder: + name: 'fuel-verify-plugin-build-macro' + builders: + - shell: | + #!/bin/bash + + echo "Not activated!" + +- builder: + name: 'fuel-verify-plugin-test-macro' + builders: + - shell: | + #!/bin/bash + + echo "Not activated!" +##################################### +# parameter macros +##################################### +- parameter: + name: 'fuel-verify-plugin-defaults' + parameters: + - string: + name: BUILD_DIRECTORY + default: $WORKSPACE/build_output + description: "Directory where the build artifact will be located upon the completion of the build." + - string: + name: CACHE_DIRECTORY + default: $HOME/opnfv/cache/$INSTALLER_TYPE + description: "Directory where the cache to be used during the build is located." + - string: + name: GS_URL + default: artifacts.opnfv.org/$PROJECT{gs-pathname} + description: "URL to Google Storage." diff --git a/utils/test/dashboard/dashboard/common/elastic_access.py b/utils/test/dashboard/dashboard/common/elastic_access.py index b454e9a12..8c6494d39 100644 --- a/utils/test/dashboard/dashboard/common/elastic_access.py +++ b/utils/test/dashboard/dashboard/common/elastic_access.py @@ -22,14 +22,9 @@ def delete_docs(url, creds=None, body=None): return _request('DELETE', url, creds=creds, body=body) -def publish_docs(docs, creds, to): - json_docs = json.dumps(docs) - if to == 'stdout': - print json_docs - return 200, None - else: - result = _post(to, creds=creds, body=json_docs) - return result.status, result.data +def publish_docs(url, creds=None, body=None): + result = _post(url, creds=creds, body=(json.dumps(body))) + return result.status, result.data def _get_docs_nr(url, creds=None, body=None): @@ -48,25 +43,3 @@ def get_docs(url, creds=None, body=None, field='_source'): for hit in res_data['hits']['hits']: docs.append(hit[field]) return docs - - -def get_elastic_docs_by_days(elastic_url, creds, days): - if days == 0: - body = '''{ - "query": { - "match_all": {} - } - }''' - elif days > 0: - body = '''{{ - "query" : {{ - "range" : {{ - "start_date" : {{ - "gte" : "now-{}d" - }} - }} - }} - }}'''.format(days) - else: - raise Exception('Update days must be non-negative') - return get_docs(elastic_url, creds, body) diff --git a/utils/test/dashboard/dashboard/conf/config.py b/utils/test/dashboard/dashboard/conf/config.py index 2e0f1cabb..b868999a2 100644 --- a/utils/test/dashboard/dashboard/conf/config.py +++ b/utils/test/dashboard/dashboard/conf/config.py @@ -25,7 +25,6 @@ class APIConfig: self._default_config_location = "../etc/config.ini" self.elastic_url = 'http://localhost:9200' self.elastic_creds = None - self.destination = 'elasticsearch' self.kibana_url = None self.is_js = True self.js_path = None @@ -67,7 +66,6 @@ class APIConfig: # Linking attributes to keys from file with their sections obj.elastic_url = obj._get_str_parameter("elastic", "url") obj.elastic_creds = obj._get_str_parameter("elastic", "creds") - obj.destination = obj._get_str_parameter("output", "destination") obj.kibana_url = obj._get_str_parameter("kibana", "url") obj.is_js = obj._get_bool_parameter("kibana", "js") obj.js_path = obj._get_str_parameter("kibana", "js_path") @@ -77,12 +75,10 @@ class APIConfig: def __str__(self): return "elastic_url = %s \n" \ "elastic_creds = %s \n" \ - "destination = %s \n" \ "kibana_url = %s \n" \ "is_js = %s \n" \ "js_path = %s \n" % (self.elastic_url, - self.elastic_creds, - self.destination, - self.kibana_url, - self.is_js, - self.js_path) + self.elastic_creds, + self.kibana_url, + self.is_js, + self.js_path) diff --git a/utils/test/dashboard/dashboard/elastic2kibana/main.py b/utils/test/dashboard/dashboard/elastic2kibana/main.py index 38a49b63f..95f758ea8 100644 --- a/utils/test/dashboard/dashboard/elastic2kibana/main.py +++ b/utils/test/dashboard/dashboard/elastic2kibana/main.py @@ -60,7 +60,7 @@ class KibanaDashboard(dict): url = urlparse.urljoin(base_elastic_url, '/.kibana/visualization/{}'.format(visualization.id)) logger.debug("publishing visualization '{}'".format(url)) # logger.error("_publish_visualization: %s" % visualization) - elastic_access.publish_docs(visualization, es_creds, url) + elastic_access.publish_docs(url, es_creds, visualization) def _construct_panels(self): size_x = 6 @@ -138,7 +138,7 @@ class KibanaDashboard(dict): def _publish(self): url = urlparse.urljoin(base_elastic_url, '/.kibana/dashboard/{}'.format(self.id)) logger.debug("publishing dashboard '{}'".format(url)) - elastic_access.publish_docs(self, es_creds, url) + elastic_access.publish_docs(url, es_creds, self) def publish(self): self._publish_visualizations() @@ -191,7 +191,6 @@ class VisualizationBuilder(object): return json.loads(vis) - class KibanaVisualization(dict): def __init__(self, project_name, case_name, installer, pod, scenario, visualization): """ diff --git a/utils/test/dashboard/dashboard/mongo2elastic/main.py b/utils/test/dashboard/dashboard/mongo2elastic/main.py index 82b01e4b3..76efb14f0 100644 --- a/utils/test/dashboard/dashboard/mongo2elastic/main.py +++ b/utils/test/dashboard/dashboard/mongo2elastic/main.py @@ -38,12 +38,12 @@ tmp_docs_file = './mongo-{}.json'.format(uuid.uuid4()) class DocumentPublisher: - def __init__(self, doc, fmt, exist_docs, creds, to): + def __init__(self, doc, fmt, exist_docs, creds, elastic_url): self.doc = doc self.fmt = fmt self.creds = creds self.exist_docs = exist_docs - self.to = to + self.elastic_url = elastic_url self.is_formatted = True def format(self): @@ -64,7 +64,7 @@ class DocumentPublisher: self._publish() def _publish(self): - status, data = elastic_access.publish_docs(self.doc, self.creds, self.to) + status, data = elastic_access.publish_docs(self.elastic_url, self.creds, self.doc) if status > 300: logger.error('Publish record[{}] failed, due to [{}]' .format(self.doc, json.loads(data)['error']['reason'])) @@ -163,14 +163,13 @@ class DocumentPublisher: class DocumentsPublisher: - def __init__(self, project, case, fmt, days, elastic_url, creds, to): + def __init__(self, project, case, fmt, days, elastic_url, creds): self.project = project self.case = case self.fmt = fmt self.days = days self.elastic_url = elastic_url self.creds = creds - self.to = to self.existed_docs = [] def export(self): @@ -200,7 +199,36 @@ class DocumentsPublisher: exit(-1) def get_existed_docs(self): - self.existed_docs = elastic_access.get_elastic_docs_by_days(self.elastic_url, self.creds, self.days) + if self.days == 0: + body = '''{{ + "query": {{ + "bool": {{ + "must": [ + {{ "match": {{ "project_name": "{}" }} }}, + {{ "match": {{ "case_name": "{}" }} }} + ] + }} + }} + }}'''.format(self.project, self.case) + elif self.days > 0: + body = '''{{ + "query": {{ + "bool": {{ + "must": [ + {{ "match": {{ "project_name": "{}" }} }}, + {{ "match": {{ "case_name": "{}" }} }} + ], + "filter": {{ + "range": {{ + "start_date": {{ "gte": "now-{}d" }} + }} + }} + }} + }} + }}'''.format(self.project, self.case, self.days) + else: + raise Exception('Update days must be non-negative') + self.existed_docs = elastic_access.get_docs(self.elastic_url, self.creds, body) return self def publish(self): @@ -211,7 +239,7 @@ class DocumentsPublisher: self.fmt, self.existed_docs, self.creds, - self.to).format().publish() + self.elastic_url).format().publish() finally: fdocs.close() self._remove() @@ -223,13 +251,9 @@ class DocumentsPublisher: def main(): base_elastic_url = urlparse.urljoin(CONF.elastic_url, '/test_results/mongo2elastic') - to = CONF.destination days = args.latest_days es_creds = CONF.elastic_creds - if to == 'elasticsearch': - to = base_elastic_url - for project, case_dicts in testcases.testcases_yaml.items(): for case_dict in case_dicts: case = case_dict.get('name') @@ -239,5 +263,4 @@ def main(): fmt, days, base_elastic_url, - es_creds, - to).export().get_existed_docs().publish() + es_creds).export().get_existed_docs().publish() diff --git a/utils/test/dashboard/etc/config.ini b/utils/test/dashboard/etc/config.ini index b94ac7b4f..1e67bd822 100644 --- a/utils/test/dashboard/etc/config.ini +++ b/utils/test/dashboard/etc/config.ini @@ -4,10 +4,6 @@ url = http://localhost:9200 creds = -[output] -# elasticsearch or console -destination = elasticsearch - [kibana] url = http://10.63.243.17:5601/app/kibana js = true |