diff options
| -rw-r--r-- | anteater/src/get_lists.py | 35 | ||||
| -rw-r--r-- | gatechecks.yaml | 31 | ||||
| -rw-r--r-- | releng_exception.yaml | 61 | ||||
| -rw-r--r-- | sandbox_exception.yaml | 11 |
4 files changed, 111 insertions, 27 deletions
diff --git a/anteater/src/get_lists.py b/anteater/src/get_lists.py index 8941510..7d6d3f1 100644 --- a/anteater/src/get_lists.py +++ b/anteater/src/get_lists.py @@ -16,6 +16,8 @@ import anteater.utils.anteater_logger as antlog import ConfigParser +import copy +import os import yaml import re @@ -27,14 +29,45 @@ gate_checks = config.get('config', 'gate_checks') with open(gate_checks, 'r') as f: yl = yaml.safe_load(f) +def _remove_nullvalue(contents): + if contents and len(contents) > 2 and 'nullvalue' in contents: + contents.remove('nullvalue') + +def _merge(org, ded): + ret = copy.deepcopy(org) + for key in list(set([k for k in org] + [k for k in ded])): + if key in org and key in ded: + ret[key] = list(set(ret[key] + ded[key])) + _remove_nullvalue(ret[key]) + elif key in ded: + ret[key] = ded[key] + return ret class GetLists(object): def __init__(self, *args): # Placeholder for future args if more filters are needed self.args = args + self.loaded = False + + def load_project_exception_file(self, project_config, project): + if self.loaded: + return + exception_file = None + for item in project_config: + if project in item: + exception_file = item.get(project) + if exception_file is not None: + with open(exception_file, 'r') as f: + ex = yaml.safe_load(f) + for key in ex: + if key in yl: + yl[key][project] = _merge(yl[key][project], ex.get(key, None)) \ + if project in yl[key] else ex.get(key, None) + self.loaded = True def binary_list(self, project): project_list = False + self.load_project_exception_file(yl.get('project_config'), project) try: default_list = (yl['binaries']['binary_ignore']) except KeyError: @@ -58,6 +91,7 @@ class GetLists(object): def file_audit_list(self, project): project_list = False + self.load_project_exception_file(yl.get('project_config'), project) try: default_list = set((yl['file_audits']['file_names'])) except KeyError: @@ -83,6 +117,7 @@ class GetLists(object): def file_content_list(self, project): project_list = False + self.load_project_exception_file(yl.get('project_config'), project) try: default_list = set((yl['file_audits']['file_contents'])) except KeyError: diff --git a/gatechecks.yaml b/gatechecks.yaml index 23b5bd5..243d5aa 100644 --- a/gatechecks.yaml +++ b/gatechecks.yaml @@ -7,41 +7,18 @@ binaries: binary_ignore: [\.pdf,\.png,\.jpeg,\.jpg,\.gif,\.idx,\.pack,\.woff,\.ttf,\.eot,\.DS_Store,\.git/objects,\.git/index, \.otf,\.ico] - releng: - binary_ignore: [nullvalue] - sandbox: - binary_ignore: [nullvalue] - testproj: - binary_ignore: [nullvalue] file_audits: file_names: [\.gpg$,\.key$,\.asc$,\.sig$,\.md5,aws_access_key_id,aws_secret_access_key,id_rsa] file_contents: [-----BEGIN\sRSA\sPRIVATE\sKEY----,secret,ssh_key,private_key,md5,wget,"curl(.*?)bash",eval, "sh(.*?)curl","git(.*?)clone",gost,md2,md4,md5,rc4,sha0,streebog,dual_ec_drbg, snefru,panama,ripemd,sslv1,sslv2,tlsv1] - releng: - file_names: [nullvalue] - file_contents: [slave_secret,self\.local_ssh_key,self\.proxy_ssh_key,jh_ssh_key='/root/\.ssh/id_rsa',fa-user-secret, - -s set secret key,paramiko\.RSAKey\.from_private_key_file\(pkey_file\),git clone the Openstack-Ansible, - secret not defined,user_secrets\.yml,wget -O /tmp/get-pip\.py,"PKG_MAP\\[wget\\]",^wget \\, - "git clone(.*)gerrit\\.opnfv\\.org","git clone(.*)\\.openstack\\.org",wget(.*)build.opnfv.org, - wget --version,"\\[wget\\]=wget",git repositories of XCI,"packages = \\['parted', 'puppet', 'wget'", - bifrost_ssh_private_key,github\.com/maestrodev/puppet-wget,"wget http://repo1\\.maven\\.org", - elif type wget >/dev/null,wget \$get_pip_url,"wget(.*)puppetlabs\\.com",git_clone(.*)MOD(.*)MODULE_PATH(.*)MODULE_NAME, - git clone \$MOD \$DEST,function git_clone,"name: SSH_KEY","name: GIT_CLONE_BASE", - wget -q -O \$ODL_ZIP \$NETVIRT_ARTIFACT_URL,git clone(.*)\$GIT_BASE(.*)\$WORKSPACE, - git clone --quiet --branch \$BRANCH \$LAB_CONFIG_URL lab-config,git clone(.*)\$OPENSTACK_OSA_GIT_URL, - wget --directory-prefix=(.*)SNAP_CACHE(.*)snap_url(.*),sudo yum -y install wget,cacheValues,isInfiniteValue, - wget(.*)WORKSPACE/opnfv\.properties(.*)GS_URL(.*)properties,wget > /dev/null,fileValidationError,eval_collection, - eval_db,if \(name === 'evals'\),evaluate,value,ast\.literal_eva,_eval_pods_db,new = eval\(obj\)] - sandbox: - file_names: [nullvalue] - file_contents: [nullvalue] - testproj: - file_names: [nullvalue] - file_contents: [nullvalue] licence: licence_ext: ['.rst','.md','.py','.sh','.java','.rb'] licence_ignore: ['__init__.py'] +project_config: + - releng: releng_exception.yaml + - sandbox: sandbox_exception.yaml + - testproj: sandbox_exception.yaml diff --git a/releng_exception.yaml b/releng_exception.yaml new file mode 100644 index 0000000..d860ce7 --- /dev/null +++ b/releng_exception.yaml @@ -0,0 +1,61 @@ +# When adding projects all `arrays: []` sections must have +# a value, Use 'nullvalue' if no waivers are available. +# +# This file uses standard regular expression syntax, however be mindful +# of escaping YAML delimiters too (such as `:`) using double quotes "". + +binaries: + binary_ignore: [nullvalue] +file_audits: + file_names: [nullvalue] + file_contents: + - -s set secret key + - "PKG_MAP\\[wget\\]" + - "\\[wget\\]=wget" + - "git clone(.*)\\.openstack\\.org" + - "git clone(.*)gerrit\\.opnfv\\.org" + - "name: GIT_CLONE_BASE" + - "name: SSH_KEY" + - "packages = \\['parted', 'puppet', 'wget'" + - "wget http://repo1\\.maven\\.org" + - "wget(.*)puppetlabs\\.com" + - ^wget \\ + - _eval_pods_db + - ast\.literal_eva + - bifrost_ssh_private_key + - cacheValues + - elif type wget >/dev/null + - eval_collection + - eval_db + - evaluate + - fa-user-secret + - fileValidationError + - function git_clone + - git clone --quiet --branch \$BRANCH \$LAB_CONFIG_URL lab-config + - git clone \$MOD \$DEST + - git clone the Openstack-Ansible + - git clone(.*)\$GIT_BASE(.*)\$WORKSPACE + - git clone(.*)\$OPENSTACK_OSA_GIT_URL + - git repositories of XCI + - git_clone(.*)MOD(.*)MODULE_PATH(.*)MODULE_NAME + - github\.com/maestrodev/puppet-wget + - if \(name === 'evals'\) + - isInfiniteValue + - jh_ssh_key='/root/\.ssh/id_rsa' + - new = eval\(obj\) + - paramiko\.RSAKey\.from_private_key_file\(pkey_file\) + - secret not defined + - self\.local_ssh_key + - self\.proxy_ssh_key + - slave_secret + - sudo yum -y install wget + - user_secrets\.yml + - value + - wget --directory-prefix=(.*)SNAP_CACHE(.*)snap_url(.*) + - wget --version + - wget -O /tmp/get-pip\.py + - wget -q -O \$ODL_ZIP \$NETVIRT_ARTIFACT_URL + - wget > /dev/null + - wget \$get_pip_url + - wget(.*)WORKSPACE/opnfv\.properties(.*)GS_URL(.*)properties + - wget(.*)build.opnfv.org diff --git a/sandbox_exception.yaml b/sandbox_exception.yaml new file mode 100644 index 0000000..14f0e81 --- /dev/null +++ b/sandbox_exception.yaml @@ -0,0 +1,11 @@ +# When adding projects all `arrays: []` sections must have +# a value, Use 'nullvalue' if no waivers are available. +# +# This file uses standard regular expression syntax, however be mindful +# of escaping YAML delimiters too (such as `:`) using double quotes "". + +binaries: + binary_ignore: [nullvalue] +file_audits: + file_names: [nullvalue] + file_contents: [nullvalue] |