diff options
Diffstat (limited to 'anteater')
| -rw-r--r-- | anteater/src/get_lists.py | 51 | ||||
| -rw-r--r-- | anteater/src/patch_scan.py | 9 | ||||
| -rw-r--r-- | anteater/src/project_scan.py | 16 |
3 files changed, 46 insertions, 30 deletions
diff --git a/anteater/src/get_lists.py b/anteater/src/get_lists.py index 17de7cb..ff63442 100644 --- a/anteater/src/get_lists.py +++ b/anteater/src/get_lists.py @@ -27,9 +27,13 @@ config = six.moves.configparser.RawConfigParser() config.read('anteater.conf') logger = logging.getLogger(__name__) master_list = config.get('config', 'master_list') +ignore_list = config.get('config', 'ignore_list') with open(master_list, 'r') as f: - yl = yaml.safe_load(f) + ml = yaml.safe_load(f) + +with open(ignore_list, 'r') as f: + il = yaml.safe_load(f) def _remove_nullvalue(contents): @@ -65,14 +69,14 @@ class GetLists(object): with open(exception_file, 'r') as f: ex = yaml.safe_load(f) for key in ex: - if key in yl: - yl[key][project] = _merge(yl[key][project], ex.get(key, None)) \ - if project in yl[key] else ex.get(key, None) + if key in ml: + ml[key][project] = _merge(ml[key][project], ex.get(key, None)) \ + if project in ml[key] else ex.get(key, None) self.loaded = True def binary_list(self, project): try: - default_list = (yl['binaries']['binary_ignore']) + default_list = (ml['binaries']['binary_ignore']) except KeyError: logger.error('Key Error processing binary list values') @@ -81,10 +85,10 @@ class GetLists(object): return binary_re def binary_hash(self, project, patch_file): - self.load_project_exception_file(yl.get('project_exceptions'), project) + self.load_project_exception_file(ml.get('project_exceptions'), project) file_name = os.path.basename(patch_file) try: - binary_hash = (yl['binaries'][project][file_name]) + binary_hash = (ml['binaries'][project][file_name]) return binary_hash except KeyError: binary_hash = 'null' @@ -93,13 +97,13 @@ class GetLists(object): def file_audit_list(self, project): project_list = False - self.load_project_exception_file(yl.get('project_exceptions'), project) + self.load_project_exception_file(ml.get('project_exceptions'), project) try: - default_list = set((yl['file_audits']['file_names'])) + default_list = set((ml['file_audits']['file_names'])) except KeyError: logger.error('Key Error processing file_names list values') try: - project_list = set((yl['file_audits'][project]['file_names'])) + project_list = set((ml['file_audits'][project]['file_names'])) logger.info('file_names waivers found for %s', project) except KeyError: logger.info('No file_names waivers found for %s', project) @@ -117,39 +121,48 @@ class GetLists(object): def file_content_list(self, project): project_list = False - self.load_project_exception_file(yl.get('project_exceptions'), project) + self.load_project_exception_file(ml.get('project_exceptions'), project) try: - master_list = (yl['file_audits']['file_contents']) + master_list = (ml['file_audits']['file_contents']) except KeyError: logger.error('Key Error processing file_contents list values') try: - project_list = set((yl['file_audits'][project]['file_contents'])) - project_list_re = re.compile("|".join(project_list), - flags=re.IGNORECASE) + ignore_list = il['file_audits']['file_contents'] + + except KeyError: + logger.error('Key Error processing file_contents list values') + + try: + project_list = ml['file_audits'][project]['file_contents'] + except KeyError: logger.info('No file_contents waivers found for %s', project) - return master_list, project_list_re + ignore_list_merge = project_list + ignore_list + + ignore_list_re = re.compile("|".join(ignore_list_merge), flags=re.IGNORECASE) + + return master_list, ignore_list_re def file_ignore(self): try: - file_ignore = (yl['file_ignore']) + file_ignore = (ml['file_ignore']) except KeyError: logger.error('Key Error processing file_ignore list values') return file_ignore def licence_extensions(self): try: - licence_extensions = (yl['licence']['licence_ext']) + licence_extensions = (ml['licence']['licence_ext']) except KeyError: logger.error('Key Error processing licence_extensions list values') return licence_extensions def licence_ignore(self): try: - licence_ignore = (yl['licence']['licence_ignore']) + licence_ignore = (ml['licence']['licence_ignore']) except KeyError: logger.error('Key Error processing licence_ignore list values') return licence_ignore diff --git a/anteater/src/patch_scan.py b/anteater/src/patch_scan.py index 3b71f0a..133b0ff 100644 --- a/anteater/src/patch_scan.py +++ b/anteater/src/patch_scan.py @@ -47,7 +47,7 @@ def prepare_patchset(project, patchset): file_audit_list, file_audit_project_list = lists.file_audit_list(project) # Get file content black list and project waivers - master_list, project_list_re = lists.file_content_list(project) + master_list, ignore_list = lists.file_content_list(project) # Get File Ignore Lists file_ignore = lists.file_ignore() @@ -69,7 +69,7 @@ def prepare_patchset(project, patchset): # Perform binary and file / content checks scan_patch(project, patch_file, binary_list, file_audit_list, file_audit_project_list, - master_list, project_list_re, licence_ext, + master_list, ignore_list, licence_ext, file_ignore, licence_ignore) # Process each file in patch set using waivers generated above @@ -79,7 +79,7 @@ def prepare_patchset(project, patchset): def scan_patch(project, patch_file, binary_list, file_audit_list, file_audit_project_list, master_list, - project_list_re, licence_ext, file_ignore, licence_ignore): + ignore_list, licence_ext, file_ignore, licence_ignore): """ Scan actions for each commited file in patch set """ global failure if is_binary(patch_file): @@ -130,7 +130,8 @@ def scan_patch(project, patch_file, binary_list, file_audit_list, for key, value in master_list.iteritems(): regex = value['regex'] desc = value['desc'] - if re.search(regex, line) and not re.search(project_list_re, line): + if re.search(regex, line) and not re.search( + ignore_list, line): logger.error('File contains violation: %s', patch_file) logger.error('Flagged Content: %s', line.rstrip()) logger.error('Matched Regular Exp: %s', regex) diff --git a/anteater/src/project_scan.py b/anteater/src/project_scan.py index 12e9a97..9bb3539 100644 --- a/anteater/src/project_scan.py +++ b/anteater/src/project_scan.py @@ -30,6 +30,7 @@ config = six.moves.configparser.RawConfigParser() config.read('anteater.conf') reports_dir = config.get('config', 'reports_dir') master_list = config.get('config', 'master_list') +ignore_list = config.get('config', 'master_list') ignore_dirs = ['.git'] hasher = hashlib.sha256() @@ -47,7 +48,7 @@ def prepare_project(project, project_dir): file_audit_list, file_audit_project_list = lists.file_audit_list(project) # Get file content black list and project waivers - master_list, project_list = lists.file_content_list(project) + master_list, ignore_list = lists.file_content_list(project) # Get File Ignore Lists file_ignore = lists.file_ignore() @@ -58,8 +59,8 @@ def prepare_project(project, project_dir): # Perform rudimentary scans scan_file(project_dir, project, binary_list,file_audit_list, - file_audit_project_list, master_list, file_ignore, - project_list) + file_audit_project_list, master_list, ignore_list, + file_ignore) # Perform licence header checks licence_check(licence_ext, licence_ignore, project, project_dir) @@ -67,8 +68,8 @@ def prepare_project(project, project_dir): def scan_file(project_dir, project, binary_list, file_audit_list, - file_audit_project_list, master_list, file_ignore, - project_list): + file_audit_project_list, master_list, ignore_list, + file_ignore): """Searches for banned strings and files that are listed """ for root, dirs, files in os.walk(project_dir): # Filter out ignored directories from list. @@ -90,9 +91,10 @@ def scan_file(project_dir, project, binary_list, file_audit_list, write('Matched String: {0}'. format(match.group())) - # Check if Binary is whitelisted + # Check if Binary is whitelisted hashlist = get_lists.GetLists() binary_hash = hashlist.binary_hash(project, full_path) + if is_binary(full_path) and not binary_list.search(full_path): with open(full_path, 'rb') as afile: buf = afile.read() @@ -124,7 +126,7 @@ def scan_file(project_dir, project, binary_list, file_audit_list, regex = value['regex'] desc = value['desc'] if re.search(regex, line) and not re.search( - project_list, line): + ignore_list, line): logger.error('File contains violation: %s', full_path) logger.error('Flagged Content: %s', |