1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
|
#!/usr/bin/env python
# -*- coding: utf-8 -*-
##############################################################################
# Copyright (c) 2017 Luke Hinds <lhinds@redhat.com>, Red Hat
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
"""
Accepts the --path argument and iterates the root directory using os.walk
If a file is a binary, or contains a blacklisted string. If any violations
are found, the script adds the violation to a log file.
"""
from __future__ import division, print_function, absolute_import
import hashlib
import six.moves.configparser
import os
import re
import logging
from binaryornot.check import is_binary
from . import get_lists
logger = logging.getLogger(__name__)
config = six.moves.configparser.RawConfigParser()
config.read('anteater.conf')
reports_dir = config.get('config', 'reports_dir')
master_list = config.get('config', 'master_list')
ignore_dirs = ['.git']
hasher = hashlib.sha256()
def prepare_project(project, project_dir):
""" Generates blacklists / whitelists and calls main functions """
# Get Various Lists / Project Waivers
lists = get_lists.GetLists()
# Get binary white list
binary_list = lists.binary_list(project)
# Get file name black list and project waivers
file_audit_list, file_audit_project_list = lists.file_audit_list(project)
# Get file content black list and project waivers
master_list, project_list = lists.file_content_list(project)
# Get Licence Lists
licence_ext = lists.licence_extensions()
licence_ignore = lists.licence_ignore()
# Perform rudimentary scans
scan_file(project_dir, project, binary_list,file_audit_list,
file_audit_project_list, master_list,
project_list)
# Perform licence header checks
licence_check(licence_ext, licence_ignore, project, project_dir)
licence_root_check(project_dir, project)
def scan_file(project_dir, project, binary_list, file_audit_list,
file_audit_project_list, master_list,
project_list):
"""Searches for banned strings and files that are listed """
for root, dirs, files in os.walk(project_dir):
# Filter out ignored directories from list.
dirs[:] = [d for d in dirs if d not in ignore_dirs]
for items in files:
full_path = os.path.join(root, items)
# Check for Blacklisted file names
if file_audit_list.search(full_path) and not \
file_audit_project_list.search(full_path):
match = file_audit_list.search(full_path)
logger.error('Blacklisted filename: %s', full_path)
logger.error('Matched String: %s', match.group())
with open(reports_dir + "file-names_" + project + ".log",
"a") as gate_report:
gate_report. \
write('Blacklisted filename: {0}\n'.
format(full_path))
gate_report. \
write('Matched String: {0}'.
format(match.group()))
if not is_binary(full_path):
try:
fo = open(full_path, 'r')
lines = fo.readlines()
except IOError:
logger.error('%s does not exist', full_path)
for line in lines:
# Check for sensitive content in project files
for key, value in master_list.iteritems():
regex = value['regex']
desc = value['desc']
if re.search(regex, line) and not re.search(project_list, line):
logger.error('File contains violation: %s', full_path)
logger.error('Flagged Content: %s', line.rstrip())
logger.error('Matched Regular Exp: %s', regex)
logger.error('Rationale: %s', desc.rstrip())
with open(reports_dir + "contents-" + project + ".log",
"a") \
as gate_report:
gate_report. \
write('File contains violation: {0}\n'.
format(full_path))
gate_report. \
write('Flagged Content: {0}'.
format(line))
gate_report. \
write('Matched Regular Exp: {0}'.
format(regex))
gate_report. \
write('Rationale: {0}\n'.
format(desc.rstrip()))
else:
# Check if Binary is whitelisted
hashlist = get_lists.GetLists()
binary_hash = hashlist.binary_hash(project, full_path)
if not binary_list.search(full_path):
with open(full_path, 'rb') as afile:
buf = afile.read()
hasher.update(buf)
if hasher.hexdigest() in binary_hash:
logger.info('Found matching file hash for file: %s',
full_path)
else:
logger.error('Non Whitelisted Binary file: %s',
full_path)
logger.error('Please submit patch with this hash: %s',
hasher.hexdigest())
with open(reports_dir + "binaries-" + project + ".log",
"a") \
as gate_report:
gate_report.write('Non Whitelisted Binary: {0}\n'.
format(full_path))
def licence_root_check(project_dir, project):
if os.path.isfile(project_dir + '/LICENSE'):
logger.info('LICENSE file present in: %s', project_dir)
else:
logger.error('LICENSE file missing in: %s', project_dir)
with open(reports_dir + "licence-" + project + ".log",
"a") \
as gate_report:
gate_report.write('LICENSE file missing in: {0}\n'.
format(project_dir))
def licence_check(licence_ext, licence_ignore, project, project_dir):
""" Peform basic checks for the presence of licence strings """
for root, dirs, files in os.walk(project_dir):
dirs[:] = [d for d in dirs if d not in ignore_dirs]
for file in files:
if file.endswith(tuple(licence_ext)) \
and file not in licence_ignore:
full_path = os.path.join(root, file)
if not is_binary(full_path):
fo = open(full_path, 'r')
content = fo.read()
# Note: Hardcoded use of 'copyright' & 'spdx' is the result
# of a decision made at 2017 plugfest to limit searches to
# just these two strings.
patterns = ['copyright', 'spdx',
'http://creativecommons.org/licenses/by/4.0']
if any(i in content.lower() for i in patterns):
logger.info('Licence string present: %s', full_path)
else:
logger.error('Licence header missing: %s', full_path)
with open(reports_dir + "licence-" + project + ".log",
"a") \
as gate_report:
gate_report.write('Licence header missing: {0}\n'.
format(full_path))
|