diff options
Diffstat (limited to 'app/discover/clique_finder.py')
| -rw-r--r-- | app/discover/clique_finder.py | 192 |
1 files changed, 121 insertions, 71 deletions
diff --git a/app/discover/clique_finder.py b/app/discover/clique_finder.py index 57b2e3b..4e04e7e 100644 --- a/app/discover/clique_finder.py +++ b/app/discover/clique_finder.py @@ -42,67 +42,90 @@ class CliqueFinder(Fetcher): return self.links.find({'target': db_id}) def find_cliques(self): - self.log.info("scanning for cliques") + self.log.info("Scanning for cliques") clique_types = self.get_clique_types().values() for clique_type in clique_types: self.find_cliques_for_type(clique_type) - self.log.info("finished scanning for cliques") + self.log.info("Finished scanning for cliques") - # Calculate priority score - def _get_priority_score(self, clique_type): - if self.env == clique_type['environment']: - return 4 - if (self.env_config['distribution'] == clique_type.get('distribution') and - self.env_config['distribution_version'] == clique_type.get('distribution_version')): - return 3 - if clique_type.get('mechanism_drivers') in self.env_config['mechanism_drivers']: - return 2 - if self.env_config['type_drivers'] == clique_type.get('type_drivers'): - return 1 - else: + # Calculate priority score for clique type per environment and configuration + def get_priority_score(self, clique_type): + # environment-specific clique type takes precedence + env = clique_type.get('environment') + config = self.env_config + # ECT - Clique Type with Environment name + if env: + if self.env == env: + return 2**6 + if env == 'ANY': + # environment=ANY serves as fallback option + return 2**0 return 0 + # NECT - Clique Type without Environment name + else: + env_type = clique_type.get('environment_type') + # TODO: remove backward compatibility ('if not env_type' check) + if env_type and env_type != config.get('environment_type'): + return 0 - # Get clique type with max priority - # for given environment configuration and focal point type - def _get_clique_type(self, focal_point, clique_types): - # If there's no configuration match for the specified environment, - # we use the default clique type definition with environment='ANY' - fallback_type = next( - filter(lambda t: t['environment'] == 'ANY', clique_types), - None - ) - if not fallback_type: - raise ValueError("No fallback clique type (ANY) " - "defined for focal point type '{}'" - .format(focal_point)) + score = 0 - clique_types.remove(fallback_type) + distribution = clique_type.get('distribution') + if distribution: + if config['distribution'] != distribution: + return 0 - priority_scores = [self._get_priority_score(clique_type) - for clique_type - in clique_types] - max_score = max(priority_scores) if priority_scores else 0 + score += 2**5 - return (fallback_type - if max_score == 0 - else clique_types[priority_scores.index(max_score)]) + dv = clique_type.get('distribution_version') + if dv: + if dv != config['distribution_version']: + return 0 + score += 2**4 - def get_clique_types(self): - if not self.clique_types_by_type: - clique_types_by_focal_point = self.clique_types.aggregate([{ - "$group": { - "_id": "$focal_point_type", - "types": {"$push": "$$ROOT"} - } - }]) + mechanism_drivers = clique_type.get('mechanism_drivers') + if mechanism_drivers: + if mechanism_drivers not in config['mechanism_drivers']: + return 0 + score += 2**3 - self.clique_types_by_type = { - cliques['_id']: self._get_clique_type(cliques['_id'], - cliques['types']) - for cliques in - clique_types_by_focal_point - } + type_drivers = clique_type.get('type_drivers') + if type_drivers: + if type_drivers != config['type_drivers']: + return 0 + score += 2**2 + + # If no configuration is specified, this clique type + # is a fallback for its environment type + return max(score, 2**1) + + # Get clique type with max priority + # for given focal point type + def _get_clique_type(self, clique_types): + scored_clique_types = [{'score': self.get_priority_score(clique_type), + 'clique_type': clique_type} + for clique_type in clique_types] + max_score = max(scored_clique_types, key=lambda t: t['score']) + if max_score['score'] == 0: + self.log.warn('No matching clique types ' + 'for focal point type: {fp_type}' + .format(fp_type=clique_types[0].get('focal_point_type'))) + return None + return max_score.get('clique_type') + def get_clique_types(self): + if not self.clique_types_by_type: + clique_types_candidates = {} + for clique in self.clique_types.find({}): + fp_type = clique.get('focal_point_type', '') + if not clique_types_candidates.get(fp_type): + clique_types_candidates[fp_type] = [] + clique_types_candidates[fp_type].append(clique) + for t in clique_types_candidates.keys(): + selected = self._get_clique_type(clique_types_candidates[t]) + if not selected: + continue + self.clique_types_by_type[t] = selected return self.clique_types_by_type def find_cliques_for_type(self, clique_type): @@ -125,11 +148,14 @@ class CliqueFinder(Fetcher): .find_one({"focal_point_type": o['type']}) constraints = [] if not constraint else constraint["constraints"] clique_types = self.get_clique_types() - clique_type = clique_types[o['type']] - new_clique = self.construct_clique_for_focal_point(o, clique_type, - constraints) - if not new_clique: + clique_type = clique_types.get(o['type']) + if not clique_type: self.cliques.delete({'_id': clique['_id']}) + else: + new_clique = self.construct_clique_for_focal_point(o, clique_type, + constraints) + if not new_clique: + self.cliques.delete({'_id': clique['_id']}) def construct_clique_for_focal_point(self, o, clique_type, constraints): # keep a hash of nodes in clique that were visited for each type @@ -146,12 +172,16 @@ class CliqueFinder(Fetcher): for c in constraints: val = o[c] if c in o else None clique["constraints"][c] = val + allow_implicit = clique_type.get('use_implicit_links', False) for link_type in clique_type["link_types"]: - self.check_link_type(clique, link_type, nodes_of_type) + if not self.check_link_type(clique, link_type, nodes_of_type, + allow_implicit=allow_implicit): + break # after adding the links to the clique, create/update the clique if not clique["links"]: return None + clique["clique_type"] = clique_type["_id"] focal_point_obj = self.inventory.find({"_id": clique["focal_point"]}) if not focal_point_obj: return None @@ -198,25 +228,33 @@ class CliqueFinder(Fetcher): '-'.join(link_type_parts) return CliqueFinder.link_type_reversed.get(link_type) - def check_link_type(self, clique, link_type, nodes_of_type): + def check_link_type(self, clique, link_type, nodes_of_type, + allow_implicit=False) -> bool: # check if it's backwards link_type_reversed = self.get_link_type_reversed(link_type) # handle case of links like T<-->T self_linked = link_type == link_type_reversed use_reversed = False if not self_linked: - matches = self.links.find_one({ + link_search_condition = { "environment": self.env, "link_type": link_type_reversed - }) + } + if not allow_implicit: + link_search_condition['implicit'] = False + matches = self.links.find_one(link_search_condition) use_reversed = True if matches else False if self_linked or not use_reversed: - self.check_link_type_forward(clique, link_type, nodes_of_type) + return self.check_link_type_forward(clique, link_type, + nodes_of_type, + allow_implicit=allow_implicit) if self_linked or use_reversed: - self.check_link_type_back(clique, link_type, nodes_of_type) + return self.check_link_type_back(clique, link_type, nodes_of_type, + allow_implicit=allow_implicit) def check_link_type_for_direction(self, clique, link_type, nodes_of_type, - is_reversed=False): + is_reversed=False, + allow_implicit=False) -> bool: if is_reversed: link_type = self.get_link_type_reversed(link_type) from_type = link_type[:link_type.index("-")] @@ -225,7 +263,7 @@ class CliqueFinder(Fetcher): other_side = 'target' if not is_reversed else 'source' match_type = to_type if is_reversed else from_type if match_type not in nodes_of_type.keys(): - return + return False other_side_type = to_type if not is_reversed else from_type nodes_to_add = set() for match_point in nodes_of_type[match_type]: @@ -233,21 +271,27 @@ class CliqueFinder(Fetcher): clique, link_type, side_to_match, - other_side) + other_side, + allow_implicit=allow_implicit) nodes_to_add = nodes_to_add | matches if other_side_type not in nodes_of_type: nodes_of_type[other_side_type] = set() nodes_of_type[other_side_type] = \ nodes_of_type[other_side_type] | nodes_to_add + return len(nodes_to_add) > 0 def find_matches_for_point(self, match_point, clique, link_type, - side_to_match, other_side) -> set: + side_to_match, other_side, + allow_implicit=False) -> set: nodes_to_add = set() - matches = self.links.find({ + link_search_condition = { "environment": self.env, "link_type": link_type, side_to_match: ObjectId(match_point) - }) + } + if not allow_implicit: + link_search_condition['implicit'] = False + matches = self.links.find(link_search_condition) for link in matches: link_id = link["_id"] if link_id in clique["links"]: @@ -260,10 +304,16 @@ class CliqueFinder(Fetcher): nodes_to_add.add(other_side_point) return nodes_to_add - def check_link_type_forward(self, clique, link_type, nodes_of_type): - self.check_link_type_for_direction(clique, link_type, nodes_of_type, - is_reversed=False) + def check_link_type_forward(self, clique, link_type, nodes_of_type, + allow_implicit=False) -> bool: + return self.check_link_type_for_direction(clique, link_type, + nodes_of_type, + is_reversed=False, + allow_implicit=allow_implicit) - def check_link_type_back(self, clique, link_type, nodes_of_type): - self.check_link_type_for_direction(clique, link_type, nodes_of_type, - is_reversed=True) + def check_link_type_back(self, clique, link_type, nodes_of_type, + allow_implicit=False) -> bool: + return self.check_link_type_for_direction(clique, link_type, + nodes_of_type, + is_reversed=True, + allow_implicit=allow_implicit) |