From ac61abf57582a8489ecd2d29d3211e154d3de217 Mon Sep 17 00:00:00 2001 From: Pawel Tomaszewski Date: Tue, 5 Nov 2019 08:46:45 +0000 Subject: Python 2 to 3 migration of collectd-ves-app Change-Id: I656a67864d821454e5ecc1ed60ce601c872da23d Signed-off-by: Pawel Tomaszewski Signed-off-by: Kamil Wiatrowski --- 3rd_party/collectd-ves-app/ves_app/normalizer.py | 45 +++++++++++------------- 3rd_party/collectd-ves-app/ves_app/ves_app.py | 16 +++------ 2 files changed, 25 insertions(+), 36 deletions(-) diff --git a/3rd_party/collectd-ves-app/ves_app/normalizer.py b/3rd_party/collectd-ves-app/ves_app/normalizer.py index dcb02f27..0e92d1a5 100644 --- a/3rd_party/collectd-ves-app/ves_app/normalizer.py +++ b/3rd_party/collectd-ves-app/ves_app/normalizer.py @@ -17,7 +17,7 @@ # Volodymyr Mytnyk # -import yaml +from . import yaml import logging import datetime import time @@ -28,17 +28,12 @@ import re # import YAML loader try: - from yaml import CLoader as Loader + from .yaml import CLoader as Loader except ImportError: - from yaml import Loader + from .yaml import Loader # import synchronized queue -try: - # python 2.x - import Queue as queue -except ImportError: - # python 3.x - import queue +import queue class Config(object): @@ -80,7 +75,7 @@ class ItemIterator(object): self._collector = collector self._index = 0 - def next(self): + def __next__(self): """Returns next item from the list""" if self._index == len(self._items): raise StopIteration @@ -151,7 +146,7 @@ class Collector(object): def _check_aging(self): """Check aging time for all items""" self.lock() - for data_hash, data in self._metrics.items(): + for data_hash, data in list(self._metrics.items()): age, item = data if ((time.time() - age) >= self._age_timeout): # aging time has expired, remove the item from the collector @@ -189,7 +184,7 @@ class Collector(object): """Returns locked (safe) item iterator""" metrics = [] self.lock() - for k, item in self._metrics.items(): + for k, item in list(self._metrics.items()): _, value = item for select in select_list: if value.match(**select): @@ -220,7 +215,7 @@ class CollectdData(object): def match(self, **kargs): # compare the metric - for key, value in kargs.items(): + for key, value in list(kargs.items()): if self.is_regular_expression(value): if re.match(value[1:-1], getattr(self, key)) is None: return False @@ -323,7 +318,7 @@ class Item(yaml.YAMLObject): class ValueItem(Item): """Class to process VlaueItem tag""" - yaml_tag = u'!ValueItem' + yaml_tag = '!ValueItem' @classmethod def from_yaml(cls, loader, node): @@ -343,7 +338,7 @@ class ValueItem(Item): # if VALUE key isn't given, use default VALUE key # format: `VALUE: !Number '{vl.value}'` if value_desc is None: - value_desc = yaml.ScalarNode(tag=u'!Number', value=u'{vl.value}') + value_desc = yaml.ScalarNode(tag='!Number', value='{vl.value}') # select collectd metric based on SELECT condition metrics = loader.collector.items(select) assert len(metrics) < 2, \ @@ -361,7 +356,7 @@ class ValueItem(Item): class ArrayItem(Item): """Class to process ArrayItem tag""" - yaml_tag = u'!ArrayItem' + yaml_tag = '!ArrayItem' @classmethod def from_yaml(cls, loader, node): @@ -415,12 +410,12 @@ class ArrayItem(Item): class Measurements(ArrayItem): """Class to process Measurements tag""" - yaml_tag = u'!Measurements' + yaml_tag = '!Measurements' class Events(Item): """Class to process Events tag""" - yaml_tag = u'!Events' + yaml_tag = '!Events' @classmethod def from_yaml(cls, loader, node): @@ -441,7 +436,7 @@ class Events(Item): class Bytes2Kibibytes(yaml.YAMLObject): """Class to process Bytes2Kibibytes tag""" - yaml_tag = u'!Bytes2Kibibytes' + yaml_tag = '!Bytes2Kibibytes' @classmethod def from_yaml(cls, loader, node): @@ -450,7 +445,7 @@ class Bytes2Kibibytes(yaml.YAMLObject): class Number(yaml.YAMLObject): """Class to process Number tag""" - yaml_tag = u'!Number' + yaml_tag = '!Number' @classmethod def from_yaml(cls, loader, node): @@ -462,7 +457,7 @@ class Number(yaml.YAMLObject): class StripExtraDash(yaml.YAMLObject): """Class to process StripExtraDash tag""" - yaml_tag = u'!StripExtraDash' + yaml_tag = '!StripExtraDash' @classmethod def from_yaml(cls, loader, node): @@ -471,7 +466,7 @@ class StripExtraDash(yaml.YAMLObject): class MapValue(yaml.YAMLObject): """Class to process MapValue tag""" - yaml_tag = u'!MapValue' + yaml_tag = '!MapValue' @classmethod def from_yaml(cls, loader, node): @@ -485,7 +480,7 @@ class MapValue(yaml.YAMLObject): assert val is not None, "Mandatory VALUE key isn't set" assert val in mapping, \ 'Value "{}" cannot be mapped to any of {} values'.format( - val, mapping.keys()) + val, list(mapping.keys())) return mapping[val] @@ -514,10 +509,10 @@ class Normalizer(object): measurements.append((key, value)) if value.tag == Events.yaml_tag: events.append((key, value)) - measurements_yaml = yaml.MappingNode(u'tag:yaml.org,2002:map', + measurements_yaml = yaml.MappingNode('tag:yaml.org,2002:map', measurements) measurements_stream = yaml.serialize(measurements_yaml) - events_yaml = yaml.MappingNode(u'tag:yaml.org,2002:map', events) + events_yaml = yaml.MappingNode('tag:yaml.org,2002:map', events) events_stream = yaml.serialize(events_yaml) # return event & measurements definition return events_stream, measurements_stream diff --git a/3rd_party/collectd-ves-app/ves_app/ves_app.py b/3rd_party/collectd-ves-app/ves_app/ves_app.py index 105c66e2..fa4e9349 100644 --- a/3rd_party/collectd-ves-app/ves_app/ves_app.py +++ b/3rd_party/collectd-ves-app/ves_app/ves_app.py @@ -15,23 +15,17 @@ import json import sys import base64 -import ConfigParser +import configparser import logging import argparse from distutils.util import strtobool from kafka import KafkaConsumer -from normalizer import Normalizer -from normalizer import CollectdValue - -try: - # For Python 3.0 and later - import urllib.request as url -except ImportError: - # Fall back to Python 2's urllib2 - import urllib2 as url +from .normalizer import Normalizer +from .normalizer import CollectdValue +import urllib.request as url class VESApp(Normalizer): """VES Application""" @@ -110,7 +104,7 @@ class VESApp(Normalizer): def init(self, configfile, schema_file): if configfile is not None: # read VES configuration file if provided - config = ConfigParser.ConfigParser() + config = configparser.ConfigParser() config.optionxform = lambda option: option config.read(configfile) self.config(config) -- cgit 1.2.3-korg From 96015a4a509dc7feb622f97325e8eb39896cc5e3 Mon Sep 17 00:00:00 2001 From: "Tomaszewski, PawelX" Date: Wed, 13 Nov 2019 04:35:06 -0800 Subject: Python 2 to 3 migration fixes Change-Id: I487632435ae575c2507e58ee5682aa03934391b8 Signed-off-by: Pawel Tomaszewski Signed-off-by: Kamil Wiatrowski --- 3rd_party/collectd-ves-app/ves_app/normalizer.py | 11 +++++++---- 3rd_party/collectd-ves-app/ves_app/ves_app.py | 13 +++++++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/3rd_party/collectd-ves-app/ves_app/normalizer.py b/3rd_party/collectd-ves-app/ves_app/normalizer.py index 0e92d1a5..95feead1 100644 --- a/3rd_party/collectd-ves-app/ves_app/normalizer.py +++ b/3rd_party/collectd-ves-app/ves_app/normalizer.py @@ -17,7 +17,7 @@ # Volodymyr Mytnyk # -from . import yaml +import yaml import logging import datetime import time @@ -28,12 +28,15 @@ import re # import YAML loader try: - from .yaml import CLoader as Loader + from yaml import CLoader as Loader except ImportError: - from .yaml import Loader + from yaml import Loader # import synchronized queue -import queue +try: + import queue +except ImportError: + import Queue as queue class Config(object): diff --git a/3rd_party/collectd-ves-app/ves_app/ves_app.py b/3rd_party/collectd-ves-app/ves_app/ves_app.py index fa4e9349..a8551239 100644 --- a/3rd_party/collectd-ves-app/ves_app/ves_app.py +++ b/3rd_party/collectd-ves-app/ves_app/ves_app.py @@ -15,17 +15,26 @@ import json import sys import base64 -import configparser import logging import argparse +try: + import configparser +except ImportError: + import ConfigParser as configparser + from distutils.util import strtobool from kafka import KafkaConsumer from .normalizer import Normalizer from .normalizer import CollectdValue -import urllib.request as url +try: + # For Python 3.0 and later + import urllib.request as url +except ImportError: + # Fall back to Python 2's urllib2 + import urllib2 as url class VESApp(Normalizer): """VES Application""" -- cgit 1.2.3-korg From 70a1a62034a9aae01901fcc02ed5a8f1c48d7566 Mon Sep 17 00:00:00 2001 From: "Tomaszewski, PawelX" Date: Mon, 25 Nov 2019 05:30:47 -0800 Subject: Relative imports fix Change-Id: If3d86e80be8de94c01ba599bd4d83be139d9886c Signed-off-by: Plwel Tomaszewski Signed-off-by: Kamil Wiatrowski --- 3rd_party/collectd-ves-app/ves_app/ves_app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/3rd_party/collectd-ves-app/ves_app/ves_app.py b/3rd_party/collectd-ves-app/ves_app/ves_app.py index a8551239..c101089f 100644 --- a/3rd_party/collectd-ves-app/ves_app/ves_app.py +++ b/3rd_party/collectd-ves-app/ves_app/ves_app.py @@ -26,8 +26,8 @@ except ImportError: from distutils.util import strtobool from kafka import KafkaConsumer -from .normalizer import Normalizer -from .normalizer import CollectdValue +from normalizer import Normalizer +from normalizer import CollectdValue try: # For Python 3.0 and later -- cgit 1.2.3-korg