summaryrefslogtreecommitdiffstats
path: root/compass-tasks/log_analyzor/file_matcher.py
blob: be3143bf3dbee14819e8fc2ae6ad615809eb0e02 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
# Copyright 2014 Huawei Technologies Co. Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module to update intalling progress by processing log file.

   .. moduleauthor:: Xiaodong Wang <xiaodongwang@huawei.com>
"""
import logging
import os.path

from compass.utils import setting_wrapper as setting


class FileFilter(object):
    """base class to filter log file."""
    def __repr__(self):
        return self.__class__.__name__

    def filter(self, pathname):
        """Filter log file.

        :param pathname: the absolute path name to the log file.
        """
        raise NotImplementedError(str(self))


class CompositeFileFilter(FileFilter):
    """filter log file based on the list of filters."""
    def __init__(self, filters):
        self.filters_ = filters

    def __str__(self):
        return 'CompositeFileFilter[%s]' % self.filters_

    def append_filter(self, file_filter):
        """append filter."""
        self.filters_.append(file_filter)

    def filter(self, pathname):
        """filter log file."""
        for file_filter in self.filters_:
            if not file_filter.filter(pathname):
                return False

        return True


class FilterFileExist(FileFilter):
    """filter log file if not exists."""
    def filter(self, pathname):
        """filter log file."""
        file_exist = os.path.isfile(pathname)
        if not file_exist:
            logging.debug("%s is not exist", pathname)

        return file_exist


def get_file_filter():
    """get file filter"""
    composite_filter = CompositeFileFilter([FilterFileExist()])
    return composite_filter


class FileReader(object):
    """Class to read log file.

    The class provide support to read log file from the position
    it has read last time. and update the position when it finish
    reading the log.
    """
    def __init__(self, pathname, log_history):
        self.pathname_ = pathname
        self.log_history_ = log_history

    def __repr__(self):
        return (
            '%s[pathname:%s, log_history:%s]' % (
                self.__class__.__name__, self.pathname_,
                self.log_history_
            )
        )

    def readline(self):
        """Generate each line of the log file."""
        old_position = self.log_history_['position']
        position = self.log_history_['position']
        partial_line = self.log_history_['partial_line']
        try:
            with open(self.pathname_) as logfile:
                logfile.seek(position)
                while True:
                    line = logfile.readline()
                    partial_line += line
                    position = logfile.tell()
                    if position > self.log_history_['position']:
                        self.log_history_['position'] = position

                    if partial_line.endswith('\n'):
                        self.log_history_['partial_line'] = ''
                        yield partial_line
                        partial_line = self.log_history_['partial_line']
                    else:
                        self.log_history_['partial_line'] = partial_line
                        break
                if partial_line:
                    yield partial_line

        except Exception as error:
            logging.error('failed to processing file %s', self.pathname_)
            raise error

        logging.debug(
            'processing file %s log %s bytes to position %s',
            self.pathname_, position - old_position, position
        )


class FileReaderFactory(object):
    """factory class to create FileReader instance."""

    def __init__(self, logdir):
        self.logdir_ = logdir
        self.filefilter_ = get_file_filter()

    def __str__(self):
        return '%s[logdir: %s filefilter: %s]' % (
            self.__class__.__name__, self.logdir_, self.filefilter_)

    def get_file_reader(self, hostname, filename, log_history):
        """Get FileReader instance.

        :param fullname: fullname of installing host.
        :param filename: the filename of the log file.

        :returns: :class:`FileReader` instance if it is not filtered.
        """
        pathname = os.path.join(self.logdir_, hostname, filename)
        logging.debug('get FileReader from %s', pathname)
        if not self.filefilter_.filter(pathname):
            logging.debug('%s is filtered', pathname)
            return None

        return FileReader(pathname, log_history)


class FileMatcher(object):
    """File matcher to get the installing progress from the log file."""
    def __init__(self, line_matchers, min_progress, max_progress, filename):
        if not 0.0 <= min_progress <= max_progress <= 1.0:
            raise IndexError(
                '%s restriction is not mat: 0.0 <= min_progress'
                '(%s) <= max_progress(%s) <= 1.0' % (
                    self.__class__.__name__,
                    min_progress,
                    max_progress))
        if 'start' not in line_matchers:
            raise KeyError(
                'key `start` does not in line matchers %s' % line_matchers
            )
        self.line_matchers_ = line_matchers
        self.min_progress_ = min_progress
        self.max_progress_ = max_progress
        self.progress_diff_ = max_progress - min_progress
        self.filename_ = filename

    def __repr__(self):
        return (
            '%r[filename: %r, progress:[%r:%r], '
            'line_matchers: %r]' % (
                self.__class__.__name__, self.filename_,
                self.min_progress_,
                self.max_progress_, self.line_matchers_)
        )

    def update_progress_from_log_history(self, state, log_history):
        file_percentage = log_history['percentage']
        percentage = max(
            self.min_progress_,
            min(
                self.max_progress_,
                self.min_progress_ + file_percentage * self.progress_diff_
            )
        )
        if (
            percentage > state['percentage'] or
            (
                percentage == state['percentage'] and
                log_history['message'] != state['message']
            )
        ):
            state['percentage'] = percentage
            state['message'] = log_history['message']
            state['severity'] = log_history['severity']
        else:
            logging.debug(
                'ingore update state %s from log history %s '
                'since the updated progress %s lag behind',
                state, log_history, percentage
            )

    def update_progress(self, file_reader_factory, name, state, log_history):
        """update progress from file.

        :param fullname: the fullname of the installing host.
        :type fullname: str
        :param total_progress: Progress instance to update.

        the function update installing progress by reading the log file.
        It contains a list of line matcher, when one log line matches
        with current line matcher, the installing progress is updated.
        and the current line matcher got updated.
        Notes: some line may be processed multi times. The case is the
        last line of log file is processed in one run, while in the other
        run, it will be reprocessed at the beginning because there is
        no line end indicator for the last line of the file.
        """
        file_reader = file_reader_factory.get_file_reader(
            name, self.filename_, log_history)
        if not file_reader:
            return

        line_matcher_name = log_history['line_matcher_name']
        for line in file_reader.readline():
            if line_matcher_name not in self.line_matchers_:
                logging.debug('early exit at\n%s\nbecause %s is not in %s',
                              line, line_matcher_name, self.line_matchers_)
                break

            same_line_matcher_name = line_matcher_name
            while same_line_matcher_name in self.line_matchers_:
                line_matcher = self.line_matchers_[same_line_matcher_name]
                same_line_matcher_name, line_matcher_name = (
                    line_matcher.update_progress(line, log_history)
                )
        log_history['line_matcher_name'] = line_matcher_name
        logging.debug(
            'updated log history %s after processing %s',
            log_history, self
        )
        self.update_progress_from_log_history(state, log_history)