1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
|
##############################################################################
# Copyright (c) 2016 NEC Corporation and others.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
import argparse
import collections
from flask import Flask
from flask import request
import json
import logger as doctor_log
import threading
import time
from keystoneauth1 import session
import novaclient.client as novaclient
import identity_auth
LOG = doctor_log.Logger('doctor_inspector').getLogger()
class ThreadedResetState(threading.Thread):
def __init__(self, nova, state, server):
threading.Thread.__init__(self)
self.nova = nova
self.state = state
self.server = server
def run(self):
self.nova.servers.reset_state(self.server, self.state)
LOG.info('doctor mark vm(%s) error at %s' % (self.server, time.time()))
class DoctorInspectorSample(object):
NOVA_API_VERSION = '2.34'
NUMBER_OF_CLIENTS = 50
# TODO(tojuvone): This could be enhanced in future with dynamic
# reuse of self.novaclients when all threads in use and
# self.NUMBER_OF_CLIENTS based on amount of cores or overriden by input
# argument
def __init__(self):
self.servers = collections.defaultdict(list)
self.novaclients = list()
auth=identity_auth.get_identity_auth()
sess=session.Session(auth=auth)
# Pool of novaclients for redundant usage
for i in range(self.NUMBER_OF_CLIENTS):
self.novaclients.append(
novaclient.Client(self.NOVA_API_VERSION, session=sess))
# Normally we use this client for non redundant API calls
self.nova=self.novaclients[0]
self.nova.servers.list(detailed=False)
self.init_servers_list()
def init_servers_list(self):
opts = {'all_tenants': True}
servers=self.nova.servers.list(search_opts=opts)
self.servers.clear()
for server in servers:
try:
host=server.__dict__.get('OS-EXT-SRV-ATTR:host')
self.servers[host].append(server)
LOG.debug('get hostname=%s from server=%s' % (host, server))
except Exception as e:
LOG.error('can not get hostname from server=%s' % server)
def disable_compute_host(self, hostname):
threads = []
if len(self.servers[hostname]) > self.NUMBER_OF_CLIENTS:
# TODO(tojuvone): This could be enhanced in future with dynamic
# reuse of self.novaclients when all threads in use
LOG.error('%d servers in %s. Can handle only %d'%(
self.servers[hostname], hostname, self.NUMBER_OF_CLIENTS))
for nova, server in zip(self.novaclients, self.servers[hostname]):
t = ThreadedResetState(nova, "error", server)
t.start()
threads.append(t)
for t in threads:
t.join()
self.nova.services.force_down(hostname, 'nova-compute', True)
LOG.info('doctor mark host(%s) down at %s' % (hostname, time.time()))
app = Flask(__name__)
inspector = DoctorInspectorSample()
@app.route('/events', methods=['POST'])
def event_posted():
LOG.info('event posted at %s' % time.time())
LOG.info('inspector = %s' % inspector)
LOG.info('received data = %s' % request.data)
d = json.loads(request.data)
for event in d:
hostname = event['details']['hostname']
event_type = event['type']
if event_type == 'compute.host.down':
inspector.disable_compute_host(hostname)
return "OK"
def get_args():
parser = argparse.ArgumentParser(description='Doctor Sample Inspector')
parser.add_argument('port', metavar='PORT', type=int, nargs='?',
help='a port for inspector')
return parser.parse_args()
def main():
args = get_args()
app.run(host='0.0.0.0', port=args.port)
if __name__ == '__main__':
main()
|
