diff options
38 files changed, 567 insertions, 219 deletions
diff --git a/compass-cobbler/Dockerfile b/compass-cobbler/Dockerfile index d963e49..9ee1e64 100644 --- a/compass-cobbler/Dockerfile +++ b/compass-cobbler/Dockerfile @@ -6,7 +6,8 @@ ARG BRANCH=master # pkgs and services... RUN yum -y update && \ yum -y install epel-release && \ - yum -y install wget dhcp bind syslinux pykickstart file initscripts net-tools tcpdump xinetd vim avahi avahi-tools ntp && \ + yum -y install wget dhcp bind syslinux pykickstart file initscripts net-tools tcpdump xinetd vim \ + avahi avahi-tools ntp fence-agents libvirt-devel python-devel gcc python-pip && \ wget http://artifacts.opnfv.org/compass4nfv/package/cobbler/cobbler-2.6.10-1.fc22.noarch.rpm && \ wget http://artifacts.opnfv.org/compass4nfv/package/cobbler/cobbler-web-2.6.10-1.fc22.noarch.rpm && \ yum -y localinstall cobbler-2.6.10-1.fc22.noarch.rpm cobbler-web-2.6.10-1.fc22.noarch.rpm && \ @@ -15,7 +16,11 @@ RUN yum -y update && \ systemctl enable httpd && \ systemctl enable dhcpd && \ systemctl enable xinetd && \ - systemctl enable ntpd + systemctl enable ntpd && \ + pip install libvirt-python click + +COPY fence_libvirt /usr/sbin/fence_libvirt +COPY fence_libvirt.template /etc/cobbler/power/fence_libvirt.template # some tweaks on services RUN sed -i -e 's/\(^.*disable.*=\) yes/\1 no/' /etc/xinetd.d/tftp && \ diff --git a/compass-cobbler/fence_libvirt b/compass-cobbler/fence_libvirt new file mode 100755 index 0000000..53e235c --- /dev/null +++ b/compass-cobbler/fence_libvirt @@ -0,0 +1,106 @@ +#!/usr/bin/env python +import libvirt +import yaml +import multiprocessing +import click +import sys + + +SETTING = "/root/cobbler/settings" +power_action_map = { + "on": "create", + "off": "destroy", + "status": "state" + } + +def get_virt_host(setting_file): + with open(setting_file) as fd: + try: + settings = yaml.load(fd) + return settings['server'] + except Exception: + raise RuntimeError("Can't get server ip from %s" % SETTING) + + +def get_libvit_connection(user, passwd): + # def request_cred(credentials, user_data): + # for credential in credentials: + # if credential[0] == libvirt.VIR_CRED_AUTHNAME: + # credential[4] = user + # elif credential[0] == libvirt.VIR_CRED_PASSPHRASE: + # credential[4] = passwd + # return 0 + # auth = [[libvirt.VIR_CRED_AUTHNAME, libvirt.VIR_CRED_PASSPHRASE], + # request_cred, None] + server = get_virt_host(SETTING) + conn = libvirt.open('qemu+tcp://%s/system' % server) + return conn + + +def libvirt_function(domain, action, rc): + function = getattr(domain, power_action_map.get(action)) + state, reason = domain.state() + if function.__name__ == "create": + if state == libvirt.VIR_DOMAIN_RUNNING: + rc.value = 1 + else: + rc.value = function() + elif function.__name__ == "destroy": + if state == libvirt.VIR_DOMAIN_SHUTOFF: + rc.value = 1 + else: + rc.value = function() + elif function.__name__ == "state": + rc.value = state + + +def power_action(action, hostname, user, passwd): + conn = get_libvit_connection(user, passwd) + domain = conn.lookupByName(hostname) + rc = multiprocessing.Value('i') + p = multiprocessing.Process(target=libvirt_function, + args=(domain, action, rc,)) + p.start() + p.join() + print rc.value + return rc.value + + +@click.command() +@click.option("--action") +@click.option("--hostname") +@click.option("--user") +@click.option("--passwd") +def cli(action, hostname, user, passwd): + power_action(action, hostname, user, passwd) + + +def no_cli(): + opt = {} + for line in sys.stdin.readlines(): + try: + line = line.strip() + name, value = line.split("=") + opt.update({name: value}) + except Exception: + continue + + if opt["action"] and opt["hostname"]: + power_action(opt["action"], opt["hostname"], + opt["user"], opt["passwd"]) + else: + raise RuntimeError("Invalid argument, \ + action: {0}, hostname: {1}, user: {2}, passwd: {3}".format( + opt.get("action", None), opt.get("hostname", None), + opt.get("user", None), opt.get("action", None))) + + +def main(): + if len(sys.argv) > 1: + cli() + else: + no_cli() + + +if __name__ == '__main__': + main() diff --git a/compass-cobbler/fence_libvirt.template b/compass-cobbler/fence_libvirt.template new file mode 100644 index 0000000..2e38628 --- /dev/null +++ b/compass-cobbler/fence_libvirt.template @@ -0,0 +1,4 @@ +action=$power_mode +hostname=$hostname +user=$power_user +passwd=$power_pass diff --git a/compass-deck/api/__init__.py b/compass-deck/api/__init__.py index 784fe23..e2d2544 100644 --- a/compass-deck/api/__init__.py +++ b/compass-deck/api/__init__.py @@ -14,7 +14,7 @@ import datetime from flask import Blueprint -from flask.ext.login import LoginManager +from flask_login import LoginManager from flask import Flask # from compass.api.v1.api import v1_app diff --git a/compass-deck/api/api.py b/compass-deck/api/api.py index e1cdd39..1fa10f2 100644 --- a/compass-deck/api/api.py +++ b/compass-deck/api/api.py @@ -22,10 +22,10 @@ import netaddr import requests import simplejson as json -from flask.ext.login import current_user -from flask.ext.login import login_required -from flask.ext.login import login_user -from flask.ext.login import logout_user +from flask_login import current_user +from flask_login import login_required +from flask_login import login_user +from flask_login import logout_user from flask import request from compass.api import app diff --git a/compass-deck/api/v1/api.py b/compass-deck/api/v1/api.py index 9dbc548..345746a 100644 --- a/compass-deck/api/v1/api.py +++ b/compass-deck/api/v1/api.py @@ -19,7 +19,7 @@ import simplejson as json from flask import Blueprint from flask import request -from flask.ext.restful import Resource +from flask_restful import Resource from compass.api.exception import BadRequest from compass.api.exception import Forbidden diff --git a/compass-deck/apiclient/restful.py b/compass-deck/apiclient/restful.py index bb82922..f5d4855 100644 --- a/compass-deck/apiclient/restful.py +++ b/compass-deck/apiclient/restful.py @@ -231,7 +231,7 @@ class Client(object): return self._get('/switches/%s/machines-hosts' % switch_id, data=data) def add_switch_machine(self, switch_id, mac=None, port=None, - vlans=None, ipmi_credentials=None, + vlans=None, power_manage=None, tag=None, location=None, raw_data=None): data = {} if raw_data: @@ -246,8 +246,8 @@ class Client(object): if vlans: data['vlans'] = vlans - if ipmi_credentials: - data['ipmi_credentials'] = ipmi_credentials + if power_manage: + data['power_manage'] = power_manage if tag: data['tag'] = tag @@ -258,7 +258,7 @@ class Client(object): return self._post('/switches/%s/machines' % switch_id, data=data) def update_switch_machine(self, switch_id, machine_id, port=None, - vlans=None, ipmi_credentials=None, tag=None, + vlans=None, power_manage=None, tag=None, location=None, raw_data=None): data = {} if raw_data: @@ -270,8 +270,8 @@ class Client(object): if vlans: data['vlans'] = vlans - if ipmi_credentials: - data['ipmi_credentials'] = ipmi_credentials + if power_manage: + data['power_manage'] = power_manage if tag: data['tag'] = tag @@ -412,14 +412,14 @@ class Client(object): return self._get('/machines/%s' % machine_id, data=data) - def update_machine(self, machine_id, ipmi_credentials=None, tag=None, + def update_machine(self, machine_id, power_manage=None, tag=None, location=None, raw_data=None): data = {} if raw_data: data = raw_data else: - if ipmi_credentials: - data['ipmi_credentials'] = ipmi_credentials + if power_manage: + data['power_manage'] = power_manage if tag: data['tag'] = tag @@ -429,15 +429,15 @@ class Client(object): return self._put('/machines/%s' % machine_id, data=data) - def patch_machine(self, machine_id, ipmi_credentials=None, + def patch_machine(self, machine_id, power_manage=None, tag=None, location=None, raw_data=None): data = {} if raw_data: data = raw_data else: - if ipmi_credentials: - data['ipmi_credentials'] = ipmi_credentials + if power_manage: + data['power_manage'] = power_manage if tag: data['tag'] = tag diff --git a/compass-deck/bin/manage_db.py b/compass-deck/bin/manage_db.py index 3e56433..2b1f035 100755 --- a/compass-deck/bin/manage_db.py +++ b/compass-deck/bin/manage_db.py @@ -23,14 +23,17 @@ import sys current_dir = os.path.dirname(os.path.realpath(__file__)) sys.path.append(current_dir) - import switch_virtualenv -from flask.ext.script import Manager +import yaml + +from flask_script import Manager from compass.api import app from compass.db.api import database from compass.db.api import switch as switch_api +from compass.db.api import machine as machine_api +from compass.db.api import network as network_api from compass.db.api import user as user_api from compass.utils import flags from compass.utils import logsetting @@ -57,6 +60,13 @@ flags.add('switch_machines_file', 'or switch,<switch_ip>,<switch_vendor>,' '<switch_version>,<switch_community>,<switch_state>'), default='') +flags.add('machine_file', + help=( + 'file for add machine ' + 'contains one or more mac address of a host ' + 'if it is a baremetal machine, ipmi credential is also ' + 'required.'), + default='') flags.add('search_cluster_properties', help='comma separated properties to search in cluster config', default='') @@ -159,6 +169,48 @@ def set_switch_machines(): ) +@app_manager.command +def set_machine(): + if not flags.OPTIONS.machine_file: + print 'flag --machine_file is missing' + return + database.init() + machine_file = flags.OPTIONS.machine_file + user = user_api.get_user_object( + setting.COMPASS_ADMIN_EMAIL + ) + with open(machine_file) as f: + machine_data = yaml.load(f) + for machine in machine_data: + power_manage = {} + power_manage.update( + {"ip": machine.get("power_ip", "")}) + power_manage.update( + {"username": machine.get("power_user", "")}) + power_manage.update( + {"password": machine.get("power_pass", "")}) + machine_api.add_machine(user=user, mac=machine["mac"], + power_type=machine["power_type"], + power_manage=power_manage) + + +@app_manager.command +def add_subnet(): + if not flags.OPTIONS.subnet: + print 'flag --subnet is missing' + return + database.init() + subnet_tuple = flags.OPTIONS.subnet + subnet_name = subnet_tuple[0] + subnet_cidr = subnet_tuple[1] + user = user_api.get_user_object( + setting.COMPASS_ADMIN_EMAIL + ) + network_api.add_subnet( + user=user, name=subnet_name, subnet=subnet_cidr + ) + + if __name__ == "__main__": flags.init() logsetting.init() diff --git a/compass-deck/db/api/cluster.py b/compass-deck/db/api/cluster.py index 7a7022c..82bcaab 100644 --- a/compass-deck/db/api/cluster.py +++ b/compass-deck/db/api/cluster.py @@ -48,9 +48,9 @@ RESP_CLUSTERHOST_FIELDS = [ 'cluster_id', 'clustername', 'location', 'tag', 'networks', 'mac', 'switch_ip', 'port', 'switches', 'os_installed', 'distributed_system_installed', - 'os_name', 'os_id', 'ip', - 'reinstall_os', 'reinstall_distributed_system', - 'owner', 'cluster_id', + 'os_name', 'os_id', 'ip', 'reinstall_os', + 'reinstall_distributed_system', 'owner', 'cluster_id', + 'power_type', 'power_manage', 'created_at', 'updated_at', 'patched_roles' ] diff --git a/compass-deck/db/api/host.py b/compass-deck/db/api/host.py index 15e0bb6..61e3ab2 100644 --- a/compass-deck/db/api/host.py +++ b/compass-deck/db/api/host.py @@ -16,6 +16,7 @@ import functools import logging import netaddr +import ipaddress import re from compass.db.api import database @@ -23,6 +24,7 @@ from compass.db.api import metadata_holder as metadata_api from compass.db.api import permission from compass.db.api import user as user_api from compass.db.api import utils +from compass.db.api import network from compass.db import exception from compass.db import models from compass.utils import util @@ -642,6 +644,35 @@ def get_hostnetwork(host_network_id, user=None, session=None, **kwargs): return _get_hostnetwork(host_network_id, session=session) +def check_ip_available(subnet, ip): + if not subnet.reserved_range: + return + ip_int = int(ipaddress.IPv4Address(ip.decode())) + reserved_ranges = [] + reserved_ips = [] + for item in subnet.reserved_range.split(','): + ip_ends = item.split('-') + if len(ip_ends) == 2: + reserved_ranges.append(item) + elif len(ip_ends) == 1: + reserved_ips.append(item) + for item in reserved_ranges: + ends = item.split('-') + check_1 = int(ipaddress.IPv4Address(ends[0].decode())) - ip_int + check_2 = int(ipaddress.IPv4Address(ends[1].decode())) - ip_int + if (check_1 > 0) ^ (check_2 > 0): + raise exception.Forbidden( + 'IP %s is reserved, reserved range: %s' + % (ip, subnet.reserved_range) + ) + for item in reserved_ips: + if ip_int == int(ipaddress.IPv4Address(item.decode())): + raise exception.Forbidden( + 'IP %s is reserved, reserved range: %s' + % (ip, subnet.reserved_range) + ) + + @utils.supported_filters( ADDED_NETWORK_FIELDS, optional_support_keys=OPTIONAL_ADDED_NETWORK_FIELDS, @@ -652,17 +683,20 @@ def get_hostnetwork(host_network_id, user=None, session=None, **kwargs): ) @utils.wrap_to_dict(RESP_NETWORK_FIELDS) def _add_host_network( - host_id, exception_when_existing=True, - session=None, user=None, interface=None, ip=None, **kwargs + host_id, exception_when_existing=True, session=None, + user=None, interface=None, ip=None, subnet_id=None, **kwargs ): """Add hostnetwork to a host.""" host = _get_host(host_id, session=session) check_host_editable(host, user=user) + subnet = network.get_subnet_internal(subnet_id, session=session) + check_ip_available(subnet, ip) user_id = user.id return utils.add_db_object( session, models.HostNetwork, exception_when_existing, - host.id, interface, user_id, ip=ip, **kwargs + host.id, interface, user_id, + ip=ip, subnet_id=subnet_id, **kwargs ) @@ -671,14 +705,13 @@ def _add_host_network( permission.PERMISSION_ADD_HOST_NETWORK ) def add_host_network( - host_id, exception_when_existing=True, - interface=None, user=None, session=None, **kwargs + host_id, exception_when_existing=True, interface=None, + user=None, session=None, subnet_id=None, **kwargs ): """Create a hostnetwork to a host.""" return _add_host_network( - host_id, - exception_when_existing, - interface=interface, session=session, user=user, **kwargs + host_id, exception_when_existing, interface=interface, + user=user, session=session, subnet_id=subnet_id, **kwargs ) @@ -747,6 +780,8 @@ def _update_host_network( ): """Update host network.""" check_host_editable(host_network.host, user=user) + subnet = network.get_subnet_internal(host_network.subnet_id, session=session) + check_ip_available(subnet, ip) return utils.update_db_object(session, host_network, **kwargs) diff --git a/compass-deck/db/api/machine.py b/compass-deck/db/api/machine.py index b7b16b2..5b3cea0 100644 --- a/compass-deck/db/api/machine.py +++ b/compass-deck/db/api/machine.py @@ -29,20 +29,20 @@ from compass.utils import util MACHINE_PRIMARY_FILEDS = ['mac', 'owner_id'] SUPPORTED_FIELDS = [ - 'mac', 'tag', 'location', - 'machine_attributes', 'owner_id'] + 'mac', 'tag', 'location', 'power_manage', + 'machine_attributes', 'owner_id', 'power_type'] IGNORE_FIELDS = ['id', 'created_at', 'updated_at'] UPDATED_FIELDS = [ - 'ipmi_credentials', 'machine_attributes', - 'tag', 'location'] + 'mac', 'tag', 'location', 'power_manage', + 'machine_attributes', 'power_type'] PATCHED_FIELDS = [ - 'patched_ipmi_credentials', 'patched_tag', + 'patched_power_manage', 'patched_tag', 'patched_location' ] RESP_FIELDS = [ - 'id', 'mac', 'ipmi_credentials', 'switches', 'switch_ip', + 'id', 'mac', 'power_manage', 'switches', 'switch_ip', 'port', 'vlans', 'machine_attributes', 'owner_id', - 'tag', 'location', 'created_at', 'updated_at' + 'tag', 'location', 'power_type', 'created_at', 'updated_at' ] RESP_DEPLOY_FIELDS = [ 'status', 'machine' @@ -68,7 +68,7 @@ def _get_machine(machine_id, session=None, **kwargs): @utils.input_validates(mac=utils.check_mac) def _add_machine(mac, owner_id=None, session=None, **kwargs): """Add a machine.""" - if isinstance(owner_id, (int, long)): + if not owner_id or isinstance(owner_id, (int, long)): return utils.add_db_object( session, models.Machine, True, @@ -149,7 +149,7 @@ def _update_machine(machine_id, session=None, **kwargs): optional_support_keys=UPDATED_FIELDS, ignore_support_keys=IGNORE_FIELDS ) -@utils.input_validates(ipmi_credentials=utils.check_ipmi_credentials) +@utils.input_validates(power_manage=utils.check_power_manage) @database.run_in_session() @user_api.check_user_permission( permission.PERMISSION_ADD_MACHINE @@ -161,11 +161,11 @@ def update_machine(machine_id, user=None, session=None, **kwargs): ) -# replace [ipmi_credentials, tag, location] to -# [patched_ipmi_credentials, patched_tag, patched_location] +# replace [power_manage, tag, location] to +# [patched_power_manage, patched_tag, patched_location] # in kwargs. It tells db these fields will be patched. @utils.replace_filters( - ipmi_credentials='patched_ipmi_credentials', + power_manage='patched_power_manage', tag='patched_tag', location='patched_location' ) @@ -174,7 +174,7 @@ def update_machine(machine_id, user=None, session=None, **kwargs): ignore_support_keys=IGNORE_FIELDS ) @database.run_in_session() -@utils.output_validates(ipmi_credentials=utils.check_ipmi_credentials) +@utils.output_validates(power_manage=utils.check_power_manage) @user_api.check_user_permission( permission.PERMISSION_ADD_MACHINE ) diff --git a/compass-deck/db/api/network.py b/compass-deck/db/api/network.py index e2bf7d3..763b0b3 100644 --- a/compass-deck/db/api/network.py +++ b/compass-deck/db/api/network.py @@ -15,6 +15,7 @@ """Network related database operations.""" import logging import netaddr +import ipaddress import re from compass.db.api import database @@ -25,16 +26,17 @@ from compass.db import exception from compass.db import models -SUPPORTED_FIELDS = ['subnet', 'name'] +SUPPORTED_FIELDS = ['subnet', 'name', 'gateway'] RESP_FIELDS = [ - 'id', 'name', 'subnet', 'created_at', 'updated_at' + 'id', 'name', 'subnet', 'gateway', 'created_at', + 'updated_at', 'reserved_range' ] ADDED_FIELDS = ['subnet'] -OPTIONAL_ADDED_FIELDS = ['name'] +OPTIONAL_ADDED_FIELDS = ['name', 'gateway', 'reserved_range'] IGNORE_FIELDS = [ 'id', 'created_at', 'updated_at' ] -UPDATED_FIELDS = ['subnet', 'name'] +UPDATED_FIELDS = ['subnet', 'name', 'gateway', 'reserved_range'] def _check_subnet(subnet): @@ -47,6 +49,29 @@ def _check_subnet(subnet): 'subnet %s format unrecognized' % subnet) +def _check_ip_range(ip_ranges): + """Check if the ip range is valid. + The valid range can be a range or individual ips. + Range should be two ips jointed with "-", different ip + ranges and ips should be separated by "," + e.g. "10.1.0.0-10.1.0.50, 10.1.0.60" + """ + for ip_range in ip_ranges.split(','): + ip_ends = ip_range.split('-') + try: + ipaddress.IPv4Address(ip_ends[0].decode()) + if len(ip_ends) == 2: + ipaddress.IPv4Address(ip_ends[1].decode()) + except Exception as error: + logging.exception(error) + raise exception.InvalidParameter( + 'ip range %s format unrecognized' % ip_ranges) + finally: + if len(ip_ends) > 2: + raise exception.InvalidParameter( + 'ip range %s format unrecognized' % ip_ranges) + + @utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS) @database.run_in_session() @user_api.check_user_permission( @@ -72,6 +97,11 @@ def _get_subnet(subnet_id, session=None, **kwargs): ) +def get_subnet_internal(subnet_id, session=None, **kwargs): + """"Helper function to get subnet.""" + return _get_subnet(subnet_id=subnet_id, session=session, **kwargs) + + @utils.supported_filters([]) @database.run_in_session() @user_api.check_user_permission( @@ -93,7 +123,7 @@ def get_subnet( ADDED_FIELDS, optional_support_keys=OPTIONAL_ADDED_FIELDS, ignore_support_keys=IGNORE_FIELDS ) -@utils.input_validates(subnet=_check_subnet) +@utils.input_validates(subnet=_check_subnet, reserved_range=_check_ip_range) @database.run_in_session() @user_api.check_user_permission( permission.PERMISSION_ADD_SUBNET @@ -114,7 +144,7 @@ def add_subnet( optional_support_keys=UPDATED_FIELDS, ignore_support_keys=IGNORE_FIELDS ) -@utils.input_validates(subnet=_check_subnet) +@utils.input_validates(subnet=_check_subnet, reserved_range=_check_ip_range) @database.run_in_session() @user_api.check_user_permission( permission.PERMISSION_ADD_SUBNET diff --git a/compass-deck/db/api/switch.py b/compass-deck/db/api/switch.py index 647eec0..92d7a7f 100644 --- a/compass-deck/db/api/switch.py +++ b/compass-deck/db/api/switch.py @@ -58,17 +58,17 @@ UPDATED_FILTERS_FIELDS = ['put_machine_filters'] PATCHED_FILTERS_FIELDS = ['patched_machine_filters'] ADDED_MACHINES_FIELDS = ['mac'] OPTIONAL_ADDED_MACHINES_FIELDS = [ - 'ipmi_credentials', 'tag', 'location', 'owner_id' + 'power_manage', 'tag', 'location', 'owner_id' ] ADDED_SWITCH_MACHINES_FIELDS = ['port'] OPTIONAL_ADDED_SWITCH_MACHINES_FIELDS = ['vlans'] UPDATED_MACHINES_FIELDS = [ - 'ipmi_credentials', + 'power_manage', 'tag', 'location' ] UPDATED_SWITCH_MACHINES_FIELDS = ['port', 'vlans', 'owner_id'] PATCHED_MACHINES_FIELDS = [ - 'patched_ipmi_credentials', + 'patched_power_manage', 'patched_tag', 'patched_location' ] PATCHED_SWITCH_MACHINES_FIELDS = ['patched_vlans'] @@ -85,13 +85,13 @@ RESP_ACTION_FIELDS = [ RESP_MACHINES_FIELDS = [ 'id', 'switch_id', 'switch_ip', 'machine_id', 'switch_machine_id', 'port', 'vlans', 'mac', 'owner_id', - 'ipmi_credentials', 'tag', 'location', + 'power_manage', 'tag', 'location', 'created_at', 'updated_at' ] RESP_MACHINES_HOSTS_FIELDS = [ 'id', 'switch_id', 'switch_ip', 'machine_id', 'switch_machine_id', 'port', 'vlans', 'mac', - 'ipmi_credentials', 'tag', 'location', 'ip', + 'power_manage', 'tag', 'location', 'ip', 'name', 'hostname', 'os_name', 'owner', 'os_installer', 'reinstall_os', 'os_installed', 'clusters', 'created_at', 'updated_at' @@ -988,13 +988,13 @@ def update_switchmachine(switch_machine_id, user=None, session=None, **kwargs): ) -# replace [vlans, ipmi_credentials, tag, location] to -# [patched_vlans, patched_ipmi_credentials, patched_tag, +# replace [vlans, power_manage, tag, location] to +# [patched_vlans, patched_power_manage, patched_tag, # patched_location] in kwargs. It tells db these fields will # be patched. @utils.replace_filters( vlans='patched_vlans', - ipmi_credentials='patched_ipmi_credentials', + power_manage='patched_power_manage', tag='patched_tag', location='patched_location' ) @@ -1024,13 +1024,13 @@ def patch_switch_machine( ) -# replace [vlans, ipmi_credentials, tag, location] to -# [patched_vlans, patched_ipmi_credentials, patched_tag, +# replace [vlans, power_manage, tag, location] to +# [patched_vlans, patched_power_manage, patched_tag, # patched_location] in kwargs. It tells db these fields will # be patched. @utils.replace_filters( vlans='patched_vlans', - ipmi_credentials='patched_ipmi_credentials', + power_manage='patched_power_manage', tag='patched_tag', location='patched_location' ) diff --git a/compass-deck/db/api/user.py b/compass-deck/db/api/user.py index db039eb..0105797 100644 --- a/compass-deck/db/api/user.py +++ b/compass-deck/db/api/user.py @@ -18,7 +18,7 @@ import functools import logging import re -from flask.ext.login import UserMixin +from flask_login import UserMixin from compass.db.api import database from compass.db.api import utils diff --git a/compass-deck/db/api/utils.py b/compass-deck/db/api/utils.py index a44f26e..8921b4a 100644 --- a/compass-deck/db/api/utils.py +++ b/compass-deck/db/api/utils.py @@ -1208,42 +1208,42 @@ def check_name(name): ) -def _check_ipmi_credentials_ip(ip): +def _check_power_manage_ip(ip): check_ip(ip) -def check_ipmi_credentials(ipmi_credentials): - """Check ipmi credentials format is correct.""" - if not ipmi_credentials: +def check_power_manage(power_manage): + """Check power manage format is correct.""" + if not power_manage: return - if not isinstance(ipmi_credentials, dict): + if not isinstance(power_manage, dict): raise exception.InvalidParameter( - 'invalid ipmi credentials %s' % ipmi_credentials + 'invalid power manage %s' % power_manage ) - for key in ipmi_credentials: + for key in power_manage: if key not in ['ip', 'username', 'password']: raise exception.InvalidParameter( - 'unrecognized field %s in ipmi credentials %s' % ( - key, ipmi_credentials + 'unrecognized field %s in power manage %s' % ( + key, power_manage ) ) for key in ['ip', 'username', 'password']: - if key not in ipmi_credentials: + if key not in power_manage: raise exception.InvalidParameter( - 'no field %s in ipmi credentials %s' % ( - key, ipmi_credentials + 'no field %s in power manage %s' % ( + key, power_manage ) ) - check_ipmi_credential_field = '_check_ipmi_credentials_%s' % key + check_power_manage_field = '_check_power_manage_%s' % key this_module = globals() - if check_ipmi_credential_field in this_module: - this_module[check_ipmi_credential_field]( - ipmi_credentials[key] + if check_power_manage_field in this_module: + this_module[check_power_manage_field]( + power_manage[key] ) else: logging.debug( - 'function %s is not defined', check_ipmi_credential_field + 'function %s is not defined', check_power_manage_field ) diff --git a/compass-deck/db/models.py b/compass-deck/db/models.py index d4b0324..124e35a 100644 --- a/compass-deck/db/models.py +++ b/compass-deck/db/models.py @@ -1532,11 +1532,12 @@ class Machine(BASE, HelperMixin, TimestampMixin): """Machine table.""" __tablename__ = 'machine' id = Column(Integer, primary_key=True) - mac = Column(String(24), unique=True, nullable=False) - ipmi_credentials = Column(JSONEncoded, default={}) + mac = Column(JSONEncoded, nullable=False) tag = Column(JSONEncoded, default={}) location = Column(JSONEncoded, default={}) - owner_id = Column(Integer, ForeignKey('user.id')) + owner_id = Column(Integer, nullable=True) + power_type = Column(String(10), default="ipmilan") + power_manage = Column(JSONEncoded, default={}) machine_attributes = Column(JSONEncoded, default={}) switch_machines = relationship( @@ -1564,22 +1565,23 @@ class Machine(BASE, HelperMixin, TimestampMixin): # TODO(xicheng): some validation can be moved to column. super(Machine, self).validate() try: - netaddr.EUI(self.mac) + for key, value in self.mac.items(): + netaddr.EUI(value) except Exception: raise exception.InvalidParameter( 'mac address %s format uncorrect' % self.mac ) @property - def patched_ipmi_credentials(self): - return self.ipmi_credentials + def patched_power_manage(self): + return self.power_manage - @patched_ipmi_credentials.setter - def patched_ipmi_credentials(self, value): + @patched_power_manage.setter + def patched_power_manage(self, value): if not value: return - ipmi_credentials = copy.deepcopy(self.ipmi_credentials) - self.ipmi_credentials = util.merge_dict(ipmi_credentials, value) + power_manage = copy.deepcopy(self.power_manage) + self.power_manage = util.merge_dict(power_manage, value) @property def patched_tag(self): @@ -1863,6 +1865,8 @@ class Subnet(BASE, TimestampMixin, HelperMixin): id = Column(Integer, primary_key=True) name = Column(String(80), unique=True, nullable=True) subnet = Column(String(80), unique=True, nullable=False) + gateway = Column(String(80), unique=True, nullable=True) + reserved_range = Column(String(80), unique=False, nullable=True) host_networks = relationship( HostNetwork, diff --git a/compass-deck/db/v1/model.py b/compass-deck/db/v1/model.py index d74e355..f84557c 100644 --- a/compass-deck/db/v1/model.py +++ b/compass-deck/db/v1/model.py @@ -28,7 +28,7 @@ from sqlalchemy.ext.hybrid import hybrid_property from compass.utils import util -from flask.ext.login import UserMixin +from flask_login import UserMixin from itsdangerous import URLSafeTimedSerializer BASE = declarative_base() diff --git a/compass-deck/requirements.txt b/compass-deck/requirements.txt index 6a3b3c7..caf96ea 100644 --- a/compass-deck/requirements.txt +++ b/compass-deck/requirements.txt @@ -1,6 +1,6 @@ amqplib argparse -celery +celery<=4.1.0 Markdown<2.5 Cheetah<=2.4.1 daemon @@ -22,3 +22,4 @@ python-daemon==2.1.1 SQLAlchemy>=0.9.0 simplejson requests +pyyaml diff --git a/compass-tasks-base/apiclient/restful.py b/compass-tasks-base/apiclient/restful.py index bb82922..f5d4855 100644 --- a/compass-tasks-base/apiclient/restful.py +++ b/compass-tasks-base/apiclient/restful.py @@ -231,7 +231,7 @@ class Client(object): return self._get('/switches/%s/machines-hosts' % switch_id, data=data) def add_switch_machine(self, switch_id, mac=None, port=None, - vlans=None, ipmi_credentials=None, + vlans=None, power_manage=None, tag=None, location=None, raw_data=None): data = {} if raw_data: @@ -246,8 +246,8 @@ class Client(object): if vlans: data['vlans'] = vlans - if ipmi_credentials: - data['ipmi_credentials'] = ipmi_credentials + if power_manage: + data['power_manage'] = power_manage if tag: data['tag'] = tag @@ -258,7 +258,7 @@ class Client(object): return self._post('/switches/%s/machines' % switch_id, data=data) def update_switch_machine(self, switch_id, machine_id, port=None, - vlans=None, ipmi_credentials=None, tag=None, + vlans=None, power_manage=None, tag=None, location=None, raw_data=None): data = {} if raw_data: @@ -270,8 +270,8 @@ class Client(object): if vlans: data['vlans'] = vlans - if ipmi_credentials: - data['ipmi_credentials'] = ipmi_credentials + if power_manage: + data['power_manage'] = power_manage if tag: data['tag'] = tag @@ -412,14 +412,14 @@ class Client(object): return self._get('/machines/%s' % machine_id, data=data) - def update_machine(self, machine_id, ipmi_credentials=None, tag=None, + def update_machine(self, machine_id, power_manage=None, tag=None, location=None, raw_data=None): data = {} if raw_data: data = raw_data else: - if ipmi_credentials: - data['ipmi_credentials'] = ipmi_credentials + if power_manage: + data['power_manage'] = power_manage if tag: data['tag'] = tag @@ -429,15 +429,15 @@ class Client(object): return self._put('/machines/%s' % machine_id, data=data) - def patch_machine(self, machine_id, ipmi_credentials=None, + def patch_machine(self, machine_id, power_manage=None, tag=None, location=None, raw_data=None): data = {} if raw_data: data = raw_data else: - if ipmi_credentials: - data['ipmi_credentials'] = ipmi_credentials + if power_manage: + data['power_manage'] = power_manage if tag: data['tag'] = tag diff --git a/compass-tasks-base/build.sh b/compass-tasks-base/build.sh index 81f3c21..6071734 100755 --- a/compass-tasks-base/build.sh +++ b/compass-tasks-base/build.sh @@ -42,6 +42,7 @@ pip install --upgrade Flask pip install --upgrade virtualenvwrapper source `which virtualenvwrapper.sh` +echo "source /usr/bin/virtualenvwrapper.sh" >> ~/.bashrc mkvirtualenv --system-site-packages compass-core workon compass-core cd $COMPASS_DIR diff --git a/compass-tasks-base/db/api/cluster.py b/compass-tasks-base/db/api/cluster.py index 7a7022c..82bcaab 100644 --- a/compass-tasks-base/db/api/cluster.py +++ b/compass-tasks-base/db/api/cluster.py @@ -48,9 +48,9 @@ RESP_CLUSTERHOST_FIELDS = [ 'cluster_id', 'clustername', 'location', 'tag', 'networks', 'mac', 'switch_ip', 'port', 'switches', 'os_installed', 'distributed_system_installed', - 'os_name', 'os_id', 'ip', - 'reinstall_os', 'reinstall_distributed_system', - 'owner', 'cluster_id', + 'os_name', 'os_id', 'ip', 'reinstall_os', + 'reinstall_distributed_system', 'owner', 'cluster_id', + 'power_type', 'power_manage', 'created_at', 'updated_at', 'patched_roles' ] diff --git a/compass-tasks-base/db/api/host.py b/compass-tasks-base/db/api/host.py index 15e0bb6..61e3ab2 100644 --- a/compass-tasks-base/db/api/host.py +++ b/compass-tasks-base/db/api/host.py @@ -16,6 +16,7 @@ import functools import logging import netaddr +import ipaddress import re from compass.db.api import database @@ -23,6 +24,7 @@ from compass.db.api import metadata_holder as metadata_api from compass.db.api import permission from compass.db.api import user as user_api from compass.db.api import utils +from compass.db.api import network from compass.db import exception from compass.db import models from compass.utils import util @@ -642,6 +644,35 @@ def get_hostnetwork(host_network_id, user=None, session=None, **kwargs): return _get_hostnetwork(host_network_id, session=session) +def check_ip_available(subnet, ip): + if not subnet.reserved_range: + return + ip_int = int(ipaddress.IPv4Address(ip.decode())) + reserved_ranges = [] + reserved_ips = [] + for item in subnet.reserved_range.split(','): + ip_ends = item.split('-') + if len(ip_ends) == 2: + reserved_ranges.append(item) + elif len(ip_ends) == 1: + reserved_ips.append(item) + for item in reserved_ranges: + ends = item.split('-') + check_1 = int(ipaddress.IPv4Address(ends[0].decode())) - ip_int + check_2 = int(ipaddress.IPv4Address(ends[1].decode())) - ip_int + if (check_1 > 0) ^ (check_2 > 0): + raise exception.Forbidden( + 'IP %s is reserved, reserved range: %s' + % (ip, subnet.reserved_range) + ) + for item in reserved_ips: + if ip_int == int(ipaddress.IPv4Address(item.decode())): + raise exception.Forbidden( + 'IP %s is reserved, reserved range: %s' + % (ip, subnet.reserved_range) + ) + + @utils.supported_filters( ADDED_NETWORK_FIELDS, optional_support_keys=OPTIONAL_ADDED_NETWORK_FIELDS, @@ -652,17 +683,20 @@ def get_hostnetwork(host_network_id, user=None, session=None, **kwargs): ) @utils.wrap_to_dict(RESP_NETWORK_FIELDS) def _add_host_network( - host_id, exception_when_existing=True, - session=None, user=None, interface=None, ip=None, **kwargs + host_id, exception_when_existing=True, session=None, + user=None, interface=None, ip=None, subnet_id=None, **kwargs ): """Add hostnetwork to a host.""" host = _get_host(host_id, session=session) check_host_editable(host, user=user) + subnet = network.get_subnet_internal(subnet_id, session=session) + check_ip_available(subnet, ip) user_id = user.id return utils.add_db_object( session, models.HostNetwork, exception_when_existing, - host.id, interface, user_id, ip=ip, **kwargs + host.id, interface, user_id, + ip=ip, subnet_id=subnet_id, **kwargs ) @@ -671,14 +705,13 @@ def _add_host_network( permission.PERMISSION_ADD_HOST_NETWORK ) def add_host_network( - host_id, exception_when_existing=True, - interface=None, user=None, session=None, **kwargs + host_id, exception_when_existing=True, interface=None, + user=None, session=None, subnet_id=None, **kwargs ): """Create a hostnetwork to a host.""" return _add_host_network( - host_id, - exception_when_existing, - interface=interface, session=session, user=user, **kwargs + host_id, exception_when_existing, interface=interface, + user=user, session=session, subnet_id=subnet_id, **kwargs ) @@ -747,6 +780,8 @@ def _update_host_network( ): """Update host network.""" check_host_editable(host_network.host, user=user) + subnet = network.get_subnet_internal(host_network.subnet_id, session=session) + check_ip_available(subnet, ip) return utils.update_db_object(session, host_network, **kwargs) diff --git a/compass-tasks-base/db/api/machine.py b/compass-tasks-base/db/api/machine.py index b7b16b2..5b3cea0 100644 --- a/compass-tasks-base/db/api/machine.py +++ b/compass-tasks-base/db/api/machine.py @@ -29,20 +29,20 @@ from compass.utils import util MACHINE_PRIMARY_FILEDS = ['mac', 'owner_id'] SUPPORTED_FIELDS = [ - 'mac', 'tag', 'location', - 'machine_attributes', 'owner_id'] + 'mac', 'tag', 'location', 'power_manage', + 'machine_attributes', 'owner_id', 'power_type'] IGNORE_FIELDS = ['id', 'created_at', 'updated_at'] UPDATED_FIELDS = [ - 'ipmi_credentials', 'machine_attributes', - 'tag', 'location'] + 'mac', 'tag', 'location', 'power_manage', + 'machine_attributes', 'power_type'] PATCHED_FIELDS = [ - 'patched_ipmi_credentials', 'patched_tag', + 'patched_power_manage', 'patched_tag', 'patched_location' ] RESP_FIELDS = [ - 'id', 'mac', 'ipmi_credentials', 'switches', 'switch_ip', + 'id', 'mac', 'power_manage', 'switches', 'switch_ip', 'port', 'vlans', 'machine_attributes', 'owner_id', - 'tag', 'location', 'created_at', 'updated_at' + 'tag', 'location', 'power_type', 'created_at', 'updated_at' ] RESP_DEPLOY_FIELDS = [ 'status', 'machine' @@ -68,7 +68,7 @@ def _get_machine(machine_id, session=None, **kwargs): @utils.input_validates(mac=utils.check_mac) def _add_machine(mac, owner_id=None, session=None, **kwargs): """Add a machine.""" - if isinstance(owner_id, (int, long)): + if not owner_id or isinstance(owner_id, (int, long)): return utils.add_db_object( session, models.Machine, True, @@ -149,7 +149,7 @@ def _update_machine(machine_id, session=None, **kwargs): optional_support_keys=UPDATED_FIELDS, ignore_support_keys=IGNORE_FIELDS ) -@utils.input_validates(ipmi_credentials=utils.check_ipmi_credentials) +@utils.input_validates(power_manage=utils.check_power_manage) @database.run_in_session() @user_api.check_user_permission( permission.PERMISSION_ADD_MACHINE @@ -161,11 +161,11 @@ def update_machine(machine_id, user=None, session=None, **kwargs): ) -# replace [ipmi_credentials, tag, location] to -# [patched_ipmi_credentials, patched_tag, patched_location] +# replace [power_manage, tag, location] to +# [patched_power_manage, patched_tag, patched_location] # in kwargs. It tells db these fields will be patched. @utils.replace_filters( - ipmi_credentials='patched_ipmi_credentials', + power_manage='patched_power_manage', tag='patched_tag', location='patched_location' ) @@ -174,7 +174,7 @@ def update_machine(machine_id, user=None, session=None, **kwargs): ignore_support_keys=IGNORE_FIELDS ) @database.run_in_session() -@utils.output_validates(ipmi_credentials=utils.check_ipmi_credentials) +@utils.output_validates(power_manage=utils.check_power_manage) @user_api.check_user_permission( permission.PERMISSION_ADD_MACHINE ) diff --git a/compass-tasks-base/db/api/network.py b/compass-tasks-base/db/api/network.py index e2bf7d3..763b0b3 100644 --- a/compass-tasks-base/db/api/network.py +++ b/compass-tasks-base/db/api/network.py @@ -15,6 +15,7 @@ """Network related database operations.""" import logging import netaddr +import ipaddress import re from compass.db.api import database @@ -25,16 +26,17 @@ from compass.db import exception from compass.db import models -SUPPORTED_FIELDS = ['subnet', 'name'] +SUPPORTED_FIELDS = ['subnet', 'name', 'gateway'] RESP_FIELDS = [ - 'id', 'name', 'subnet', 'created_at', 'updated_at' + 'id', 'name', 'subnet', 'gateway', 'created_at', + 'updated_at', 'reserved_range' ] ADDED_FIELDS = ['subnet'] -OPTIONAL_ADDED_FIELDS = ['name'] +OPTIONAL_ADDED_FIELDS = ['name', 'gateway', 'reserved_range'] IGNORE_FIELDS = [ 'id', 'created_at', 'updated_at' ] -UPDATED_FIELDS = ['subnet', 'name'] +UPDATED_FIELDS = ['subnet', 'name', 'gateway', 'reserved_range'] def _check_subnet(subnet): @@ -47,6 +49,29 @@ def _check_subnet(subnet): 'subnet %s format unrecognized' % subnet) +def _check_ip_range(ip_ranges): + """Check if the ip range is valid. + The valid range can be a range or individual ips. + Range should be two ips jointed with "-", different ip + ranges and ips should be separated by "," + e.g. "10.1.0.0-10.1.0.50, 10.1.0.60" + """ + for ip_range in ip_ranges.split(','): + ip_ends = ip_range.split('-') + try: + ipaddress.IPv4Address(ip_ends[0].decode()) + if len(ip_ends) == 2: + ipaddress.IPv4Address(ip_ends[1].decode()) + except Exception as error: + logging.exception(error) + raise exception.InvalidParameter( + 'ip range %s format unrecognized' % ip_ranges) + finally: + if len(ip_ends) > 2: + raise exception.InvalidParameter( + 'ip range %s format unrecognized' % ip_ranges) + + @utils.supported_filters(optional_support_keys=SUPPORTED_FIELDS) @database.run_in_session() @user_api.check_user_permission( @@ -72,6 +97,11 @@ def _get_subnet(subnet_id, session=None, **kwargs): ) +def get_subnet_internal(subnet_id, session=None, **kwargs): + """"Helper function to get subnet.""" + return _get_subnet(subnet_id=subnet_id, session=session, **kwargs) + + @utils.supported_filters([]) @database.run_in_session() @user_api.check_user_permission( @@ -93,7 +123,7 @@ def get_subnet( ADDED_FIELDS, optional_support_keys=OPTIONAL_ADDED_FIELDS, ignore_support_keys=IGNORE_FIELDS ) -@utils.input_validates(subnet=_check_subnet) +@utils.input_validates(subnet=_check_subnet, reserved_range=_check_ip_range) @database.run_in_session() @user_api.check_user_permission( permission.PERMISSION_ADD_SUBNET @@ -114,7 +144,7 @@ def add_subnet( optional_support_keys=UPDATED_FIELDS, ignore_support_keys=IGNORE_FIELDS ) -@utils.input_validates(subnet=_check_subnet) +@utils.input_validates(subnet=_check_subnet, reserved_range=_check_ip_range) @database.run_in_session() @user_api.check_user_permission( permission.PERMISSION_ADD_SUBNET diff --git a/compass-tasks-base/db/api/switch.py b/compass-tasks-base/db/api/switch.py index 647eec0..92d7a7f 100644 --- a/compass-tasks-base/db/api/switch.py +++ b/compass-tasks-base/db/api/switch.py @@ -58,17 +58,17 @@ UPDATED_FILTERS_FIELDS = ['put_machine_filters'] PATCHED_FILTERS_FIELDS = ['patched_machine_filters'] ADDED_MACHINES_FIELDS = ['mac'] OPTIONAL_ADDED_MACHINES_FIELDS = [ - 'ipmi_credentials', 'tag', 'location', 'owner_id' + 'power_manage', 'tag', 'location', 'owner_id' ] ADDED_SWITCH_MACHINES_FIELDS = ['port'] OPTIONAL_ADDED_SWITCH_MACHINES_FIELDS = ['vlans'] UPDATED_MACHINES_FIELDS = [ - 'ipmi_credentials', + 'power_manage', 'tag', 'location' ] UPDATED_SWITCH_MACHINES_FIELDS = ['port', 'vlans', 'owner_id'] PATCHED_MACHINES_FIELDS = [ - 'patched_ipmi_credentials', + 'patched_power_manage', 'patched_tag', 'patched_location' ] PATCHED_SWITCH_MACHINES_FIELDS = ['patched_vlans'] @@ -85,13 +85,13 @@ RESP_ACTION_FIELDS = [ RESP_MACHINES_FIELDS = [ 'id', 'switch_id', 'switch_ip', 'machine_id', 'switch_machine_id', 'port', 'vlans', 'mac', 'owner_id', - 'ipmi_credentials', 'tag', 'location', + 'power_manage', 'tag', 'location', 'created_at', 'updated_at' ] RESP_MACHINES_HOSTS_FIELDS = [ 'id', 'switch_id', 'switch_ip', 'machine_id', 'switch_machine_id', 'port', 'vlans', 'mac', - 'ipmi_credentials', 'tag', 'location', 'ip', + 'power_manage', 'tag', 'location', 'ip', 'name', 'hostname', 'os_name', 'owner', 'os_installer', 'reinstall_os', 'os_installed', 'clusters', 'created_at', 'updated_at' @@ -988,13 +988,13 @@ def update_switchmachine(switch_machine_id, user=None, session=None, **kwargs): ) -# replace [vlans, ipmi_credentials, tag, location] to -# [patched_vlans, patched_ipmi_credentials, patched_tag, +# replace [vlans, power_manage, tag, location] to +# [patched_vlans, patched_power_manage, patched_tag, # patched_location] in kwargs. It tells db these fields will # be patched. @utils.replace_filters( vlans='patched_vlans', - ipmi_credentials='patched_ipmi_credentials', + power_manage='patched_power_manage', tag='patched_tag', location='patched_location' ) @@ -1024,13 +1024,13 @@ def patch_switch_machine( ) -# replace [vlans, ipmi_credentials, tag, location] to -# [patched_vlans, patched_ipmi_credentials, patched_tag, +# replace [vlans, power_manage, tag, location] to +# [patched_vlans, patched_power_manage, patched_tag, # patched_location] in kwargs. It tells db these fields will # be patched. @utils.replace_filters( vlans='patched_vlans', - ipmi_credentials='patched_ipmi_credentials', + power_manage='patched_power_manage', tag='patched_tag', location='patched_location' ) diff --git a/compass-tasks-base/db/api/user.py b/compass-tasks-base/db/api/user.py index db039eb..0105797 100644 --- a/compass-tasks-base/db/api/user.py +++ b/compass-tasks-base/db/api/user.py @@ -18,7 +18,7 @@ import functools import logging import re -from flask.ext.login import UserMixin +from flask_login import UserMixin from compass.db.api import database from compass.db.api import utils diff --git a/compass-tasks-base/db/api/utils.py b/compass-tasks-base/db/api/utils.py index a44f26e..8921b4a 100644 --- a/compass-tasks-base/db/api/utils.py +++ b/compass-tasks-base/db/api/utils.py @@ -1208,42 +1208,42 @@ def check_name(name): ) -def _check_ipmi_credentials_ip(ip): +def _check_power_manage_ip(ip): check_ip(ip) -def check_ipmi_credentials(ipmi_credentials): - """Check ipmi credentials format is correct.""" - if not ipmi_credentials: +def check_power_manage(power_manage): + """Check power manage format is correct.""" + if not power_manage: return - if not isinstance(ipmi_credentials, dict): + if not isinstance(power_manage, dict): raise exception.InvalidParameter( - 'invalid ipmi credentials %s' % ipmi_credentials + 'invalid power manage %s' % power_manage ) - for key in ipmi_credentials: + for key in power_manage: if key not in ['ip', 'username', 'password']: raise exception.InvalidParameter( - 'unrecognized field %s in ipmi credentials %s' % ( - key, ipmi_credentials + 'unrecognized field %s in power manage %s' % ( + key, power_manage ) ) for key in ['ip', 'username', 'password']: - if key not in ipmi_credentials: + if key not in power_manage: raise exception.InvalidParameter( - 'no field %s in ipmi credentials %s' % ( - key, ipmi_credentials + 'no field %s in power manage %s' % ( + key, power_manage ) ) - check_ipmi_credential_field = '_check_ipmi_credentials_%s' % key + check_power_manage_field = '_check_power_manage_%s' % key this_module = globals() - if check_ipmi_credential_field in this_module: - this_module[check_ipmi_credential_field]( - ipmi_credentials[key] + if check_power_manage_field in this_module: + this_module[check_power_manage_field]( + power_manage[key] ) else: logging.debug( - 'function %s is not defined', check_ipmi_credential_field + 'function %s is not defined', check_power_manage_field ) diff --git a/compass-tasks-base/db/models.py b/compass-tasks-base/db/models.py index d4b0324..124e35a 100644 --- a/compass-tasks-base/db/models.py +++ b/compass-tasks-base/db/models.py @@ -1532,11 +1532,12 @@ class Machine(BASE, HelperMixin, TimestampMixin): """Machine table.""" __tablename__ = 'machine' id = Column(Integer, primary_key=True) - mac = Column(String(24), unique=True, nullable=False) - ipmi_credentials = Column(JSONEncoded, default={}) + mac = Column(JSONEncoded, nullable=False) tag = Column(JSONEncoded, default={}) location = Column(JSONEncoded, default={}) - owner_id = Column(Integer, ForeignKey('user.id')) + owner_id = Column(Integer, nullable=True) + power_type = Column(String(10), default="ipmilan") + power_manage = Column(JSONEncoded, default={}) machine_attributes = Column(JSONEncoded, default={}) switch_machines = relationship( @@ -1564,22 +1565,23 @@ class Machine(BASE, HelperMixin, TimestampMixin): # TODO(xicheng): some validation can be moved to column. super(Machine, self).validate() try: - netaddr.EUI(self.mac) + for key, value in self.mac.items(): + netaddr.EUI(value) except Exception: raise exception.InvalidParameter( 'mac address %s format uncorrect' % self.mac ) @property - def patched_ipmi_credentials(self): - return self.ipmi_credentials + def patched_power_manage(self): + return self.power_manage - @patched_ipmi_credentials.setter - def patched_ipmi_credentials(self, value): + @patched_power_manage.setter + def patched_power_manage(self, value): if not value: return - ipmi_credentials = copy.deepcopy(self.ipmi_credentials) - self.ipmi_credentials = util.merge_dict(ipmi_credentials, value) + power_manage = copy.deepcopy(self.power_manage) + self.power_manage = util.merge_dict(power_manage, value) @property def patched_tag(self): @@ -1863,6 +1865,8 @@ class Subnet(BASE, TimestampMixin, HelperMixin): id = Column(Integer, primary_key=True) name = Column(String(80), unique=True, nullable=True) subnet = Column(String(80), unique=True, nullable=False) + gateway = Column(String(80), unique=True, nullable=True) + reserved_range = Column(String(80), unique=False, nullable=True) host_networks = relationship( HostNetwork, diff --git a/compass-tasks-base/db/v1/model.py b/compass-tasks-base/db/v1/model.py index d74e355..f84557c 100644 --- a/compass-tasks-base/db/v1/model.py +++ b/compass-tasks-base/db/v1/model.py @@ -28,7 +28,7 @@ from sqlalchemy.ext.hybrid import hybrid_property from compass.utils import util -from flask.ext.login import UserMixin +from flask_login import UserMixin from itsdangerous import URLSafeTimedSerializer BASE = declarative_base() diff --git a/compass-tasks-base/deployment/deploy_manager.py b/compass-tasks-base/deployment/deploy_manager.py index baf7cd6..225c31e 100644 --- a/compass-tasks-base/deployment/deploy_manager.py +++ b/compass-tasks-base/deployment/deploy_manager.py @@ -111,7 +111,34 @@ class DeployManager(object): self.os_installer.set_package_installer_config(pk_installer_config) # start to deploy OS - return self.os_installer.deploy() + result = self.os_installer.deploy() + self.reset_server() + + return result + + def poweron_server(self): + if not self.os_installer: + return + + host_id_list = self.os_installer.config_manager.get_host_id_list() + for host_id in host_id_list: + self.os_installer.poweron(host_id) + + def poweroff_server(self): + if not self.os_installer: + return + + host_id_list = self.os_installer.config_manager.get_host_id_list() + for host_id in host_id_list: + self.os_installer.poweroff(host_id) + + def reset_server(self): + if not self.os_installer: + return + + host_id_list = self.os_installer.config_manager.get_host_id_list() + for host_id in host_id_list: + self.os_installer.reset(host_id) def deploy_target_system(self): """Deploy target system to all hosts in the cluster. diff --git a/compass-tasks-base/deployment/installers/config_manager.py b/compass-tasks-base/deployment/installers/config_manager.py index 597c3a6..ebee727 100644 --- a/compass-tasks-base/deployment/installers/config_manager.py +++ b/compass-tasks-base/deployment/installers/config_manager.py @@ -169,7 +169,8 @@ class HostInfo(object): self.package_config = self.host_info.setdefault(const.PK_CONFIG, {}) self.roles = self.host_info.setdefault(const.ROLES, []) self.patched_roles = self.host_info.setdefault(const.PATCHED_ROLES, []) - self.ipmi = deepcopy(self.host_info.setdefault(const.IPMI, {})) + self.power_type = deepcopy(self.host_info.setdefault(const.POWER_TYPE, {})) + self.power_manage = deepcopy(self.host_info.setdefault(const.POWER_MANAGE, {})) self.reinstall_os_flag = self.host_info.get(const.REINSTALL_OS_FLAG) self.deployed_os_config = self.host_info.setdefault( const.DEPLOYED_OS_CONFIG, {} @@ -275,6 +276,8 @@ class HostInfo(object): def baseinfo(self): return { const.REINSTALL_OS_FLAG: self.reinstall_os_flag, + const.POWER_TYPE: self.power_type, + const.POWER_MANAGE: self.power_manage, const.MAC_ADDR: self.mac, const.NAME: self.name, const.HOSTNAME: self.hostname, @@ -514,14 +517,15 @@ class BaseConfigManager(object): self.validate_host(host_id) return self.hosts_info[host_id].roles_mapping - def get_host_ipmi_info(self, host_id): + def get_host_power_info(self, host_id): self.validate_host(host_id) - if self.hosts_info[host_id].ipmi: + if self.hosts_info[host_id].power_manage: return ( - self.hosts_info[host_id].ipmi[const.IP_ADDR], - self.hosts_info[host_id].ipmi - [const.IPMI_CREDS][const.USERNAME], - self.hosts_info[host_id].ipmi - [const.IPMI_CREDS][const.USERNAME]) + self.hosts_info[host_id].power_manage + [const.IP_ADDR], + self.hosts_info[host_id].power_manage + [const.USERNAME], + self.hosts_info[host_id].power_manage + [const.PASSWORD]) else: return (None, None, None) diff --git a/compass-tasks-base/deployment/installers/os_installers/cobbler/cobbler.py b/compass-tasks-base/deployment/installers/os_installers/cobbler/cobbler.py index 9c2a935..aa74f68 100644 --- a/compass-tasks-base/deployment/installers/os_installers/cobbler/cobbler.py +++ b/compass-tasks-base/deployment/installers/os_installers/cobbler/cobbler.py @@ -397,25 +397,25 @@ class CobblerInstaller(OSInstaller): return cluster_vas_dict - def _check_and_set_system_impi(self, host_id, sys_id): + def _check_and_set_system_power(self, host_id, sys_id): if not sys_id: logging.info("System is None!") return False system = self.dump_system_info(host_id) - if system[self.POWER_TYPE] != 'ipmilan' or not system[self.POWER_USER]: + if not system.get(self.POWER_TYPE): # Set sytem power type to ipmilan if needs and set IPMI info - ipmi_info = self.config_manager.get_host_ipmi_info(host_id) - if not ipmi_info: + power_info = self.config_manager.get_host_power_info(host_id) + if not power_info: logging.info('No IPMI information found! Failed power on.') return False - ipmi_ip, ipmi_user, ipmi_pass = ipmi_info + ip, username, password = power_info power_opts = {} power_opts[self.POWER_TYPE] = 'ipmilan' - power_opts[self.POWER_ADDR] = ipmi_ip - power_opts[self.POWER_USER] = ipmi_user - power_opts[self.POWER_PASS] = ipmi_pass + power_opts[self.POWER_ADDR] = ip + power_opts[self.POWER_USER] = username + power_opts[self.POWER_PASS] = password self._update_system_config(sys_id, power_opts) @@ -424,26 +424,26 @@ class CobblerInstaller(OSInstaller): def poweron(self, host_id): hostname = self.config_manager.get_hostname(host_id) sys_id = self._get_create_system(hostname) - if not self._check_and_set_system_impi(sys_id): + if not self._check_and_set_system_power(host_id, sys_id): return - self.remote.power_system(sys_id, self.token, power='on') + self.remote.power_system(sys_id, 'on', self.token) logging.info("Host with ID=%d starts to power on!" % host_id) def poweroff(self, host_id): hostname = self.config_manager.get_hostname(host_id) sys_id = self._get_create_system(hostname) - if not self._check_and_set_system_impi(sys_id): + if not self._check_and_set_system_power(host_id, sys_id): return - self.remote.power_system(sys_id, self.token, power='off') + self.remote.power_system(sys_id, 'off', self.token) logging.info("Host with ID=%d starts to power off!" % host_id) def reset(self, host_id): hostname = self.config_manager.get_hostname(host_id) sys_id = self._get_create_system(hostname) - if not self._check_and_set_system_impi(sys_id): + if not self._check_and_set_system_power(host_id, sys_id): return - self.remote.power_system(sys_id, self.token, power='reboot') + self.remote.power_system(sys_id, 'reboot', self.token) logging.info("Host with ID=%d starts to reboot!" % host_id) diff --git a/compass-tasks-base/deployment/utils/constants.py b/compass-tasks-base/deployment/utils/constants.py index e90b1b2..18ee56a 100644 --- a/compass-tasks-base/deployment/utils/constants.py +++ b/compass-tasks-base/deployment/utils/constants.py @@ -54,8 +54,8 @@ DOMAIN = 'domain' HOST_ID = 'host_id' HOSTNAME = 'hostname' IP_ADDR = 'ip' -IPMI = 'ipmi' -IPMI_CREDS = 'ipmi_credentials' +POWER_TYPE = 'power_type' +POWER_MANAGE = 'power_manage' MAC_ADDR = 'mac' MGMT_NIC_FLAG = 'is_mgmt' NETMASK = 'netmask' diff --git a/compass-tasks-k8s/run.sh b/compass-tasks-k8s/run.sh index 3a7c0c8..f40d853 100644 --- a/compass-tasks-k8s/run.sh +++ b/compass-tasks-k8s/run.sh @@ -9,5 +9,8 @@ systemctl mask firewalld rm -rf /opt/kargo_k8s git clone https://github.com/kubernetes-incubator/kubespray.git /opt/kargo_k8s cd /opt/kargo_k8s -git checkout f4180503c891bea4b4b77a2f7cc93923411a7449 -b k8s1.9.1 +git checkout 05dabb7e7b5eb7cd9a075064868bafe4dc1cf51f -b k8s1.11.3 +source /root/.virtualenvs/compass-core/bin/activate pip install ansible==2.4.2.0 +ln -s /root/.virtualenvs/compass-core/bin/ansible /usr/bin/ansible +ln -s /root/.virtualenvs/compass-core/bin/ansible-playbook /usr/bin/ansible-playbook diff --git a/compass-tasks-osa/Dockerfile b/compass-tasks-osa/Dockerfile index 91abdc8..bc2a0c2 100644 --- a/compass-tasks-osa/Dockerfile +++ b/compass-tasks-osa/Dockerfile @@ -2,7 +2,7 @@ FROM opnfv/compass-tasks-base ARG BRANCH=master ADD ./run.sh /root/ -ADD ./tacker_conf /opt/tacker_conf +#ADD ./tacker_conf /opt/tacker_conf ADD ./setup-complete.yml /opt/ RUN chmod +x /root/run.sh RUN /root/run.sh diff --git a/compass-tasks-osa/Dockerfile-arm64 b/compass-tasks-osa/Dockerfile-arm64 index 811d546..cb560b3 100644 --- a/compass-tasks-osa/Dockerfile-arm64 +++ b/compass-tasks-osa/Dockerfile-arm64 @@ -1,3 +1,5 @@ FROM opnfv/compass-tasks-base +ARG BRANCH=master -# TODO: support arm64 +ADD ./run.sh /root/ +# TODO diff --git a/compass-tasks-osa/run.sh b/compass-tasks-osa/run.sh index 63e5cb0..aa924b8 100644 --- a/compass-tasks-osa/run.sh +++ b/compass-tasks-osa/run.sh @@ -1,35 +1,38 @@ #!/bin/bash +pip install pyyaml + # add ansible-playbook for normal use ln -s /root/.virtualenvs/compass-core/bin/ansible-playbook /usr/bin/ansible-playbook -yum install https://rdoproject.org/repos/openstack-pike/rdo-release-pike.rpm -y +yum install https://rdoproject.org/repos/openstack-queens/rdo-release-queens.rpm -y yum install git ntp wget ntpdate openssh-server python-devel sudo '@Development Tools' -y systemctl stop firewalld systemctl mask firewalld -mkdir -p /opt/git/ -cd /opt/git/ -wget artifacts.opnfv.org/compass4nfv/package/openstack_pike.tar.gz -tar -zxvf openstack_pike.tar.gz -rm -rf openstack_pike.tar.gz +#mkdir -p /opt/git/ +#cd /opt/git/ +#wget artifacts.opnfv.org/compass4nfv/package/openstack_queens.tar.gz +#tar -zxvf openstack_queens.tar.gz +#rm -rf openstack_queens.tar.gz git clone https://git.openstack.org/openstack/openstack-ansible /opt/openstack-ansible cd /opt/openstack-ansible -git checkout 16c69046bfd90d1b984de43bc6267fece6b75f1c +git checkout 2f52fec3cdefcfb0bcc41a807380ecd88fae072e +#git checkout 16c69046bfd90d1b984de43bc6267fece6b75f1c #git checkout 4cde8f86aaea1fde7c43016f661119879068a133 -git checkout -b stable/pike +git checkout -b stable/queens -/bin/cp -rf /opt/tacker_conf/ansible-role-requirements.yml /opt/openstack-ansible/ -/bin/cp -rf /opt/tacker_conf/openstack_services.yml /opt/openstack-ansible/playbooks/defaults/repo_packages/ -/bin/cp -rf /opt/tacker_conf/os-tacker-install.yml /opt/openstack-ansible/playbooks/ -/bin/cp -rf /opt/tacker_conf/tacker.yml /opt/openstack-ansible/playbooks/inventory/env.d/ -/bin/cp -rf /opt/tacker_conf/tacker_all.yml /opt/openstack-ansible/group_vars/ -/bin/cp -rf /opt/tacker_conf/user_secrets.yml /opt/openstack-ansible/etc/openstack_deploy/ +#/bin/cp -rf /opt/tacker_conf/ansible-role-requirements.yml /opt/openstack-ansible/ +#/bin/cp -rf /opt/tacker_conf/openstack_services.yml /opt/openstack-ansible/playbooks/defaults/repo_packages/ +#/bin/cp -rf /opt/tacker_conf/os-tacker-install.yml /opt/openstack-ansible/playbooks/ +#/bin/cp -rf /opt/tacker_conf/tacker.yml /opt/openstack-ansible/playbooks/inventory/env.d/ +#/bin/cp -rf /opt/tacker_conf/tacker_all.yml /opt/openstack-ansible/group_vars/ +#/bin/cp -rf /opt/tacker_conf/user_secrets.yml /opt/openstack-ansible/etc/openstack_deploy/ /bin/cp -rf /opt/openstack-ansible/etc/openstack_deploy /etc/openstack_deploy @@ -42,7 +45,7 @@ rm -f /usr/local/bin/ansible-playbook cd /opt/openstack-ansible/scripts/ python pw-token-gen.py --file /etc/openstack_deploy/user_secrets.yml -cd /opt/openstack-ansible/group_vars +cd /opt/openstack-ansible/inventory/group_vars sed -i 's/#repo_build_git_cache/repo_build_git_cache/g' repo_all.yml cp /opt/setup-complete.yml /opt/openstack-ansible/playbooks/ diff --git a/compass-tasks/Dockerfile-arm64 b/compass-tasks/Dockerfile-arm64 index 811d546..cb560b3 100644 --- a/compass-tasks/Dockerfile-arm64 +++ b/compass-tasks/Dockerfile-arm64 @@ -1,3 +1,5 @@ FROM opnfv/compass-tasks-base +ARG BRANCH=master -# TODO: support arm64 +ADD ./run.sh /root/ +# TODO |