diff options
| -rw-r--r-- | docs/jenkins-job-builder/index.rst | 6 | ||||
| -rw-r--r-- | jjb/apex/apex.yml | 2 | ||||
| -rw-r--r-- | jjb/global/releng-macros.yml | 1 | ||||
| -rw-r--r-- | jjb/qtip/helpers/validate-deploy.sh | 22 | ||||
| -rw-r--r-- | jjb/xci/xci-daily-jobs.yml | 64 | ||||
| -rwxr-xr-x | jjb/xci/xci-deploy.sh | 39 | ||||
| -rwxr-xr-x | jjb/xci/xci-provision.sh | 40 | ||||
| -rw-r--r-- | modules/opnfv/utils/ssh_utils.py | 13 | ||||
| -rwxr-xr-x | prototypes/bifrost/scripts/osa-bifrost-deployment.sh | 6 | ||||
| -rw-r--r-- | prototypes/openstack-ansible/playbooks/configure-xcimaster.yml | 17 | ||||
| -rw-r--r-- | prototypes/openstack-ansible/var/ubuntu.yml | 2 | ||||
| -rwxr-xr-x | utils/test/testapi/install.sh | 3 | ||||
| -rw-r--r-- | utils/test/testapi/opnfv_testapi/common/config.py | 9 | ||||
| -rw-r--r-- | utils/test/testapi/opnfv_testapi/resources/handlers.py | 6 | ||||
| -rwxr-xr-x | utils/test/testapi/run_test.sh | 14 | ||||
| -rw-r--r-- | utils/test/testapi/test-requirements.txt | 5 |
16 files changed, 184 insertions, 65 deletions
diff --git a/docs/jenkins-job-builder/index.rst b/docs/jenkins-job-builder/index.rst index b85b1320f..4d23ade38 100644 --- a/docs/jenkins-job-builder/index.rst +++ b/docs/jenkins-job-builder/index.rst @@ -1,9 +1,9 @@ -*************************** +=========================== Release Engineering Project -*************************** +=========================== .. toctree:: :numbered: :maxdepth: 2 - opnfv-jjb-usage.rst + opnfv-jjb-usage diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml index 93eaa6c6c..aaf6c499a 100644 --- a/jjb/apex/apex.yml +++ b/jjb/apex/apex.yml @@ -383,7 +383,7 @@ block: true same-node: true - trigger-builds: - - project: 'cperf-apex-intel-pod2-daily-{stream}' + - project: 'cperf-apex-intel-pod2-daily-master' predefined-parameters: DEPLOY_SCENARIO=os-odl_l3-nofeature-noha block: true diff --git a/jjb/global/releng-macros.yml b/jjb/global/releng-macros.yml index 63613f88d..6fdb4caf4 100644 --- a/jjb/global/releng-macros.yml +++ b/jjb/global/releng-macros.yml @@ -75,7 +75,6 @@ submodule: recursive: true timeout: 20 - shallow-clone: true - trigger: name: 'daily-trigger-disabled' triggers: diff --git a/jjb/qtip/helpers/validate-deploy.sh b/jjb/qtip/helpers/validate-deploy.sh index a73e33c2c..aa07986ce 100644 --- a/jjb/qtip/helpers/validate-deploy.sh +++ b/jjb/qtip/helpers/validate-deploy.sh @@ -10,7 +10,9 @@ set -e envs="INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} -e NODE_NAME=${NODE_NAME} -e CI_DEBUG=${CI_DEBUG}" +ramfs=/tmp/qtip/ramfs dir_imgstore="${HOME}/imgstore" +ramfs_volume="$ramfs:/mnt/ramfs" echo "--------------------------------------------------------" echo "POD: $NODE_NAME" @@ -21,7 +23,24 @@ echo "--------------------------------------------------------" echo "Qtip: Pulling docker image: opnfv/qtip:${DOCKER_TAG}" docker pull opnfv/qtip:$DOCKER_TAG -cmd=" docker run -id -e $envs opnfv/qtip:${DOCKER_TAG} /bin/bash" +# use ramfs to fix docker socket connection issue with overlay mode in centos +if [ ! -d $ramfs ]; then + mkdir -p $ramfs +fi + +if [ ! -z $(df $ramfs | tail -n -1 | grep $ramfs) ]; then + sudo mount -t tmpfs -o size=32M tmpfs $ramfs +fi + +# enable contro path in docker +echo <<EOF > /tmp/ansible.cfg +[defaults] +callback_whitelist = profile_tasks +[ssh_connection] +control_path=/mnt/ramfs/ansible-ssh-%%h-%%p-%%r +EOF + +cmd=" docker run -id -e $envs -v ${ramfs_volume} opnfv/qtip:${DOCKER_TAG} /bin/bash" echo "Qtip: Running docker command: ${cmd}" ${cmd} @@ -32,6 +51,7 @@ if [ $(docker ps | grep 'opnfv/qtip' | wc -l) == 0 ]; then else echo "The container ID is: ${container_id}" QTIP_REPO=/home/opnfv/repos/qtip + docker cp /tmp/ansible.cfg ${container_id}:/home/opnfv/.ansible.cfg # TODO(zhihui_wu): use qtip cli to execute benchmark test in the future docker exec -t ${container_id} bash -c "cd ${QTIP_REPO}/qtip/runner/ && python runner.py -d /home/opnfv/qtip/results/ -b all" diff --git a/jjb/xci/xci-daily-jobs.yml b/jjb/xci/xci-daily-jobs.yml index ce2e4f382..f42f86263 100644 --- a/jjb/xci/xci-daily-jobs.yml +++ b/jjb/xci/xci-daily-jobs.yml @@ -3,15 +3,22 @@ #-------------------------------- # BRANCH ANCHORS #-------------------------------- +# the versions stated here default to branches which then later +# on used for checking out the branches, pulling in head of the branch. +# but they also allow us to state sha1 so instead of checking out the +# branches, we can check out sha1 if we want to use locked/specific +# sha1 or manually enter sha1. master: &master stream: master - openstack-branch: '{stream}' - opnfv-branch: 'master' + openstack-osa-version: '{stream}' + openstack-bifrost-version: '{stream}' + opnfv-releng-version: 'master' gs-pathname: '' ocata: &ocata stream: ocata - openstack-branch: 'stable/{stream}' - opnfv-branch: 'master' + openstack-osa-version: 'stable/{stream}' + openstack-bifrost-version: 'stable/{stream}' + opnfv-releng-version: 'master' gs-pathname: '/{stream}' #-------------------------------- # scenarios @@ -87,6 +94,18 @@ parameters: - string: + name: OPENSTACK_OSA_VERSION + default: '{openstack-osa-version}' + - string: + name: OPENSTACK_BIFROST_VERSION + default: '{openstack-osa-version}' + - string: + name: OPNFV_RELENG_VERSION + default: '{opnfv-releng-version}' + - string: + name: USE_PROMOTED_VERSIONS + default: 'true' + - string: name: DEPLOY_SCENARIO default: '{scenario}' - label: @@ -105,22 +124,34 @@ - trigger-builds: - project: 'xci-provision-{pod}-{distro}-daily-{stream}' current-parameters: false - predefined-parameters: - DEPLOY_SCENARIO={scenario} + predefined-parameters: | + OPENSTACK_OSA_VERSION=$OPENSTACK_OSA_VERSION + OPENSTACK_BIFROST_VERSION=$OPENSTACK_BIFROST_VERSION + OPNFV_RELENG_VERSION=$OPNFV_RELENG_VERSION + USE_PROMOTED_VERSIONS=$USE_PROMOTED_VERSIONS + DEPLOY_SCENARIO=$DEPLOY_SCENARIO same-node: true block: true - trigger-builds: - project: 'xci-deploy-{pod}-{distro}-daily-{stream}' current-parameters: false - predefined-parameters: - DEPLOY_SCENARIO={scenario} + predefined-parameters: | + OPENSTACK_OSA_VERSION=$OPENSTACK_OSA_VERSION + OPENSTACK_BIFROST_VERSION=$OPENSTACK_BIFROST_VERSION + OPNFV_RELENG_VERSION=$OPNFV_RELENG_VERSION + USE_PROMOTED_VERSIONS=$USE_PROMOTED_VERSIONS + DEPLOY_SCENARIO=$DEPLOY_SCENARIO same-node: true block: true - trigger-builds: - project: 'xci-functest-{pod}-{distro}-daily-{stream}' current-parameters: false - predefined-parameters: - DEPLOY_SCENARIO={scenario} + predefined-parameters: | + OPENSTACK_OSA_VERSION=$OPENSTACK_OSA_VERSION + OPENSTACK_BIFROST_VERSION=$OPENSTACK_BIFROST_VERSION + OPNFV_RELENG_VERSION=$OPNFV_RELENG_VERSION + USE_PROMOTED_VERSIONS=$USE_PROMOTED_VERSIONS + DEPLOY_SCENARIO=$DEPLOY_SCENARIO same-node: true block: true block-thresholds: @@ -151,11 +182,14 @@ parameters: - string: - name: OPENSTACK_BRANCH - default: '{openstack-branch}' + name: OPENSTACK_OSA_VERSION + default: '{openstack-osa-version}' + - string: + name: OPENSTACK_BIFROST_VERSION + default: '{openstack-osa-version}' - string: - name: OPNFV_BRANCH - default: '{opnfv-branch}' + name: OPNFV_RELENG_VERSION + default: '{opnfv-releng-version}' - string: name: USE_PROMOTED_VERSIONS default: 'true' @@ -201,7 +235,7 @@ - pre-scm-buildstep: - shell: | #!/bin/bash - sudo -R chown $USER $WORKSPACE || exit 1 + sudo chown -R $USER $WORKSPACE || exit 1 #--------------------------- # builder macros diff --git a/jjb/xci/xci-deploy.sh b/jjb/xci/xci-deploy.sh index dae67cf84..87f9ec8db 100755 --- a/jjb/xci/xci-deploy.sh +++ b/jjb/xci/xci-deploy.sh @@ -46,18 +46,32 @@ fix_ownership # openstack-ansible enables strict host key checking by default export ANSIBLE_HOST_KEY_CHECKING=False +# ensure the versions to checkout are set +export OPENSTACK_OSA_VERSION=${OPENSTACK_OSA_VERSION:-master} +export OPNFV_RELENG_VERSION=${OPNFV_RELENG_VERSION:-master} + +# log some info +echo -e "\n" +echo "***********************************************************************" +echo "* *" +echo "* Deploy OpenStack *" +echo "* *" +echo " openstack-ansible version: $OPENSTACK_OSA_VERSION" +echo " releng version: $OPNFV_RELENG_VERSION" +echo "* *" +echo "***********************************************************************" +echo -e "\n" +# clone releng repo +sudo git clone --quiet https://gerrit.opnfv.org/gerrit/releng /opt/releng +cd /opt/releng && sudo git checkout --quiet $OPNFV_RELENG_VERSION +echo "xci: using openstack-ansible commit" +git show --oneline -s --pretty=format:'%h - %s (%cr) <%an>' + # display the nodes +echo "xci: OpenStack nodes" cd /opt/bifrost source env-vars ironic node-list -virsh list - -# ensure the branches to use are set -export OPNFV_BRANCH=${OPNFV_BRANCH:-master} -export OPENSTACK_BRANCH=${OPENSTACK_BRANCH:-master} - -# clone releng repo -sudo git clone -b $OPNFV_BRANCH https://gerrit.opnfv.org/gerrit/releng /opt/releng # this script will be reused for promoting openstack-ansible versions and using # promoted openstack-ansible versions as part of xci daily. @@ -68,3 +82,12 @@ fi cd /opt/releng/prototypes/openstack-ansible/scripts sudo -E ./osa-deploy.sh + +# log some info +echo -e "\n" +echo "***********************************************************************" +echo "* *" +echo "* OpenStack deployment is completed! *" +echo "* *" +echo "***********************************************************************" +echo -e "\n" diff --git a/jjb/xci/xci-provision.sh b/jjb/xci/xci-provision.sh index e474093b8..a1ba78bf0 100755 --- a/jjb/xci/xci-provision.sh +++ b/jjb/xci/xci-provision.sh @@ -43,11 +43,32 @@ sudo /bin/rm -rf /opt/bifrost /opt/openstack-ansible /opt/stack /opt/releng /opt # Fix up permissions fix_ownership -# ensure the branches to use are set -export OPENSTACK_BRANCH=${OPENSTACK_BRANCH:-master} -export OPNFV_BRANCH=${OPNFV_BRANCH:-master} -sudo git clone -b $OPENSTACK_BRANCH https://git.openstack.org/openstack/bifrost /opt/bifrost -sudo git clone -b $OPNFV_BRANCH https://gerrit.opnfv.org/gerrit/releng /opt/releng +# ensure the versions to checkout are set +export OPENSTACK_BIFROST_VERSION=${OPENSTACK_BIFROST_VERSION:-master} +export OPNFV_RELENG_VERSION=${OPNFV_RELENG_VERSION:-master} + +# log some info +echo -e "\n" +echo "***********************************************************************" +echo "* *" +echo "* Provision OpenStack Nodes *" +echo "* *" +echo " bifrost version: $OPENSTACK_BIFROST_VERSION" +echo " releng version: $OPNFV_RELENG_VERSION" +echo "* *" +echo "***********************************************************************" +echo -e "\n" + +# clone the repos and checkout the versions +sudo git clone --quiet https://git.openstack.org/openstack/bifrost /opt/bifrost +cd /opt/bifrost && sudo git checkout --quiet $OPENSTACK_BIFROST_VERSION +echo "xci: using bifrost commit" +git show --oneline -s --pretty=format:'%h - %s (%cr) <%an>' + +sudo git clone --quiet https://gerrit.opnfv.org/gerrit/releng /opt/releng +cd /opt/releng && sudo git checkout --quiet $OPNFV_RELENG_VERSION +echo "xci: using releng commit" +git show --oneline -s --pretty=format:'%h - %s (%cr) <%an>' # this script will be reused for promoting bifrost versions and using # promoted bifrost versions as part of xci daily. @@ -72,3 +93,12 @@ cd /opt/bifrost source env-vars ironic node-list virsh list + +# log some info +echo -e "\n" +echo "***********************************************************************" +echo "* *" +echo "* OpenStack nodes are provisioned! *" +echo "* *" +echo "***********************************************************************" +echo -e "\n" diff --git a/modules/opnfv/utils/ssh_utils.py b/modules/opnfv/utils/ssh_utils.py index d17f5ae81..cbc62c75d 100644 --- a/modules/opnfv/utils/ssh_utils.py +++ b/modules/opnfv/utils/ssh_utils.py @@ -15,6 +15,7 @@ import paramiko from opnfv.utils import opnfv_logger as logger logger = logger.Logger("SSH utils").getLogger() +SSH_TIMEOUT = 60 def get_ssh_client(hostname, @@ -40,11 +41,13 @@ def get_ssh_client(hostname, client.load_system_host_keys() client.connect(hostname, username=username, - pkey=key) + pkey=key, + timeout=SSH_TIMEOUT) else: client.connect(hostname, username=username, - password=password) + password=password, + timeout=SSH_TIMEOUT) return client except Exception as e: @@ -96,7 +99,8 @@ class ProxyHopClient(paramiko.SSHClient): self.proxy_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) self.proxy_ssh.connect(jh_ip, username=jh_user, - password=jh_pass) + password=jh_pass, + timeout=SSH_TIMEOUT) self.proxy_transport = self.proxy_ssh.get_transport() def connect(self, hostname, port=22, username='root', password=None, @@ -126,7 +130,8 @@ class ProxyHopClient(paramiko.SSHClient): super(ProxyHopClient, self).connect(hostname, username=username, pkey=proxy_key, - sock=self.proxy_channel) + sock=self.proxy_channel, + timeout=timeout) os.remove(self.local_ssh_key) except Exception as e: logger.error(e) diff --git a/prototypes/bifrost/scripts/osa-bifrost-deployment.sh b/prototypes/bifrost/scripts/osa-bifrost-deployment.sh index 33ad10887..fb66ae9e3 100755 --- a/prototypes/bifrost/scripts/osa-bifrost-deployment.sh +++ b/prototypes/bifrost/scripts/osa-bifrost-deployment.sh @@ -20,11 +20,9 @@ USE_VENV="false" BUILD_IMAGE=true PROVISION_WAIT_TIMEOUT=${PROVISION_WAIT_TIMEOUT:-3600} -# ensure the branch is set -export OPENSTACK_BRANCH=${OPENSTACK_BRANCH:-master} - # ensure the right inventory files is used based on branch -if [ $OPENSTACK_BRANCH = "master" ]; then +CURRENT_BIFROST_BRANCH=$(git rev-parse --abbrev-ref HEAD) +if [ $CURRENT_BIFROST_BRANCH = "master" ]; then export BIFROST_INVENTORY_SOURCE=${BIFROST_INVENTORY_SOURCE:-'/tmp/baremetal.json'} else export BIFROST_INVENTORY_SOURCE=${BIFROST_INVENTORY_SOURCE:-'/tmp/baremetal.csv'} diff --git a/prototypes/openstack-ansible/playbooks/configure-xcimaster.yml b/prototypes/openstack-ansible/playbooks/configure-xcimaster.yml index b6c79a418..fbbde640c 100644 --- a/prototypes/openstack-ansible/playbooks/configure-xcimaster.yml +++ b/prototypes/openstack-ansible/playbooks/configure-xcimaster.yml @@ -10,10 +10,19 @@ creates: /root/.ssh/id_rsa - name: fetch public key fetch: src="/root/.ssh/id_rsa.pub" dest="/" - - name: remove the directory - shell: "rm -rf {{OSA_PATH}} {{OSA_ETC_PATH}}" - - name: git openstack ansible - shell: "git clone {{OSA_URL}} {{OSA_PATH}} -b {{OPENSTACK_BRANCH}}" + - name: remove openstack-ansible directories + file: + path={{ item }} + state=absent + recurse=no + with_items: + - "{{OSA_PATH}}" + - "{{OSA_ETC_PATH}}" + - name: clone openstack-ansible + git: + repo: "{{OSA_URL}}" + dest: "{{OSA_PATH}}" + version: "{{OPENSTACK_OSA_VERSION}}" - name: copy opnfv-setup-openstack.yml to /opt/openstack-ansible/playbooks copy: src: ../file/opnfv-setup-openstack.yml diff --git a/prototypes/openstack-ansible/var/ubuntu.yml b/prototypes/openstack-ansible/var/ubuntu.yml index 321a7c4e5..eb595beaa 100644 --- a/prototypes/openstack-ansible/var/ubuntu.yml +++ b/prototypes/openstack-ansible/var/ubuntu.yml @@ -2,7 +2,7 @@ OSA_URL: https://git.openstack.org/openstack/openstack-ansible OSA_PATH: /opt/openstack-ansible OSA_ETC_PATH: /etc/openstack_deploy -OPENSTACK_BRANCH: "{{ lookup('env','OPENSTACK_BRANCH') }}" +OPENSTACK_OSA_VERSION: "{{ lookup('env','OPENSTACK_OSA_VERSION') }}" XCIMASTER_IP: 192.168.122.2 host_info: {'xcimaster':{'MGMT_IP': '172.29.236.10','VLAN_IP': '192.168.122.2', 'STORAGE_IP': '172.29.244.10'},'controller00':{'MGMT_IP': '172.29.236.11','VLAN_IP': '192.168.122.3', 'STORAGE_IP': '172.29.244.11'},'controller01':{'MGMT_IP': '172.29.236.12','VLAN_IP': '192.168.122.4', 'STORAGE_IP': '172.29.244.12'},'controller02':{'MGMT_IP': '172.29.236.13','VLAN_IP': '192.168.122.5', 'STORAGE_IP': '172.29.240.13'},'compute00':{'MGMT_IP': '172.29.236.14','VLAN_IP': '192.168.122.6','VLAN_IP_SECOND': '173.29.241.1','VXLAN_IP': '172.29.240.14', 'STORAGE_IP': '172.29.244.14'},'compute01':{'MGMT_IP': '172.29.236.15','VLAN_IP': '192.168.122.7','VLAN_IP_SECOND': '173.29.241.2','VXLAN_IP': '172.29.240.15', 'STORAGE_IP': '172.29.244.15'}} diff --git a/utils/test/testapi/install.sh b/utils/test/testapi/install.sh index c55691aed..bf828b580 100755 --- a/utils/test/testapi/install.sh +++ b/utils/test/testapi/install.sh @@ -26,3 +26,6 @@ fi cp -fr 3rd_party/static opnfv_testapi/tornado_swagger python setup.py install rm -fr opnfv_testapi/tornado_swagger/static +if [ ! -z "$VIRTUAL_ENV" ]; then + sed -i -e 's#etc/opnfv_testapi =#/etc/opnfv_testapi =#g' setup.cfg +fi
\ No newline at end of file diff --git a/utils/test/testapi/opnfv_testapi/common/config.py b/utils/test/testapi/opnfv_testapi/common/config.py index 105d4fabf..362fca640 100644 --- a/utils/test/testapi/opnfv_testapi/common/config.py +++ b/utils/test/testapi/opnfv_testapi/common/config.py @@ -30,7 +30,7 @@ class APIConfig(object): """ def __init__(self): - self._default_config_location = "/etc/opnfv_testapi/config.ini" + self._set_default_config() self.mongo_url = None self.mongo_dbname = None self.api_port = None @@ -39,6 +39,11 @@ class APIConfig(object): self._parser = None self.swagger_base_url = None + def _set_default_config(self): + venv = os.getenv('VIRTUAL_ENV') + self._default_config = os.path.join('/' if not venv else venv, + 'etc/opnfv_testapi/config.ini') + def _get_parameter(self, section, param): try: return self._parser.get(section, param) @@ -66,7 +71,7 @@ class APIConfig(object): obj = APIConfig() if config_location is None: - config_location = obj._default_config_location + config_location = obj._default_config if not os.path.exists(config_location): raise ParseError("%s not found" % config_location) diff --git a/utils/test/testapi/opnfv_testapi/resources/handlers.py b/utils/test/testapi/opnfv_testapi/resources/handlers.py index 8255b526a..15096468c 100644 --- a/utils/test/testapi/opnfv_testapi/resources/handlers.py +++ b/utils/test/testapi/opnfv_testapi/resources/handlers.py @@ -92,8 +92,6 @@ class GenericApiHandler(web.RequestHandler): raise gen.Return(ret) return wrapper - @web.asynchronous - @gen.coroutine @authenticate def _create(self, miss_checks, db_checks, **kwargs): """ @@ -158,8 +156,6 @@ class GenericApiHandler(web.RequestHandler): .format(query, self.table)) self.finish_request(self.format_data(data)) - @web.asynchronous - @gen.coroutine @authenticate def _delete(self, query): data = yield self._eval_db_find_one(query) @@ -171,8 +167,6 @@ class GenericApiHandler(web.RequestHandler): yield self._eval_db(self.table, 'remove', query) self.finish_request() - @web.asynchronous - @gen.coroutine @authenticate def _update(self, query, db_keys): if self.json_args is None: diff --git a/utils/test/testapi/run_test.sh b/utils/test/testapi/run_test.sh index 4efc7af3b..1e05dd6ba 100755 --- a/utils/test/testapi/run_test.sh +++ b/utils/test/testapi/run_test.sh @@ -8,15 +8,17 @@ SCRIPTDIR=`dirname $0` echo "Running unit tests..." # Creating virtual environment -virtualenv $SCRIPTDIR/testapi_venv -source $SCRIPTDIR/testapi_venv/bin/activate +if [ ! -z $VIRTUAL_ENV ]; then + venv=$VIRTUAL_ENV +else + venv=$SCRIPTDIR/.venv + virtualenv $venv +fi +source $venv/bin/activate # Install requirements pip install -r $SCRIPTDIR/requirements.txt -pip install coverage -pip install nose>=1.3.1 -pip install pytest -pip install mock +pip install -r $SCRIPTDIR/test-requirements.txt find . -type f -name "*.pyc" -delete diff --git a/utils/test/testapi/test-requirements.txt b/utils/test/testapi/test-requirements.txt index 4633ad637..645687b14 100644 --- a/utils/test/testapi/test-requirements.txt +++ b/utils/test/testapi/test-requirements.txt @@ -2,10 +2,7 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -tox mock pytest -pytest-cov coverage -pykwalify -pip_check_reqs +nose>=1.3.1 |