Upgrade tacker from mitaka to newton

JIRA: COMPASS-528

Add tacker package making scripts into repo/pip/ directory.
Compass will install tacker using pip in this release.
Upgrade tacker role including tasks, templates and vars.
Modify the installation method from using source code to using whl package.

Change-Id: Iead323c7b46d8d1df084ed868d0e01bc1b0f7e55
Signed-off-by: Yifei Xue <xueyifei@huawei.com>

16 files changed, 214 insertions, 97 deletions

diff --git a/build/build.conf b/build/build.conf
index 68fc1cfd..cb56e32a 100644
--- a/build/build.conf
+++ b/build/build.conf
@@ -22,7 +22,7 @@ export UBUNTU_ISO=${UBUNTU_ISO:-$PACKAGE_URL/ubuntu-16.04-server-amd64.iso} # ub
 
 # OpenStack Packages for deployment
 export XENIAL_NEWTON_PPA=${XENIAL_NEWTON_PPA:-$PPA_URL/xenial-newton-ppa.tar.gz}
-export CENTOS7_NEWTON_PPA=${CENTOS7_NEWTON_PPA:-$PACKAGE_URL/centos7-newton-ppa.tar.gz}
+export CENTOS7_NEWTON_PPA=${CENTOS7_NEWTON_PPA:-$PPA_URL/centos7-newton-ppa.tar.gz}
 
 # SDN Packages for integration
 export APP_PACKAGE=${APP_PACKAGE:-$FEATURE_URL/packages.tar.gz}
diff --git a/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml
index 2efdbdb4..f328d959 100644
--- a/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml
+++ b/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml
@@ -232,6 +232,12 @@
   remote_user: root
   max_fail_percentage: 0
   roles:
+#    - tacker
+
+- hosts: controller
+  remote_user: root
+  max_fail_percentage: 0
+  roles:
     - openstack-post
 
 - hosts: controller
diff --git a/deploy/adapters/ansible/roles/tacker/files/tacker.conf b/deploy/adapters/ansible/roles/tacker/files/tacker.conf
new file mode 100644
index 00000000..0c90dcb9
--- /dev/null
+++ b/deploy/adapters/ansible/roles/tacker/files/tacker.conf
@@ -0,0 +1,36 @@
+description "OpenStack Tacker Server"
+author "Yifei Xue <xueyifei@huawei.com>"
+
+start on runlevel [2345]
+stop on runlevel [!2345]
+
+chdir /var/run
+
+respawn
+respawn limit 20 5
+limit nofile 65535 65535
+
+pre-start script
+        for i in lock run log lib ; do
+                mkdir -p /var/$i/tacker
+                chown root /var/$i/tacker
+        done
+end script
+
+script
+        [ -x "/usr/local/bin/tacker-server" ] || exit 0
+        DAEMON_ARGS=""
+        CONFIG_FILE="/usr/local/etc/tacker/tacker.conf"
+        USE_SYSLOG=""
+        USE_LOGFILE=""
+        NO_OPENSTACK_CONFIG_FILE_DAEMON_ARG=""
+        [ -r /etc/default/openstack ] && . /etc/default/openstack
+        [ -r /etc/default/$UPSTART_JOB ] && . /etc/default/$UPSTART_JOB
+        [ "x$USE_SYSLOG" = "xyes" ] && DAEMON_ARGS="$DAEMON_ARGS --use-syslog"
+        [ "x$USE_LOGFILE" != "xno" ] && DAEMON_ARGS="$DAEMON_ARGS --log-file=/var/log/tacker/tacker.log"
+        [ -z "$NO_OPENSTACK_CONFIG_FILE_DAEMON_ARG" ] && DAEMON_ARGS="$DAEMON_ARGS --config-file=$CONFIG_FILE"
+
+        exec start-stop-daemon --start --chdir /var/lib/tacker \
+                --chuid root:root --make-pidfile --pidfile /var/run/tacker/tacker.pid \
+                --exec /usr/local/bin/tacker-server -- ${DAEMON_ARGS}
+end script
diff --git a/deploy/adapters/ansible/roles/tacker/files/tacker.service b/deploy/adapters/ansible/roles/tacker/files/tacker.service
new file mode 100644
index 00000000..5fd61916
--- /dev/null
+++ b/deploy/adapters/ansible/roles/tacker/files/tacker.service
@@ -0,0 +1,19 @@
+[Unit]
+Description=OpenStack Tacker Server
+After=
+
+[Service]
+User=root
+Group=root
+Type=simple
+WorkingDirectory=/var/lib/tacker
+PermissionsStartOnly=true
+ExecStartPre=/bin/mkdir -p /var/lock/tacker /var/log/tacker /var/lib/tacker
+ExecStartPre=/usr/bin/touch /var/log/tacker/tacker.log
+ExecStart=/usr/local/bin/tacker-server --config-file /usr/local/etc/tacker/tacker.conf
+Restart=on-failure
+LimitNOFILE=65535
+TimeoutStopSec=15
+
+[Install]
+WantedBy=multi-user.target
diff --git a/deploy/adapters/ansible/roles/tacker/tasks/tacker_controller.yml b/deploy/adapters/ansible/roles/tacker/tasks/tacker_controller.yml
index 2d37ff13..f2c39bd7 100755
--- a/deploy/adapters/ansible/roles/tacker/tasks/tacker_controller.yml
+++ b/deploy/adapters/ansible/roles/tacker/tasks/tacker_controller.yml
@@ -7,46 +7,6 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 ---
-- name: get http server
-  shell: awk -F'=' '/compass_server/ {print $2}' /etc/compass.conf
-  register: http_server
-
-- name: creat tacker_home, tacker_client_home, tacker_horizon_home
-  shell: |
-    mkdir -p /opt/tacker;
-    mkdir -p /opt/tacker_client;
-    mkdir -p /opt/tacker_horizon;
-
-- name: download tacker package
-  get_url:
-    url: "http://{{ http_server.stdout_lines[0] }}/packages/tacker/{{ tacker_pkg_name }}"
-    dest: /opt/{{ tacker_pkg_name }}
-
-- name: download tacker_client package
-  get_url:
-    url: "http://{{ http_server.stdout_lines[0] }}/packages/tacker/{{ tacker_client_pkg_name }}"
-    dest: /opt/{{ tacker_client_pkg_name }}
-
-- name: download tacker_horizon package
-  get_url:
-    url: "http://{{ http_server.stdout_lines[0] }}/packages/tacker/{{ tacker_horizon_pkg_name }}"
-    dest: /opt/{{ tacker_horizon_pkg_name }}
-
-- name: extract tacker package
-  command: |
-    su -s /bin/sh -c "tar xzf /opt/{{ tacker_pkg_name }} -C {{ tacker_home }} \
-        --strip-components 1 --no-overwrite-dir -k --skip-old-files"
-
-- name: extract tacker_client package
-  command: |
-    su -s /bin/sh -c "tar xzf /opt/{{ tacker_client_pkg_name }} -C {{ tacker_client_home }} \
-        --strip-components 1 --no-overwrite-dir -k --skip-old-files"
-
-- name: extract tacker_horizon package
-  command: |
-    su -s /bin/sh -c "tar xzf /opt/{{ tacker_horizon_pkg_name }} -C {{ tacker_horizon_home }} \
-        --strip-components 1 --no-overwrite-dir -k --skip-old-files"
-
 - name: edit ml2_conf.ini
   shell: |
     crudini --set /etc/neutron/plugins/ml2/ml2_conf.ini ml2 extension_drivers port_security;
@@ -65,7 +25,7 @@
 
 - name: "combination of the haproxy configuration"
   shell: "cat /tmp/haproxy-tacker.cfg >> /etc/haproxy/haproxy.cfg"
-  when: ha_cfg.stdout.find('8888') == -1
+  when: ha_cfg.stdout.find('9890') == -1
 
 - name: "delete temporary configuration file"
   file:
@@ -136,29 +96,39 @@
   shell: |
     . /opt/admin-openrc.sh; \
     openstack endpoint create --region RegionOne \
-        --publicurl 'http://{{ public_vip.ip }}:8888/' \
-        --adminurl 'http://{{ internal_vip.ip }}:8888/' \
-        --internalurl 'http://{{ internal_vip.ip }}:8888/' tacker
+        nfv-orchestration public 'http://{{ public_vip.ip }}:9890/';
+    openstack endpoint create --region RegionOne \
+        nfv-orchestration admin 'http://{{ internal_vip.ip }}:9890/';
+    openstack endpoint create --region RegionOne \
+        nfv-orchestration internal 'http://{{ internal_vip.ip }}:9890/';
   when: inventory_hostname == haproxy_hosts.keys()[0]
 
-- name: install pip package
-  pip: name=Babel state=present version=2.3.4
-
-- name: install pip packages
-  shell: |
-    pip install tosca-parser heat-translator oslosphinx;
-
-- name: install tacker
+- name: install tacker and tacker client
   shell: |
-    . /opt/admin-openrc.sh; cd {{ tacker_home }}; python setup.py install
+    pip install tacker python-tackerclient
 
-- name: create 'tacker' directory in '/var/cache', set ownership and permissions
+- name: create 'tacker' directory in '/var/cache'
   shell: |
     mkdir -p /var/cache/tacker
 
 - name: create 'tacker' directory in '/var/log'
   shell: mkdir -p /var/log/tacker
 
+- name: get the policy file of heat
+  shell: cat /etc/heat/policy.json | grep Flavor
+  register: heat_info
+
+- name: update the policy file of heat
+  lineinfile:
+    dest: /etc/heat/policy.json
+    regexp: '    "resource_types:OS::Nova::Flavor'
+    line: '    "resource_types:OS::Nova::Flavor": "role:admin",'
+  when: heat_info.stdout.find('role') == -1
+
+- name: restart heat services
+  service: name={{ item }} state=restarted enabled=yes
+  with_items: "{{ heat_services }}"
+
 - name: copy tacker configs
   template: src={{ item.src }} dest=/opt/os_templates
   with_items: "{{ tacker_configs_templates }}"
@@ -175,29 +145,67 @@
         --config-file /usr/local/etc/tacker/tacker.conf upgrade head
   when: inventory_hostname == haproxy_hosts.keys()[0]
 
-- name: install tacker client
-  shell: |
-    . /opt/admin-openrc.sh; cd {{ tacker_client_home }};
-    python setup.py install
+- name: create tacker service
+  copy: src=tacker.service dest=/lib/systemd/system/
+
+- name: create tacker service work dir
+  file: path=/var/lib/tacker state=directory
+
+- name: link the tacker service
+  file:
+    src: /lib/systemd/system/tacker.service
+    dest: /etc/systemd/system/multi-user.target.wants/tacker.service
+    state: link
+
+- name: start tacker service
+  shell: service tacker start
+
+- name: create tackerc file
+  template: src=tackerc.sh dest=/opt/tackerc.sh mode=777
+
+- name: create nfvrc file
+  template: src=nfvrc.sh dest=/opt/nfvrc.sh mode=777
+
+- name: get the nfv_user info
+  shell: . /opt/tackerc.sh; openstack user list
+  register: nfvuser_info
+
+- name: delete the existed nfv user
+  shell: . /opt/tackerc.sh; openstack user delete nfv_user
+  when: nfvuser_info.stdout.find('nfv') != -1 and inventory_hostname == haproxy_hosts.keys()[0]
+
+- name: get the openstack project info
+  shell: . /opt/tackerc.sh; openstack project list
+  register: nfvproject_info
 
-- name: install tacker horizon
+- name: delete the existed nfv project
   shell: |
-    . /opt/admin-openrc.sh; cd {{ tacker_horizon_home }};
-    python setup.py install
+    . /opt/tackerc.sh;
+    openstack project delete $(openstack project list | grep nfv | awk '{print $2}')
+  when: nfvproject_info.stdout.find('nfv') != -1 and inventory_hostname == haproxy_hosts.keys()[0]
 
-- name: enable tacker horizon in dashboard
+- name: create an nfv project
+  shell: . /opt/tackerc.sh; openstack project create --description "NFV Project" nfv
+  when: inventory_hostname == haproxy_hosts.keys()[0]
+
+- name: create nfv user with admin privileges
   shell: |
-    cp {{ tacker_horizon_home }}/openstack_dashboard_extensions/* \
-        /usr/share/openstack-dashboard/openstack_dashboard/enabled/
+    . /opt/tackerc.sh;
+    openstack user create --password console nfv_user;
+    openstack role add --project nfv --user nfv_user admin;
+  when: inventory_hostname == haproxy_hosts.keys()[0]
+
+- name: create config.yml
+  template: src=config.yaml dest=/opt/config.yaml
 
-- name: restart apache server
-  shell: service apache2 restart
+- name: check if tacker running
+  shell: . /opt/tackerc.sh; while (!(tacker ext-list)); do sleep 30; done
 
-- name: launch tacker-server
+- name: register VIM to tacker
   shell: |
-    . /opt/admin-openrc.sh; \
-    python /usr/local/bin/tacker-server \
-        --config-file /usr/local/etc/tacker/tacker.conf |
-        --log-file /var/log/tacker/tacker.log
-  async: 9999999999999
-  poll: 0
+    . /opt/tackerc.sh;
+    tacker vim-register --config-file /opt/config.yaml --description "Default VIM" VIM0
+  when: inventory_hostname == haproxy_hosts.keys()[0]
+
+- name: restart tacker service
+  shell: service tacker stop; service tacker start
diff --git a/deploy/adapters/ansible/roles/tacker/templates/config.yaml b/deploy/adapters/ansible/roles/tacker/templates/config.yaml
new file mode 100644
index 00000000..0a0cea66
--- /dev/null
+++ b/deploy/adapters/ansible/roles/tacker/templates/config.yaml
@@ -0,0 +1,8 @@
+---
+
+auth_url: http://{{ public_vip.ip }}:5000/v3
+username: nfv_user
+password: console
+project_name: nfv
+project_domain_name: default
+user_domain_name: default
diff --git a/deploy/adapters/ansible/roles/tacker/templates/haproxy-tacker-cfg.j2 b/deploy/adapters/ansible/roles/tacker/templates/haproxy-tacker-cfg.j2
index 93bbe798..6cb294ac 100644
--- a/deploy/adapters/ansible/roles/tacker/templates/haproxy-tacker-cfg.j2
+++ b/deploy/adapters/ansible/roles/tacker/templates/haproxy-tacker-cfg.j2
@@ -1,10 +1,10 @@
 listen  proxy-tacker_api_cluster
-    bind {{ internal_vip.ip }}:8888
-    bind {{ public_vip.ip }}:8888
+    bind {{ internal_vip.ip }}:9890
+    bind {{ public_vip.ip }}:9890
     mode tcp
     option tcp-check
     option tcplog
     balance source
 {% for host,ip in haproxy_hosts.items() %}
-    server {{ host }} {{ ip }}:8888 weight 1 check inter 2000 rise 2 fall 5
+    server {{ host }} {{ ip }}:9890 weight 1 check inter 2000 rise 2 fall 5
 {% endfor %}
diff --git a/deploy/adapters/ansible/roles/tacker/templates/nfvrc.sh b/deploy/adapters/ansible/roles/tacker/templates/nfvrc.sh
new file mode 100644
index 00000000..b8074052
--- /dev/null
+++ b/deploy/adapters/ansible/roles/tacker/templates/nfvrc.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+export LC_ALL=C
+export OS_NO_CACHE=true
+export OS_TENANT_NAME=nfv
+export OS_PROJECT_NAME=nfv
+export OS_USERNAME=nfv_user
+export OS_PASSWORD=console
+export OS_AUTH_URL=http://{{ internal_vip.ip }}:35357/v3
+export OS_PROJECT_DOMAIN_NAME=default
+export OS_USER_DOMAIN_NAME=default
+export OS_AUTH_STRATEGY=keystone
+export OS_REGION_NAME=RegionOne
+export OS_IDENTITY_API_VERSION=3
+export OS_IMAGE_API_VERSION=2
diff --git a/deploy/adapters/ansible/roles/tacker/templates/tacker.j2 b/deploy/adapters/ansible/roles/tacker/templates/tacker.j2
index ae0f644a..89c8fea1 100644
--- a/deploy/adapters/ansible/roles/tacker/templates/tacker.j2
+++ b/deploy/adapters/ansible/roles/tacker/templates/tacker.j2
@@ -1,3 +1,12 @@
+{% set memcached_servers = [] %}
+{% set rabbitmq_servers = [] %}
+{% for host in haproxy_hosts.values() %}
+{% set _ = memcached_servers.append('%s:11211'% host) %}
+{% set _ = rabbitmq_servers.append('%s:5672'% host) %}
+{% endfor %}
+{% set memcached_servers = memcached_servers|join(',') %}
+{% set rabbitmq_servers = rabbitmq_servers|join(',') %}
+
 [DEFAULT]
 # Print more verbose output (set logging level to INFO instead of default WARNING level).
 verbose = True
@@ -38,7 +47,7 @@ use_syslog = False
 bind_host = {{ internal_ip }}
 
 # Port the bind the API server to
-bind_port = 8888
+bind_port = 9890
 
 # Path to the extensions.  Note that this can be a colon-separated list of
 # paths.  For example:
@@ -333,6 +342,10 @@ signing_dir = /var/cache/tacker
 #project_domain_id = default
 project_name = service
 #user_domain_id = default
+project_domain_name = default
+user_domain_name = default
+region_name = RegionOne
+memcached_servers = {{ memcached_servers }}
 password = console
 username = tacker
 auth_url = http://{{ internal_vip.ip }}:35357
@@ -398,14 +411,10 @@ connection = mysql://tacker:TACKER_DBPASS@{{ internal_vip.ip }}:3306/tacker?char
 # mgmt_driver = noop,openwrt
 
 # Specify drivers for monitoring
-# monitor_driver = ping, http_ping
-
-[nfvo_vim]
-# Supported VIM drivers, resource orchestration controllers such as OpenStack, kvm
-#Default VIM driver is OpenStack
-#vim_drivers = openstack
-#Default VIM placement if vim id is not provided
-default_vim = VIM0
+monitor_driver = ping, http_ping
+
+[nfvo]
+vim_drivers = openstack
 
 [vim_keys]
 #openstack = /etc/tacker/vim/fernet_keys
diff --git a/deploy/adapters/ansible/roles/tacker/templates/tackerc.sh b/deploy/adapters/ansible/roles/tacker/templates/tackerc.sh
new file mode 100644
index 00000000..83dc188a
--- /dev/null
+++ b/deploy/adapters/ansible/roles/tacker/templates/tackerc.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+export LC_ALL=C
+export OS_NO_CACHE=true
+export OS_TENANT_NAME=service
+export OS_PROJECT_NAME=service
+export OS_USERNAME=tacker
+export OS_PASSWORD=console
+export OS_AUTH_URL=http://{{ internal_vip.ip }}:35357/v3
+export OS_PROJECT_DOMAIN_NAME=default
+export OS_USER_DOMAIN_NAME=default
+export OS_AUTH_STRATEGY=keystone
+export OS_REGION_NAME=RegionOne
+export OS_IDENTITY_API_VERSION=3
+export OS_IMAGE_API_VERSION=2
diff --git a/deploy/adapters/ansible/roles/tacker/vars/Debian.yml b/deploy/adapters/ansible/roles/tacker/vars/Debian.yml
index 59a4dbd9..039e1680 100755
--- a/deploy/adapters/ansible/roles/tacker/vars/Debian.yml
+++ b/deploy/adapters/ansible/roles/tacker/vars/Debian.yml
@@ -12,3 +12,8 @@ packages:
   - crudini
 
 services: []
+
+heat_services:
+  - heat-api
+  - heat-api-cfn
+  - heat-engine
diff --git a/deploy/adapters/ansible/roles/tacker/vars/main.yml b/deploy/adapters/ansible/roles/tacker/vars/main.yml
index 601f3721..c934090c 100755
--- a/deploy/adapters/ansible/roles/tacker/vars/main.yml
+++ b/deploy/adapters/ansible/roles/tacker/vars/main.yml
@@ -7,12 +7,6 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
-tacker_pkg_name: tacker-0.3.1.tar.gz
-tacker_client_pkg_name: python-tackerclient-0.3.1.tar.gz
-tacker_horizon_pkg_name: tacker-horizon-0.3.1.tar.gz
-tacker_home: /opt/tacker/
-tacker_client_home: /opt/tacker_client/
-tacker_horizon_home: /opt/tacker_horizon/
 
 tacker_configs_templates:
   - src: tacker.j2
diff --git a/repo/gen_ins_pkg_script.py b/repo/gen_ins_pkg_script.py
index 73139655..38d08c23 100644
--- a/repo/gen_ins_pkg_script.py
+++ b/repo/gen_ins_pkg_script.py
@@ -86,7 +86,7 @@ def generate_download_script(root="", arch="", tmpl="", docker_tmpl="", default_
             for filename in filenames:
                 if os.path.isfile(os.path.join(parent, filename)):
                     special_packages_names.append(filename)
-        searchList.update({'spcial_packages': special_packages_names})
+        searchList.update({'special_packages': special_packages_names})
 
     Dockerfile = os.path.basename(docker_tmpl).split('.')[0]
     tmpl = Template(file=docker_tmpl, searchList=searchList)
diff --git a/repo/openstack/make_ppa/ubuntu/xenial/newton/download_pkg.tmpl b/repo/openstack/make_ppa/ubuntu/xenial/newton/download_pkg.tmpl
index e10d4e22..f2a14f48 100644
--- a/repo/openstack/make_ppa/ubuntu/xenial/newton/download_pkg.tmpl
+++ b/repo/openstack/make_ppa/ubuntu/xenial/newton/download_pkg.tmpl
@@ -5,7 +5,7 @@ set -ex
 
 # add openstack $OPV repo
 apt-get update && apt-get install -y software-properties-common
-# sudo add-apt-repository -y cloud-archive:$OPV
+add-apt-repository -y cloud-archive:$OPV
 apt-get install -d nova-compute-kvm -y
 
 #make pernoca database
diff --git a/repo/pip/Dockerfile b/repo/pip/Dockerfile
index e8fed1ba..356c1459 100644
--- a/repo/pip/Dockerfile
+++ b/repo/pip/Dockerfile
@@ -23,6 +23,10 @@ RUN apt-get install -y git python-pip
 
 RUN apt-get install -y libssl-dev libffi-dev
 
+RUN pip install --upgrade setuptools
+
+RUN pip install --upgrade pip
+
 RUN mkdir -p /home/tmp
 
 RUN mkdir -p /pip-openstack
diff --git a/repo/pip/code_url.conf b/repo/pip/code_url.conf
index faf360e2..c3c90c4c 100644
--- a/repo/pip/code_url.conf
+++ b/repo/pip/code_url.conf
@@ -17,8 +17,8 @@ export GIT_URL="https://github.com/openstack/keystone.git \
     https://github.com/openstack/python-heatclient.git \
     https://github.com/openstack/aodh.git \
     https://github.com/openstack/python-aodhclient.git \
-    https://github.com/openstack/tacker.git \
-    https://github.com/openstack/python-tackerclient.git "
 
 export PIP_GIT_URL="https://github.com/openstack/congress.git \
-                    https://github.com/openstack/python-congressclient.git"
+                    https://github.com/openstack/python-congressclient.git \
+                    https://github.com/openstack/tacker.git \
+                    https://github.com/openstack/python-tackerclient.git"