Adds ability to deploy from upstream openstack

To deploy with upstream openstack branch, use new deploy setting 'os_version'.
A default scenario file for nosdn with pike has been included in this patch.
If 'os_version' is a version other than the default version for this OPNFV
release, then upstream is used.

In order to use upstream with the current OS version use '--upstream'
argument to the deploy command, to force an upstream deployment.

Also include '-e upstream-environment.yaml' to use default upstream
deployment settings.

Supports nosdn and odl-nofeature deployments.

Change-Id: Ic07e308827b449637b4e86cdd086434e4de2fb69
Signed-off-by: Tim Rozet <trozet@redhat.com>

2 files changed, 70 insertions, 1 deletions

diff --git a/apex/common/constants.py b/apex/common/constants.py
index 0df71526..943e3220 100644
--- a/apex/common/constants.py
+++ b/apex/common/constants.py
@@ -33,14 +33,20 @@ DEFAULT_ROOT_DEV = 'sda'
 LIBVIRT_VOLUME_PATH = '/var/lib/libvirt/images'
 
 VIRT_UPLOAD = '--upload'
-VIRT_INSTALL = '-install'
+VIRT_INSTALL = '--install'
 VIRT_RUN_CMD = '--run-command'
 VIRT_PW = '--root-password'
 
 THT_DIR = '/usr/share/openstack-tripleo-heat-templates'
 THT_ENV_DIR = os.path.join(THT_DIR, 'environments')
 
+DEFAULT_OS_VERSION = 'ocata'
 DEFAULT_ODL_VERSION = 'carbon'
+VALID_ODL_VERSIONS = ['carbon', 'nitrogen', 'oxygen', 'master']
+PUPPET_ODL_URL = 'https://git.opendaylight.org/gerrit/integration/packaging' \
+                 '/puppet-opendaylight'
 DEBUG_OVERCLOUD_PW = 'opnfvapex'
 NET_ENV_FILE = 'network-environment.yaml'
 DEPLOY_TIMEOUT = 90
+UPSTREAM_RDO = 'https://images.rdoproject.org/ocata/delorean/current-tripleo/'
+OPENSTACK_GERRIT = 'https://review.openstack.org'
diff --git a/apex/common/utils.py b/apex/common/utils.py
index f418d429..13250a45 100644
--- a/apex/common/utils.py
+++ b/apex/common/utils.py
@@ -7,11 +7,17 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
+import datetime
 import json
 import logging
 import os
 import pprint
 import subprocess
+import tarfile
+import time
+import urllib.error
+import urllib.request
+import urllib.parse
 import yaml
 
 
@@ -129,3 +135,60 @@ def run_ansible(ansible_vars, playbook, host='localhost', user='root',
         e = "Ansible playbook failed. See Ansible logs for details."
         logging.error(e)
         raise Exception(e)
+
+
+def fetch_upstream_and_unpack(dest, url, targets):
+    """
+    Fetches targets from a url destination and downloads them if they are
+    newer.  Also unpacks tar files in dest dir.
+    :param dest: Directory to download and unpack files to
+    :param url: URL where target files are located
+    :param targets: List of target files to download
+    :return: None
+    """
+    os.makedirs(dest, exist_ok=True)
+    assert isinstance(targets, list)
+    for target in targets:
+        download_target = True
+        target_url = urllib.parse.urljoin(url, target)
+        target_dest = os.path.join(dest, target)
+        logging.debug("Fetching and comparing upstream target: \n{}".format(
+            target_url))
+        try:
+            u = urllib.request.urlopen(target_url)
+        except urllib.error.URLError as e:
+            logging.error("Failed to fetch target url. Error: {}".format(
+                e.reason))
+            raise
+        if os.path.isfile(target_dest):
+            logging.debug("Previous file found: {}".format(target_dest))
+            metadata = u.info()
+            headers = metadata.items()
+            target_url_date = None
+            for header in headers:
+                if isinstance(header, tuple) and len(header) == 2:
+                    if header[0] == 'Last-Modified':
+                        target_url_date = header[1]
+                        break
+            if target_url_date is not None:
+                target_dest_mtime = os.path.getmtime(target_dest)
+                target_url_mtime = time.mktime(
+                    datetime.datetime.strptime(target_url_date,
+                                               "%a, %d %b %Y %X "
+                                               "GMT").timetuple())
+                if target_url_mtime > target_dest_mtime:
+                    logging.debug('URL target is newer than disk...will '
+                                  'download')
+                else:
+                    logging.info('URL target does not need to be downloaded')
+                    download_target = False
+            else:
+                logging.debug('Unable to find last modified url date')
+        if download_target:
+            urllib.request.urlretrieve(target_url, filename=target_dest)
+            logging.info("Target downloaded: {}".format(target))
+        if target.endswith('.tar'):
+            logging.info('Unpacking tar file')
+            tar = tarfile.open(target_dest)
+            tar.extractall(path=dest)
+            tar.close()