Add function to upload image from local/url in GUI

JIRA: YARDSTICK-782

As user, we need to upload image from local/url.
If upload image from local, user need to choose local image, then
we will load it to openstack.
If upload image from url, we will download it and load it to openstack.

Change-Id: Ia9a42fda15a1dfc91476643635343a2f77a94a6b
Signed-off-by: chenjiankun <chenjiankun1@huawei.com>

6 files changed, 328 insertions, 31 deletions

diff --git a/api/database/v2/handlers.py b/api/database/v2/handlers.py
index 1bc32bf0e..e4f1dd668 100644
--- a/api/database/v2/handlers.py
+++ b/api/database/v2/handlers.py
@@ -87,6 +87,11 @@ class V2ImageHandler(object):
             raise ValueError
         return image
 
+    def delete_by_uuid(self, uuid):
+        image = self.get_by_uuid(uuid)
+        db_session.delete(image)
+        db_session.commit()
+
 
 class V2PodHandler(object):
 
diff --git a/api/database/v2/models.py b/api/database/v2/models.py
index 1e85559cb..59dab3ebc 100644
--- a/api/database/v2/models.py
+++ b/api/database/v2/models.py
@@ -48,9 +48,6 @@ class V2Image(Base):
     name = Column(String(30))
     description = Column(Text)
     environment_id = Column(String(30))
-    size = Column(String(30))
-    status = Column(String(30))
-    time = Column(DateTime)
 
 
 class V2Container(Base):
diff --git a/api/resources/v2/environments.py b/api/resources/v2/environments.py
index f021a3c5a..158e98be7 100644
--- a/api/resources/v2/environments.py
+++ b/api/resources/v2/environments.py
@@ -35,6 +35,9 @@ class V2Environments(ApiResource):
             container_info = e['container_id']
             e['container_id'] = jsonutils.loads(container_info) if container_info else {}
 
+            image_id = e['image_id']
+            e['image_id'] = image_id.split(',') if image_id else []
+
         data = {
             'environments': environments
         }
@@ -78,8 +81,13 @@ class V2Environment(ApiResource):
             return result_handler(consts.API_ERROR, 'no such environment id')
 
         environment = change_obj_to_dict(environment)
+
         container_id = environment['container_id']
         environment['container_id'] = jsonutils.loads(container_id) if container_id else {}
+
+        image_id = environment['image_id']
+        environment['image_id'] = image_id.split(',') if image_id else []
+
         return result_handler(consts.API_SUCCESS, {'environment': environment})
 
     def delete(self, environment_id):
diff --git a/api/resources/v2/images.py b/api/resources/v2/images.py
index 8359e105b..0c36a0a26 100644
--- a/api/resources/v2/images.py
+++ b/api/resources/v2/images.py
@@ -7,76 +7,361 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 import logging
-import subprocess
+import os
+import uuid
 import threading
+import requests
+import datetime
 
 from api import ApiResource
+from api.database.v2.handlers import V2ImageHandler
+from api.database.v2.handlers import V2EnvironmentHandler
 from yardstick.common.utils import result_handler
 from yardstick.common.utils import source_env
 from yardstick.common.utils import change_obj_to_dict
 from yardstick.common.openstack_utils import get_nova_client
+from yardstick.common.openstack_utils import get_glance_client
 from yardstick.common import constants as consts
 
 LOG = logging.getLogger(__name__)
 LOG.setLevel(logging.DEBUG)
 
+IMAGE_MAP = {
+    'yardstick-image': {
+        'path': os.path.join(consts.IMAGE_DIR, 'yardstick-image.img'),
+        'url': 'http://artifacts.opnfv.org/yardstick/images/yardstick-image.img'
+    },
+    'Ubuntu-16.04': {
+        'path': os.path.join(consts.IMAGE_DIR, 'xenial-server-cloudimg-amd64-disk1.img'),
+        'url': 'cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img'
+    },
+    'cirros-0.3.5': {
+        'path': os.path.join(consts.IMAGE_DIR, 'cirros-0.3.5-x86_64-disk.img'),
+        'url': 'http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img'
+    }
+}
+
 
 class V2Images(ApiResource):
 
     def get(self):
         try:
             source_env(consts.OPENRC)
-        except:
+        except Exception:
             return result_handler(consts.API_ERROR, 'source openrc error')
 
         nova_client = get_nova_client()
         try:
             images_list = nova_client.images.list()
-        except:
+        except Exception:
             return result_handler(consts.API_ERROR, 'get images error')
         else:
-            images = [self.get_info(change_obj_to_dict(i)) for i in images_list]
-            status = 1 if all(i['status'] == 'ACTIVE' for i in images) else 0
-            if not images:
-                status = 0
+            images = {i.name: self.get_info(change_obj_to_dict(i)) for i in images_list}
 
-        return result_handler(consts.API_SUCCESS, {'status': status, 'images': images})
+        return result_handler(consts.API_SUCCESS, {'status': 1, 'images': images})
 
     def post(self):
         return self._dispatch_post()
 
     def get_info(self, data):
+        try:
+            size = data['OS-EXT-IMG-SIZE:size']
+        except KeyError:
+            size = None
+        else:
+            size = float(size) / 1024 / 1024
+
         result = {
             'name': data.get('name', ''),
-            'size': data.get('OS-EXT-IMG-SIZE:size', ''),
-            'status': data.get('status', ''),
-            'time': data.get('updated', '')
+            'discription': data.get('description', ''),
+            'size': size,
+            'status': data.get('status'),
+            'time': data.get('updated')
         }
         return result
 
     def load_image(self, args):
-        thread = threading.Thread(target=self._load_images)
+        try:
+            image_name = args['name']
+        except KeyError:
+            return result_handler(consts.API_ERROR, 'image name must provided')
+
+        if image_name not in IMAGE_MAP:
+            return result_handler(consts.API_ERROR, 'wrong image name')
+
+        thread = threading.Thread(target=self._do_load_image, args=(image_name,))
         thread.start()
+        return result_handler(consts.API_SUCCESS, {'image': image_name})
+
+    def upload_image(self, args):
+        try:
+            image_file = args['file']
+        except KeyError:
+            return result_handler(consts.API_ERROR, 'file must be provided')
+
+        try:
+            environment_id = args['environment_id']
+        except KeyError:
+            return result_handler(consts.API_ERROR, 'environment_id must be provided')
+
+        try:
+            uuid.UUID(environment_id)
+        except ValueError:
+            return result_handler(consts.API_ERROR, 'invalid environment id')
+
+        environment_handler = V2EnvironmentHandler()
+        try:
+            environment = environment_handler.get_by_uuid(environment_id)
+        except ValueError:
+            return result_handler(consts.API_ERROR, 'no such environment')
+
+        file_path = os.path.join(consts.IMAGE_DIR, image_file.filename)
+        LOG.info('saving file')
+        image_file.save(file_path)
+
+        LOG.info('loading image')
+        self._load_image(image_file.filename, file_path)
+
+        LOG.info('creating image in DB')
+        image_handler = V2ImageHandler()
+        image_id = str(uuid.uuid4())
+        image_init_data = {
+            'uuid': image_id,
+            'name': image_file.filename,
+            'environment_id': environment_id
+        }
+        image_handler.insert(image_init_data)
+
+        LOG.info('update image in environment')
+        if environment.image_id:
+            image_list = environment.image_id.split(',')
+            image_list.append(image_id)
+            new_image_id = ','.join(image_list)
+        else:
+            new_image_id = image_id
+
+        environment_handler.update_attr(environment_id, {'image_id': new_image_id})
+
+        return result_handler(consts.API_SUCCESS, {'uuid': image_id})
+
+    def upload_image_by_url(self, args):
+        try:
+            url = args['url']
+        except KeyError:
+            return result_handler(consts.API_ERROR, 'url must be provided')
+
+        try:
+            environment_id = args['environment_id']
+        except KeyError:
+            return result_handler(consts.API_ERROR, 'environment_id must be provided')
+
+        try:
+            uuid.UUID(environment_id)
+        except ValueError:
+            return result_handler(consts.API_ERROR, 'invalid environment id')
+
+        environment_handler = V2EnvironmentHandler()
+        try:
+            environment = environment_handler.get_by_uuid(environment_id)
+        except ValueError:
+            return result_handler(consts.API_ERROR, 'no such environment')
+
+        thread = threading.Thread(target=self._do_upload_image_by_url, args=(url,))
+        thread.start()
+
+        file_name = url.split('/')[-1]
+
+        LOG.info('creating image in DB')
+        image_handler = V2ImageHandler()
+        image_id = str(uuid.uuid4())
+        image_init_data = {
+            'uuid': image_id,
+            'name': file_name,
+            'environment_id': environment_id
+        }
+        image_handler.insert(image_init_data)
+
+        LOG.info('update image in environment')
+        if environment.image_id:
+            image_list = environment.image_id.split(',')
+            image_list.append(image_id)
+            new_image_id = ','.join(image_list)
+        else:
+            new_image_id = image_id
+
+        environment_handler.update_attr(environment_id, {'image_id': new_image_id})
+
+        return result_handler(consts.API_SUCCESS, {'uuid': image_id})
+
+    def delete_image(self, args):
+        try:
+            image_name = args['name']
+        except KeyError:
+            return result_handler(consts.API_ERROR, 'image name must provided')
+
+        if image_name not in IMAGE_MAP:
+            return result_handler(consts.API_ERROR, 'wrong image name')
+
+        glance_client = get_glance_client()
+        try:
+            image = next((i for i in glance_client.images.list() if i.name == image_name))
+        except StopIteration:
+            return result_handler(consts.API_ERROR, 'can not find image')
+
+        glance_client.images.delete(image.id)
+
         return result_handler(consts.API_SUCCESS, {})
 
-    def _load_images(self):
+    def _do_upload_image_by_url(self, url):
+        file_name = url.split('/')[-1]
+        path = os.path.join(consts.IMAGE_DIR, file_name)
+
+        LOG.info('download image')
+        self._download_image(url, path)
+
+        LOG.info('loading image')
+        self._load_image(file_name, path)
+
+    def _do_load_image(self, image_name):
+        if not os.path.exists(IMAGE_MAP[image_name]['path']):
+            self._download_image(IMAGE_MAP[image_name]['url'],
+                                 IMAGE_MAP[image_name]['path'])
+
+        self._load_image(image_name, IMAGE_MAP[image_name]['path'])
+
+    def _load_image(self, image_name, image_path):
         LOG.info('source openrc')
         source_env(consts.OPENRC)
 
-        LOG.info('clean images')
-        cmd = [consts.CLEAN_IMAGES_SCRIPT]
-        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                             cwd=consts.REPOS_DIR)
-        _, err = p.communicate()
-        if p.returncode != 0:
-            LOG.error('clean image failed: %s', err)
-
-        LOG.info('load images')
-        cmd = [consts.LOAD_IMAGES_SCRIPT]
-        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                             cwd=consts.REPOS_DIR)
-        _, err = p.communicate()
-        if p.returncode != 0:
-            LOG.error('load image failed: %s', err)
+        LOG.info('load image')
+        glance_client = get_glance_client()
+        image = glance_client.images.create(name=image_name,
+                                            visibility='public',
+                                            disk_format='qcow2',
+                                            container_format='bare')
+        with open(image_path, 'rb') as f:
+            glance_client.images.upload(image.id, f)
 
         LOG.info('Done')
+
+    def _download_image(self, url, path):
+        start = datetime.datetime.now().replace(microsecond=0)
+
+        LOG.info('download image from: %s', url)
+        self._download_file(url, path)
+
+        end = datetime.datetime.now().replace(microsecond=0)
+        LOG.info('download image success, total: %s s', end - start)
+
+    def _download_handler(self, start, end, url, filename):
+
+        headers = {'Range': 'bytes=%d-%d' % (start, end)}
+        r = requests.get(url, headers=headers, stream=True)
+
+        with open(filename, "r+b") as fp:
+            fp.seek(start)
+            fp.tell()
+            fp.write(r.content)
+
+    def _download_file(self, url, path, num_thread=5):
+
+        r = requests.head(url)
+        try:
+            file_size = int(r.headers['content-length'])
+        except Exception:
+            return
+
+        with open(path, 'wb') as f:
+            f.truncate(file_size)
+
+        thread_list = []
+        part = file_size // num_thread
+        for i in range(num_thread):
+            start = part * i
+            end = start + part if i != num_thread - 1 else file_size
+
+            kwargs = {'start': start, 'end': end, 'url': url, 'filename': path}
+            t = threading.Thread(target=self._download_handler, kwargs=kwargs)
+            t.setDaemon(True)
+            t.start()
+            thread_list.append(t)
+
+        for t in thread_list:
+            t.join()
+
+
+class V2Image(ApiResource):
+    def get(self, image_id):
+        try:
+            uuid.UUID(image_id)
+        except ValueError:
+            return result_handler(consts.API_ERROR, 'invalid image id')
+
+        image_handler = V2ImageHandler()
+        try:
+            image = image_handler.get_by_uuid(image_id)
+        except ValueError:
+            return result_handler(consts.API_ERROR, 'no such image id')
+
+        nova_client = get_nova_client()
+        images = nova_client.images.list()
+        try:
+            image = next((i for i in images if i.name == image.name))
+        except StopIteration:
+            pass
+
+        return_image = self.get_info(change_obj_to_dict(image))
+        return_image['id'] = image_id
+
+        return result_handler(consts.API_SUCCESS, {'image': return_image})
+
+    def delete(self, image_id):
+        try:
+            uuid.UUID(image_id)
+        except ValueError:
+            return result_handler(consts.API_ERROR, 'invalid image id')
+
+        image_handler = V2ImageHandler()
+        try:
+            image = image_handler.get_by_uuid(image_id)
+        except ValueError:
+            return result_handler(consts.API_ERROR, 'no such image id')
+
+        LOG.info('delete image in openstack')
+        glance_client = get_glance_client()
+        try:
+            image_o = next((i for i in glance_client.images.list() if i.name == image.name))
+        except StopIteration:
+            return result_handler(consts.API_ERROR, 'can not find image')
+
+        glance_client.images.delete(image_o.id)
+
+        LOG.info('delete image in environment')
+        environment_id = image.environment_id
+        environment_handler = V2EnvironmentHandler()
+        environment = environment_handler.get_by_uuid(environment_id)
+        image_list = environment.image_id.split(',')
+        image_list.remove(image_id)
+        environment_handler.update_attr(environment_id, {'image_id': ','.join(image_list)})
+
+        LOG.info('delete image in DB')
+        image_handler.delete_by_uuid(image_id)
+
+        return result_handler(consts.API_SUCCESS, {'image': image_id})
+
+    def get_info(self, data):
+        try:
+            size = data['OS-EXT-IMG-SIZE:size']
+        except KeyError:
+            size = None
+        else:
+            size = float(size) / 1024 / 1024
+
+        result = {
+            'name': data.get('name', ''),
+            'description': data.get('description', ''),
+            'size': size,
+            'status': data.get('status'),
+            'time': data.get('updated')
+        }
+        return result
diff --git a/api/server.py b/api/server.py
index 158b8a508..37a1ab6a6 100644
--- a/api/server.py
+++ b/api/server.py
@@ -35,6 +35,7 @@ except ImportError:
 LOG = logging.getLogger(__name__)
 
 app = Flask(__name__)
+app.config['MAX_CONTENT_LENGTH'] = 2 * 1024 * 1024 * 1024
 
 Swagger(app)
 
diff --git a/api/urls.py b/api/urls.py
index 83cf4daf9..9b0040b6c 100644
--- a/api/urls.py
+++ b/api/urls.py
@@ -36,6 +36,7 @@ urlpatterns = [
 
     Url('/api/v2/yardstick/images', 'v2_images'),
     Url('/api/v2/yardstick/images/action', 'v2_images'),
+    Url('/api/v2/yardstick/images/<image_id>', 'v2_image'),
 
     Url('/api/v2/yardstick/containers', 'v2_containers'),
     Url('/api/v2/yardstick/containers/action', 'v2_containers'),