From fe167ee5b7cb84897c06fdc7a8b20342defff9bf Mon Sep 17 00:00:00 2001
From: SerenaFeng <feng.xiaowei@zte.com.cn>
Date: Tue, 27 Jun 2017 19:13:32 +0800
Subject: bugfix: pagination crash due to memory limitation

MongoDB sorts the results in memory, and the default mem limitation is
32M, if the sort operation consumes more than that it will return an
error:
OperationFailure: Executor error during find command: OperationFailed
Sort operation used more than the maximum 33554432 bytes of RAM. Add an
index, or specify a smaller limit.

To solve this problem, here we leverage aggregate() and
allowDiskUse=True, it is said will not be limited by memory

Change-Id: Id698ad1d02912e8b350a33a926fcccc390814fcc
Signed-off-by: SerenaFeng <feng.xiaowei@zte.com.cn>
---
 .../testapi/opnfv_testapi/resources/handlers.py    | 35 +++++++++++++++-------
 1 file changed, 25 insertions(+), 10 deletions(-)

(limited to 'utils/test/testapi/opnfv_testapi/resources/handlers.py')

diff --git a/utils/test/testapi/opnfv_testapi/resources/handlers.py b/utils/test/testapi/opnfv_testapi/resources/handlers.py
index 42372e837..0234c8a73 100644
--- a/utils/test/testapi/opnfv_testapi/resources/handlers.py
+++ b/utils/test/testapi/opnfv_testapi/resources/handlers.py
@@ -105,21 +105,36 @@ class GenericApiHandler(web.RequestHandler):
             query = {}
         data = []
         sort = kwargs.get('sort')
-        page = kwargs.get('page')
-        last = kwargs.get('last')
-        per_page = kwargs.get('per_page')
+        page = kwargs.get('page', 0)
+        last = kwargs.get('last', 0)
+        per_page = kwargs.get('per_page', 0)
 
         cursor = self._eval_db(self.table, 'find', query)
+        records_count = yield cursor.count()
+        records_nr = records_count
+        if (records_count > last) and (last > 0):
+            records_nr = last
+
+        pipelines = list()
+        if query:
+            pipelines.append({'$match': query})
         if sort:
-            cursor = cursor.sort(sort)
-        if last and last != 0:
-            cursor = cursor.limit(last)
-        if page:
-            records_count = yield cursor.count()
-            total_pages, remainder = divmod(records_count, per_page)
+            pipelines.append({'$sort': sort})
+
+        if page > 0:
+            total_pages, remainder = divmod(records_nr, per_page)
             if remainder > 0:
                 total_pages += 1
-            cursor = cursor.skip((page - 1) * per_page).limit(per_page)
+            pipelines.append({'$skip': (page - 1) * per_page})
+            pipelines.append({'$limit': per_page})
+        else:
+            pipelines.append({'$limit': records_nr})
+
+        cursor = self._eval_db(self.table,
+                               'aggregate',
+                               pipelines,
+                               allowDiskUse=True)
+
         while (yield cursor.fetch_next):
             data.append(self.format_data(cursor.next_object()))
         if res_op is None:
-- 
cgit 1.2.3-korg