diff options
| -rwxr-xr-x | clover/spark/docker/clover-spark/build.sh | 2 | ||||
| -rwxr-xr-x | clover/spark/docker/spark-submit/runner.sh | 2 | ||||
| -rwxr-xr-x | clover/spark/docker/spark-submit/runner_fast.sh | 2 | ||||
| -rw-r--r-- | clover/spark/src/main/scala/CloverSlow.scala | 50 |
4 files changed, 28 insertions, 28 deletions
diff --git a/clover/spark/docker/clover-spark/build.sh b/clover/spark/docker/clover-spark/build.sh index a1a8788..d139b35 100755 --- a/clover/spark/docker/clover-spark/build.sh +++ b/clover/spark/docker/clover-spark/build.sh @@ -7,7 +7,7 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 -IMAGE_PATH=${IMAGE_PATH:-"kube1-node1:5000"} +IMAGE_PATH=${IMAGE_PATH:-"localhost:5000"} IMAGE_NAME=${IMAGE_NAME:-"clover-spark:latest"} # Copy clover-spark jar first diff --git a/clover/spark/docker/spark-submit/runner.sh b/clover/spark/docker/spark-submit/runner.sh index b98ff32..5de3824 100755 --- a/clover/spark/docker/spark-submit/runner.sh +++ b/clover/spark/docker/spark-submit/runner.sh @@ -12,7 +12,7 @@ ./runner_fast.sh & IMAGE_NAME=${IMAGE_NAME:-"clover-spark:latest"} -IMAGE_PATH=${IMAGE_PATH:-"localhost:5000"} +IMAGE_PATH=${IMAGE_PATH:-"opnfv"} CLASS_NAME=${CLASS_NAME:-"CloverSlow"} JAR_NAME=${JAR_NAME:-"clover-spark_2.11-1.0.jar"} diff --git a/clover/spark/docker/spark-submit/runner_fast.sh b/clover/spark/docker/spark-submit/runner_fast.sh index 2381351..0a387b2 100755 --- a/clover/spark/docker/spark-submit/runner_fast.sh +++ b/clover/spark/docker/spark-submit/runner_fast.sh @@ -9,7 +9,7 @@ # IMAGE_NAME=${IMAGE_NAME:-"clover-spark:latest"} -IMAGE_PATH=${IMAGE_PATH:-"localhost:5000"} +IMAGE_PATH=${IMAGE_PATH:-"opnfv"} CLASS_NAME=${CLASS_NAME:-"CloverFast"} JAR_NAME=${JAR_NAME:-"clover-spark_2.11-1.0.jar"} diff --git a/clover/spark/src/main/scala/CloverSlow.scala b/clover/spark/src/main/scala/CloverSlow.scala index 1866d72..c389967 100644 --- a/clover/spark/src/main/scala/CloverSlow.scala +++ b/clover/spark/src/main/scala/CloverSlow.scala @@ -42,38 +42,38 @@ object CloverSlow { .config("spark.cassandra.connection.port", "9042") .getOrCreate() - val services = redis.smembers("visibility_services") - spark .read.cassandraFormat("spans", "visibility") .load() .createOrReplaceTempView("curspans") - if (distinct_url_service) { - // Get number of distinct URLs per service (node_id) - for (s <- services.get) { - val service = s.get - val perurl = spark.sql( - s""" - |SELECT node_id,count(distinct http_url) - |as urls,collect_set(http_url) as values - |FROM curspans - |WHERE node_id LIKE '%$service%' - |GROUP BY node_id - """.stripMargin) - for ((row) <- perurl.collect) { - println(row) - val node_id = row.get(0) - val url_count = row.get(1) - val url_distinct = row.getList(2).toString - redis.hmset(service, Map("node_id" -> node_id, - "url_count" -> url_count, - "url_distinct" -> url_distinct)) + for( x <- 1 to 500 ) { + + val services = redis.smembers("visibility_services") + + if (distinct_url_service) { + // Get number of distinct URLs per service (node_id) + for (s <- services.get) { + val service = s.get + val perurl = spark.sql( + s""" + |SELECT node_id,count(distinct http_url) + |as urls,collect_set(http_url) as values + |FROM curspans + |WHERE node_id LIKE '%$service%' + |GROUP BY node_id + """.stripMargin) + for ((row) <- perurl.collect) { + println(row) + val node_id = row.get(0) + val url_count = row.get(1) + val url_distinct = row.getList(2).toString + redis.hmset(service, Map("node_id" -> node_id, + "url_count" -> url_count, + "url_distinct" -> url_distinct)) + } } } - } - - for( x <- 1 to 500 ) { if (response_times) { try { |