From 47f636d596a95d3333c2e89c3396c3eb94e83b96 Mon Sep 17 00:00:00 2001 From: earrage Date: Fri, 2 Nov 2018 18:26:46 -0700 Subject: Modify container tag to opnfv in spark-submit - Change tag from local repo to opnfv for clover-spark container, which spark-submit references - Move update of configured visibility services into loop Change-Id: I9d09a851977c6006ea2cbba45d7a28be6a4660b9 Signed-off-by: earrage --- clover/spark/docker/clover-spark/build.sh | 2 +- clover/spark/docker/spark-submit/runner.sh | 2 +- clover/spark/docker/spark-submit/runner_fast.sh | 2 +- clover/spark/src/main/scala/CloverSlow.scala | 50 ++++++++++++------------- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/clover/spark/docker/clover-spark/build.sh b/clover/spark/docker/clover-spark/build.sh index a1a8788..d139b35 100755 --- a/clover/spark/docker/clover-spark/build.sh +++ b/clover/spark/docker/clover-spark/build.sh @@ -7,7 +7,7 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 -IMAGE_PATH=${IMAGE_PATH:-"kube1-node1:5000"} +IMAGE_PATH=${IMAGE_PATH:-"localhost:5000"} IMAGE_NAME=${IMAGE_NAME:-"clover-spark:latest"} # Copy clover-spark jar first diff --git a/clover/spark/docker/spark-submit/runner.sh b/clover/spark/docker/spark-submit/runner.sh index b98ff32..5de3824 100755 --- a/clover/spark/docker/spark-submit/runner.sh +++ b/clover/spark/docker/spark-submit/runner.sh @@ -12,7 +12,7 @@ ./runner_fast.sh & IMAGE_NAME=${IMAGE_NAME:-"clover-spark:latest"} -IMAGE_PATH=${IMAGE_PATH:-"localhost:5000"} +IMAGE_PATH=${IMAGE_PATH:-"opnfv"} CLASS_NAME=${CLASS_NAME:-"CloverSlow"} JAR_NAME=${JAR_NAME:-"clover-spark_2.11-1.0.jar"} diff --git a/clover/spark/docker/spark-submit/runner_fast.sh b/clover/spark/docker/spark-submit/runner_fast.sh index 2381351..0a387b2 100755 --- a/clover/spark/docker/spark-submit/runner_fast.sh +++ b/clover/spark/docker/spark-submit/runner_fast.sh @@ -9,7 +9,7 @@ # IMAGE_NAME=${IMAGE_NAME:-"clover-spark:latest"} -IMAGE_PATH=${IMAGE_PATH:-"localhost:5000"} +IMAGE_PATH=${IMAGE_PATH:-"opnfv"} CLASS_NAME=${CLASS_NAME:-"CloverFast"} JAR_NAME=${JAR_NAME:-"clover-spark_2.11-1.0.jar"} diff --git a/clover/spark/src/main/scala/CloverSlow.scala b/clover/spark/src/main/scala/CloverSlow.scala index 1866d72..c389967 100644 --- a/clover/spark/src/main/scala/CloverSlow.scala +++ b/clover/spark/src/main/scala/CloverSlow.scala @@ -42,38 +42,38 @@ object CloverSlow { .config("spark.cassandra.connection.port", "9042") .getOrCreate() - val services = redis.smembers("visibility_services") - spark .read.cassandraFormat("spans", "visibility") .load() .createOrReplaceTempView("curspans") - if (distinct_url_service) { - // Get number of distinct URLs per service (node_id) - for (s <- services.get) { - val service = s.get - val perurl = spark.sql( - s""" - |SELECT node_id,count(distinct http_url) - |as urls,collect_set(http_url) as values - |FROM curspans - |WHERE node_id LIKE '%$service%' - |GROUP BY node_id - """.stripMargin) - for ((row) <- perurl.collect) { - println(row) - val node_id = row.get(0) - val url_count = row.get(1) - val url_distinct = row.getList(2).toString - redis.hmset(service, Map("node_id" -> node_id, - "url_count" -> url_count, - "url_distinct" -> url_distinct)) + for( x <- 1 to 500 ) { + + val services = redis.smembers("visibility_services") + + if (distinct_url_service) { + // Get number of distinct URLs per service (node_id) + for (s <- services.get) { + val service = s.get + val perurl = spark.sql( + s""" + |SELECT node_id,count(distinct http_url) + |as urls,collect_set(http_url) as values + |FROM curspans + |WHERE node_id LIKE '%$service%' + |GROUP BY node_id + """.stripMargin) + for ((row) <- perurl.collect) { + println(row) + val node_id = row.get(0) + val url_count = row.get(1) + val url_distinct = row.getList(2).toString + redis.hmset(service, Map("node_id" -> node_id, + "url_count" -> url_count, + "url_distinct" -> url_distinct)) + } } } - } - - for( x <- 1 to 500 ) { if (response_times) { try { -- cgit 1.2.3-korg