diff options
Diffstat (limited to 'clover/spark/docker/spark-submit')
| -rw-r--r-- | clover/spark/docker/spark-submit/Dockerfile | 23 | ||||
| -rwxr-xr-x | clover/spark/docker/spark-submit/build.sh | 17 | ||||
| -rwxr-xr-x | clover/spark/docker/spark-submit/runner.sh | 31 | ||||
| -rwxr-xr-x | clover/spark/docker/spark-submit/runner_fast.sh | 26 |
4 files changed, 97 insertions, 0 deletions
diff --git a/clover/spark/docker/spark-submit/Dockerfile b/clover/spark/docker/spark-submit/Dockerfile new file mode 100644 index 0000000..898df1a --- /dev/null +++ b/clover/spark/docker/spark-submit/Dockerfile @@ -0,0 +1,23 @@ +# Copyright (c) Authors of Clover +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 + +FROM java:8 + +RUN mkdir /spark +WORKDIR /spark +RUN wget https://archive.apache.org/dist/spark/spark-2.3.2/spark-2.3.2-bin-hadoop2.7.tgz +# COPY spark-2.3.2-bin-hadoop2.7.tgz /spark +RUN tar -xvzf spark-2.3.2-bin-hadoop2.7.tgz + +COPY runner.sh /spark/spark-2.3.2-bin-hadoop2.7 +COPY runner_fast.sh /spark/spark-2.3.2-bin-hadoop2.7 +WORKDIR /spark/spark-2.3.2-bin-hadoop2.7 + +RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl +RUN chmod +x ./kubectl + +CMD ./runner.sh diff --git a/clover/spark/docker/spark-submit/build.sh b/clover/spark/docker/spark-submit/build.sh new file mode 100755 index 0000000..1bcbbc2 --- /dev/null +++ b/clover/spark/docker/spark-submit/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright (c) Authors of Clover +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# + +IMAGE_PATH=${IMAGE_PATH:-"localhost:5000"} +IMAGE_NAME=${IMAGE_NAME:-"clover-spark-submit:latest"} + +docker build -f Dockerfile -t $IMAGE_NAME . +docker tag $IMAGE_NAME $IMAGE_PATH/$IMAGE_NAME +docker push $IMAGE_PATH/$IMAGE_NAME + diff --git a/clover/spark/docker/spark-submit/runner.sh b/clover/spark/docker/spark-submit/runner.sh new file mode 100755 index 0000000..b98ff32 --- /dev/null +++ b/clover/spark/docker/spark-submit/runner.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Copyright (c) Authors of Clover +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# + +# Fast refresh analytics +./runner_fast.sh & + +IMAGE_NAME=${IMAGE_NAME:-"clover-spark:latest"} +IMAGE_PATH=${IMAGE_PATH:-"localhost:5000"} +CLASS_NAME=${CLASS_NAME:-"CloverSlow"} +JAR_NAME=${JAR_NAME:-"clover-spark_2.11-1.0.jar"} + +bin/spark-submit \ + --master k8s://https://kubernetes.default.svc \ + --deploy-mode cluster \ + --name "clover-spark" \ + --class $CLASS_NAME \ + --conf spark.executor.instances=2 \ + --conf spark.kubernetes.container.image="$IMAGE_PATH/$IMAGE_NAME" \ + --conf spark.kubernetes.authenticate.driver.serviceAccountName="clover-spark" \ + --conf spark.kubernetes.namespace="clover-system" \ + --jars local:///opt/spark/jars/redisclient_2.11-3.7.jar,local:///opt/spark/jars/datastax_spark-cassandra-connector-2.3.0-s_2.11.jar \ + local:///opt/spark/jars/$JAR_NAME + +./kubectl -n clover-system delete pod,svc -l spark-role=driver diff --git a/clover/spark/docker/spark-submit/runner_fast.sh b/clover/spark/docker/spark-submit/runner_fast.sh new file mode 100755 index 0000000..2381351 --- /dev/null +++ b/clover/spark/docker/spark-submit/runner_fast.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Copyright (c) Authors of Clover +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# + +IMAGE_NAME=${IMAGE_NAME:-"clover-spark:latest"} +IMAGE_PATH=${IMAGE_PATH:-"localhost:5000"} +CLASS_NAME=${CLASS_NAME:-"CloverFast"} +JAR_NAME=${JAR_NAME:-"clover-spark_2.11-1.0.jar"} + +bin/spark-submit \ + --master k8s://https://kubernetes.default.svc \ + --deploy-mode cluster \ + --name "clover-spark-fast" \ + --class $CLASS_NAME \ + --conf spark.executor.instances=2 \ + --conf spark.kubernetes.container.image="$IMAGE_PATH/$IMAGE_NAME" \ + --conf spark.kubernetes.authenticate.driver.serviceAccountName="clover-spark" \ + --conf spark.kubernetes.namespace="clover-system" \ + --jars local:///opt/spark/jars/redisclient_2.11-3.7.jar,local:///opt/spark/jars/datastax_spark-cassandra-connector-2.3.0-s_2.11.jar \ + local:///opt/spark/jars/$JAR_NAME |