update docker to support spark master (#1448)

intel-analytics · Jun 18, 2019 · 278e4a9dd2eb00e4e1eb2c965de1fb4ef3558ec9 · 278e4a9
1 parent 11d31d1
commit 278e4a9dd2eb00e4e1eb2c965de1fb4ef3558ec9
Unified Split

Showing with 34 additions and 3 deletions.

+3 −1 docker/zoo/Dockerfile

+1 −0 docker/zoo/README.md

+30 −2 docker/zoo/start-notebook.sh
diff --git a/docker/zoo/Dockerfile b/docker/zoo/Dockerfile
@@ -23,7 +23,8 @@ WORKDIR /opt/work
 ARG PY_VERSION_3
 ARG ANALYTICS_ZOO_VERSION=0.6.0-SNAPSHOT
 ARG BIGDL_VERSION=0.8.0
-  
-ARG SPARK_VERSION=2.4.0
+  
+ARG SPARK_VERSION=2.4.3
+  
+ARG RUNTIME_SPARK_MASTER=local[4]
 ARG RUNTIME_DRIVER_CORES=4
 ARG RUNTIME_DRIVER_MEMORY=20g
 ARG RUNTIME_EXECUTOR_CORES=4
@@ -32,6 +33,7 @@ ARG RUNTIME_TOTAL_EXECUTOR_CORES=4
 ENV ANALYTICS_ZOO_VERSION           ${ANALYTICS_ZOO_VERSION}
 ENV SPARK_VERSION                   ${SPARK_VERSION}
 ENV BIGDL_VERSION                   ${BIGDL_VERSION}
+  
+ENV RUNTIME_SPARK_MASTER            ${RUNTIME_SPARK_MASTER}
 ENV RUNTIME_DRIVER_CORES            ${RUNTIME_DRIVER_CORES}
 ENV RUNTIME_DRIVER_MEMORY           ${RUNTIME_DRIVER_MEMORY}
 ENV RUNTIME_EXECUTOR_CORES          ${RUNTIME_EXECUTOR_CORES}

diff --git a/docker/zoo/README.md b/docker/zoo/README.md
@@ -143,6 +143,7 @@
        -e NotebookToken="1234qwer" \
        -e http_proxy=http://your-proxy-host:your-proxy-port  \
        -e https_proxy=https://your-proxy-host:your-proxy-port  \
+  
+        -e RUNTIME_SPARK_MASTER=spark://your-spark-master-host:your-spark-master-port or local[*] \
        -e RUNTIME_DRIVER_CORES=4 \
        -e RUNTIME_DRIVER_MEMORY=20g \
        -e RUNTIME_EXECUTOR_CORES=4 \

diff --git a/docker/zoo/start-notebook.sh b/docker/zoo/start-notebook.sh
@@ -30,15 +30,42 @@ echo $SPARK_MAJOR_VERSION
 export PYSPARK_DRIVER_PYTHON=jupyter
 export PYSPARK_DRIVER_PYTHON_OPTS="notebook --notebook-dir=$ANALYTICS_ZOO_TUTORIALS_HOME --ip=0.0.0.0 --port=$NotebookPort --no-browser --NotebookApp.token=$NotebookToken --allow-root"
+  
+echo $RUNTIME_SPARK_MASTER
 echo $RUNTIME_EXECUTOR_CORES
 echo $RUNTIME_DRIVER_CORES
 echo $RUNTIME_DRIVER_MEMORY
 echo $RUNTIME_EXECUTOR_CORES
 echo $RUNTIME_EXECUTOR_MEMORY
 echo $RUNTIME_TOTAL_EXECUTOR_CORES
+  
+if [ -z "${KMP_AFFINITY}" ]; then
+  
+    export KMP_AFFINITY=granularity=fine,compact,1,0
+  
+fi
+  
+
+  
+if [ -z "${OMP_NUM_THREADS}" ]; then
+  
+    if [ -z "${ZOO_NUM_MKLTHREADS}" ]; then
+  
+        export OMP_NUM_THREADS=1
+  
+    else
+  
+        if [ `echo $ZOO_NUM_MKLTHREADS | tr '[A-Z]' '[a-z]'` == "all" ]; then
+  
+            export OMP_NUM_THREADS=`nproc`
+  
+        else
+  
+            export OMP_NUM_THREADS=${ZOO_NUM_MKLTHREADS}
+  
+        fi
+  
+    fi
+  
+fi
+  
+
+  
+if [ -z "${KMP_BLOCKTIME}" ]; then
+  
+    export KMP_BLOCKTIME=0
+  
+fi
+  
+
+  
+# verbose for OpenMP
+  
+if [[ $* == *"verbose"* ]]; then
+  
+    export KMP_SETTINGS=1
+  
+    export KMP_AFFINITY=${KMP_AFFINITY},verbose
+  
+fi
+  
+
 ${SPARK_HOME}/bin/pyspark \
-  
-  --master local[${RUNTIME_EXECUTOR_CORES}] \
+  
+  --master ${RUNTIME_SPARK_MASTER} \
  --driver-cores ${RUNTIME_DRIVER_CORES} \
  --driver-memory ${RUNTIME_DRIVER_MEMORY} \
  --executor-cores ${RUNTIME_EXECUTOR_CORES} \
@@ -49,4 +76,5 @@ ${SPARK_HOME}/bin/pyspark \
  --jars ${ANALYTICS_ZOO_HOME}/lib/analytics-zoo-bigdl_${BIGDL_VERSION}-spark_${SPARK_VERSION}-${ANALYTICS_ZOO_VERSION}-jar-with-dependencies.jar \
  --conf spark.driver.extraClassPath=${ANALYTICS_ZOO_HOME}/lib/analytics-zoo-bigdl_${BIGDL_VERSION}-spark_${SPARK_VERSION}-${ANALYTICS_ZOO_VERSION}-jar-with-dependencies.jar \
  --conf spark.executor.extraClassPath=${ANALYTICS_ZOO_HOME}/lib/analytics-zoo-bigdl_${BIGDL_VERSION}-spark_${SPARK_VERSION}-${ANALYTICS_ZOO_VERSION}-jar-with-dependencies.jar \
-  
-  --conf spark.driver.extraJavaOptions=-Dderby.stream.error.file=/tmp
+  
+  --conf spark.driver.extraJavaOptions=-Dderby.stream.error.file=/tmp \
+  
+  --conf spark.sql.catalogImplementation='in-memory'