Init the folder structure (#1)

* update * update * update
intel-analytics · Mar 29, 2018 · 302f98e895ada22a389b02bcbecf85545cd42439 · 302f98e
1 parent 53a9957
commit 302f98e895ada22a389b02bcbecf85545cd42439
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,37 @@
+  
+# Compile products
+  
+*.pyc
+  
+*.class
+  
+target/
+  
+/dist/
+  
+
+  
+# SBT, Maven specific
+  
+.cache
+  
+.history
+  
+.lib/
+  
+lib_managed/
+  
+src_managed/
+  
+project/boot/
+  
+project/plugins/project/
+  
+dependency-reduced-pom.xml
+  
+
+  
+# IDE specific
+  
+.scala_dependencies
+  
+.worksheet
+  
+*.iml
+  
+.idea/
+  
+
+  
+# macOS specific
+  
+.DS_Store
+  
+
+  
+# data files
+  
+*.csv
+  
+model*.[0-9]*
+  
+state*.[0-9]*
+  
+
+  
+# other
+  
+nohup.out
+  
+*.log
+  
+*.un~
+  
+.ipynb_checkpoints/
+  
+pyspark/dist/
+  
+pyspark/build/
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+  
+[submodule "backend/bigdl"]
+  
+	path = backend/bigdl
+  
+	url = https://github.com/intel-analytics/BigDL.git
diff --git a/assembly/python-zip.xml b/assembly/python-zip.xml
@@ -0,0 +1,18 @@
+  
+<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.0.0"
+  
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  
+          xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.0.0 http://maven.apache.org/xsd/assembly-2.0.0.xsd">
+  
+    <id>python-api</id>
+  
+    <formats>
+  
+        <format>zip</format>
+  
+    </formats>
+  
+    <includeBaseDirectory>false</includeBaseDirectory>
+  
+    <fileSets>
+  
+        <fileSet>
+  
+            <includes>
+  
+                <include>**/*.py</include>
+  
+            </includes>
+  
+            <outputDirectory>/..</outputDirectory>
+  
+            <directory>pyzoo/</directory>
+  
+        </fileSet>
+  
+    </fileSets>
+  
+</assembly>
diff --git a/backend/bigdl b/backend/bigdl
diff --git a/docs/.keep b/docs/.keep
diff --git a/make-dist.sh b/make-dist.sh
@@ -0,0 +1,69 @@
+  
+#!/usr/bin/env bash
+  
+
+  
+#
+  
+# Copyright 2016 The BigDL Authors.
+  
+#
+  
+# Licensed under the Apache License, Version 2.0 (the "License");
+  
+# you may not use this file except in compliance with the License.
+  
+# You may obtain a copy of the License at
+  
+#
+  
+#     http://www.apache.org/licenses/LICENSE-2.0
+  
+#
+  
+# Unless required by applicable law or agreed to in writing, software
+  
+# distributed under the License is distributed on an "AS IS" BASIS,
+  
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  
+# See the License for the specific language governing permissions and
+  
+# limitations under the License.
+  
+#
+  
+
+  
+#
+  
+# Create a folder contain all files for dist
+  
+#
+  
+
+  
+set -e
+  
+
+  
+# Check java
+  
+if type -p java>/dev/null; then
+  
+    _java=java
+  
+else
+  
+    echo "Java is not installed"
+  
+    exit 1
+  
+fi
+  
+
+  
+MVN_OPTS_LIST="-Xmx2g -XX:ReservedCodeCacheSize=512m"
+  
+
+  
+if [[ "$_java" ]]; then
+  
+    version=$("$_java" -version 2>&1 | awk -F '"' '/version/ {print $2}')
+  
+    if [[ "$version" < "1.7" ]]; then
+  
+        echo Require a java version not lower than 1.7
+  
+        exit 1
+  
+    fi
+  
+    # For jdk7
+  
+    if [[ "$version" < "1.8" ]]; then
+  
+        MVN_OPTS_LIST="$MVN_OPTS_LIST -XX:MaxPermSize=1G"
+  
+    fi
+  
+fi
+  
+
+  
+export MAVEN_OPTS=${MAVEN_OPTS:-"$MVN_OPTS_LIST"}
+  
+
+  
+# Check if mvn installed
+  
+MVN_INSTALL=$(which mvn 2>/dev/null | grep mvn | wc -l)
+  
+if [ $MVN_INSTALL -eq 0 ]; then
+  
+  echo "MVN is not installed. Exit"
+  
+  exit 1
+  
+fi
+  
+
+  
+mvn clean package -DskipTests $*
+  
+
+  
+BASEDIR=$(dirname "$0")
+  
+DIST_DIR=$BASEDIR/dist
+  
+
+  
+if [ ! -d "$DIST_DIR" ]
+  
+then
+  
+  mkdir $DIST_DIR
+  
+else
+  
+  rm -r $DIST_DIR
+  
+  mkdir $DIST_DIR
+  
+fi
+  
+
+  
+cp -r $BASEDIR/scala/dist/target/*/* ./dist/
diff --git a/pom.xml b/pom.xml
@@ -0,0 +1,120 @@
+  
+<?xml version="1.0" encoding="UTF-8"?>
+  
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  
+    <modelVersion>4.0.0</modelVersion>
+  
+
+  
+    <groupId>com.intel.analytics.zoo</groupId>
+  
+    <artifactId>zoo-parent</artifactId>
+  
+    <packaging>pom</packaging>
+  
+    <version>0.1.0-SNAPSHOT</version>
+  
+
+  
+    <name>zoo-parent</name>
+  
+    <description>A distributed analytics library for Apache Spark.</description>
+  
+    <url>https://github.com/intel-analytics/zoo</url>
+  
+
+  
+    <developers>
+  
+        <developer>
+  
+            <name>Jason Dai</name>
+  
+            <email>jason.dai@intel.com</email>
+  
+            <organization>Intel</organization>
+  
+            <organizationUrl>http://www.intel.com</organizationUrl>
+  
+        </developer>
+  
+    </developers>
+  
+
+  
+    <modules>
+  
+        <module>zoo</module>
+  
+    </modules>
+  
+    <scm>
+  
+        <connection>scm:git:git@github.com:intel-analytics/zoo.git</connection>
+  
+        <url>https://github.com/intel-analytics/zoo/tree/master</url>
+  
+    </scm>
+  
+
+  
+    <repositories>
+  
+        <repository>
+  
+            <id>central</id>
+  
+            <name>Maven Repository</name>
+  
+            <url>https://repo1.maven.org/maven2</url>
+  
+            <releases>
+  
+                <enabled>true</enabled>
+  
+            </releases>
+  
+            <snapshots>
+  
+                <enabled>false</enabled>
+  
+            </snapshots>
+  
+        </repository>
+  
+        <repository>
+  
+            <id>apache-repo</id>
+  
+            <name>Apache Repository</name>
+  
+            <url>https://repository.apache.org/content/repositories/releases</url>
+  
+            <releases>
+  
+                <enabled>true</enabled>
+  
+            </releases>
+  
+            <snapshots>
+  
+                <enabled>false</enabled>
+  
+            </snapshots>
+  
+        </repository>
+  
+        <repository>
+  
+            <id>jboss-repo</id>
+  
+            <name>JBoss Repository</name>
+  
+            <url>https://repository.jboss.org/nexus/content/repositories/releases</url>
+  
+            <releases>
+  
+                <enabled>true</enabled>
+  
+            </releases>
+  
+            <snapshots>
+  
+                <enabled>false</enabled>
+  
+            </snapshots>
+  
+        </repository>
+  
+        <repository>
+  
+            <id>sonatype</id>
+  
+            <name>sonatype repository</name>
+  
+            <url>https://oss.sonatype.org/content/groups/public/</url>
+  
+            <releases>
+  
+                <enabled>true</enabled>
+  
+            </releases>
+  
+            <snapshots>
+  
+                <enabled>true</enabled>
+  
+            </snapshots>
+  
+        </repository>
+  
+    </repositories>
+  
+
+  
+    <licenses>
+  
+        <license>
+  
+            <name>Apache License, Version 2.0</name>
+  
+            <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+  
+        </license>
+  
+    </licenses>
+  
+
+  
+    <properties>
+  
+        <scala.major.version>2.11</scala.major.version>
+  
+        <scala.version>2.11.8</scala.version>
+  
+        <scala.macros.version>2.1.0</scala.macros.version>
+  
+        <spark.version>2.1.0</spark.version>
+  
+        <bigdl.artifactId>bigdl-SPARK_2.1</bigdl.artifactId>
+  
+    </properties>
+  
+
+  
+
+  
+    <build>
+  
+        <plugins>
+  
+            <plugin>
+  
+                <groupId>org.apache.maven.plugins</groupId>
+  
+                <artifactId>maven-assembly-plugin</artifactId>
+  
+                <executions>
+  
+                    <execution>
+  
+                        <id>python</id>
+  
+                        <inherited>false</inherited>
+  
+                        <configuration>
+  
+                            <descriptors>
+  
+                                <descriptor>assembly/python-zip.xml</descriptor>
+  
+                            </descriptors>
+  
+                        </configuration>
+  
+                        <phase>package</phase>
+  
+                        <goals>
+  
+                            <goal>single</goal>
+  
+                        </goals>
+  
+                    </execution>
+  
+                </executions>
+  
+            </plugin>
+  
+        </plugins>
+  
+    </build>
+  
+
+  
+</project>
diff --git a/pyzoo/dev/prepare_env.sh b/pyzoo/dev/prepare_env.sh
@@ -0,0 +1,52 @@
+  
+#!/usr/bin/env bash
+  
+
+  
+#
+  
+# Licensed under the Apache License, Version 2.0 (the "License");
+  
+# you may not use this file except in compliance with the License.
+  
+# You may obtain a copy of the License at
+  
+#
+  
+#     http://www.apache.org/licenses/LICENSE-2.0
+  
+#
+  
+# Unless required by applicable law or agreed to in writing, software
+  
+# distributed under the License is distributed on an "AS IS" BASIS,
+  
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  
+# See the License for the specific language governing permissions and
+  
+# limitations under the License.
+  
+#
+  
+
+  
+SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
+  
+echo "SCRIPT_DIR": $SCRIPT_DIR
+  
+export DL_PYTHON_HOME="$(cd ${SCRIPT_DIR}/../../; pwd)"
+  
+
+  
+export BIGDL_HOME="$(cd ${SCRIPT_DIR}/../../..; pwd)"
+  
+
+  
+echo "BIGDL_HOME: $BIGDL_HOME"
+  
+echo "SPARK_HOME": $SPARK_HOME
+  
+echo "DL_PYTHON_HOME": $DL_PYTHON_HOME
+  
+
+  
+if [ -z ${SPARK_HOME+x} ]; then echo "SPARK_HOME is unset"; exit 1; else echo "SPARK_HOME is set to '$SPARK_HOME'"; fi
+  
+
+  
+export PYSPARK_ZIP=`find $SPARK_HOME/python/lib  -type f -iname '*.zip' | tr "\n" ":"`
+  
+
+  
+export PYTHONPATH=$PYTHONPATH:$PYSPARK_ZIP:$DL_PYTHON_HOME:$DL_PYTHON_HOME/:$DL_PYTHON_HOME/test/dev:$BIGDL_HOME/spark/dl/src/main/resources/spark-bigdl.conf
+  
+
+  
+export BIGDL_CLASSPATH=$(find $BIGDL_HOME/spark/dl/target/ -name "*with-dependencies.jar" | head -n 1)
+  
+echo "BIGDL_CLASSPATH": $BIGDL_CLASSPATH
+  
+
+  
+if [[ ($SPARK_HOME == *"2.2.0"*) || ($SPARK_HOME == *"2.1.1"*) || ($SPARK_HOME == *"1.6.4"*) ]]; then
+  
+    export PYTHON_EXECUTABLES=("python2.7" "python3.5" "python3.6")
+  
+else
+  
+    export PYTHON_EXECUTABLES=("python2.7" "python3.5")
+  
+fi
+  
+
+  
+function run_notebook() {
+  
+    notebook_path=$1
+  
+    target_notebook_path=${DL_PYTHON_HOME}/tmp_${PYTHON_EXECUTABLE}.ipynb
+  
+    echo "Change kernel to $PYTHON_EXECUTABLE"
+  
+    sed "s/\"python.\"/\"$PYTHON_EXECUTABLE\"/g" $notebook_path > ${target_notebook_path}
+  
+    jupyter nbconvert --to notebook --execute \
+  
+      --ExecutePreprocessor.timeout=360 --output ${DL_PYTHON_HOME}/tmp_out.ipynb \
+  
+      $target_notebook_path
+  
+}
+  
+
+  
+export -f run_notebook
diff --git a/pyzoo/docs/.keep b/pyzoo/docs/.keep
diff --git a/pyzoo/setup.py b/pyzoo/setup.py
@@ -0,0 +1,18 @@
+  
+#!/usr/bin/env python
+  
+
+  
+#
+  
+# Copyright 2016 The BigDL Authors.
+  
+#
+  
+# Licensed under the Apache License, Version 2.0 (the "License");
+  
+# you may not use this file except in compliance with the License.
+  
+# You may obtain a copy of the License at
+  
+#
+  
+#     http://www.apache.org/licenses/LICENSE-2.0
+  
+#
+  
+# Unless required by applicable law or agreed to in writing, software
+  
+# distributed under the License is distributed on an "AS IS" BASIS,
+  
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  
+# See the License for the specific language governing permissions and
+  
+# limitations under the License.
+  
+#
+  
+
diff --git a/pyzoo/test/api/keras/.keep b/pyzoo/test/api/keras/.keep
diff --git a/pyzoo/test/pipeline/.keep b/pyzoo/test/pipeline/.keep