/Anserini-Spark

Permalink
Browse files

Initial implementation of HdfsReadOnlyDirectory, basic pom from bespin.

  • Loading branch information...
@lintool
lintool committed Dec 5, 2018
1 parent 069f28c commit 6cd59c3bd047fd5b99c9ff8cb639d65da84f619c
Unified Split
Showing with 471 additions and 0 deletions.
  1. +14 −0 .gitignore
  2. +275 −0 pom.xml
  3. +182 −0 src/main/java/io/anserini/hadoop/HdfsReadOnlyDirectory.java
View file
14 .gitignore
@@ -0,0 +1,14 @@
.DS_Store
.classpath
.project
.idea
target/
bin/
*~
*.iml
*.pyc
*.out
lucene-index*
run.*
log*
out.*
View file
275 pom.xml
@@ -0,0 +1,275 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

<modelVersion>4.0.0</modelVersion>
<groupId>io.anserini</groupId>
<artifactId>anserini-spark</artifactId>
<packaging>jar</packaging>
<name>Anserini-Spark</name>
<version>0.0.1-SNAPSHOT</version>
<description>Anserini/Spark integration</description>
<url>http://anserini.io/</url>
<licenses>
<license>
<name>The Apache Software License, Version 2.0</name>
<url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
<distribution>repo</distribution>
</license>
</licenses>
<scm>
<connection>scm:git:git@github.com:lintool/bespin.git</connection>
<developerConnection>scm:git:git@github.com:lintool/bespin.git</developerConnection>
<url>git@github.com:lintool/bespin.git</url>
</scm>
<developers>
<developer>
<id>lintool</id>
<name>Jimmy Lin</name>
<email>jimmylin@uwaterloo.ca</email>
</developer>
</developers>

<distributionManagement>
<snapshotRepository>
<id>ossrh</id>
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
</snapshotRepository>
</distributionManagement>

<repositories>
<repository>
<id>maven</id>
<url>http://repo.maven.apache.org/maven2/</url>
</repository>
</repositories>

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<scala.version>2.11.8</scala.version>
<hbase.version>1.2.6</hbase.version>
<hadoop.version>3.0.3</hadoop.version>
<spark.version>2.3.1</spark.version>
</properties>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<version>2.15.2</version>
<executions>
<execution>
<phase>process-resources</phase>
<goals>
<goal>add-source</goal>
<goal>compile</goal>
</goals>
</execution>
<execution>
<id>scala-test-compile</id>
<phase>process-test-resources</phase>
<goals>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.1.0</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<finalName>${project.artifactId}-${project.version}-fatjar</finalName>
<createDependencyReducedPom>false</createDependencyReducedPom>
<!-- This is an Mac OSX issue because the default filesystem is case-insensitive,
so multiple versions of files (w/ different cases) clash when Hadoop tries
to unpack the jar. -->
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>META-INF/LICENSE*</exclude>
<exclude>license/*</exclude>
</excludes>
</filter>
</filters>
</configuration>
</execution>
</executions>
</plugin>

<plugin>
<groupId>org.sonatype.plugins</groupId>
<artifactId>nexus-staging-maven-plugin</artifactId>
<version>1.6.8</version>
<extensions>true</extensions>
<configuration>
<serverId>ossrh</serverId>
<nexusUrl>https://oss.sonatype.org/</nexusUrl>
<autoReleaseAfterClose>true</autoReleaseAfterClose>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>3.0.1</version>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.0.0</version>
<executions>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-gpg-plugin</artifactId>
<version>1.6</version>
<executions>
<execution>
<id>sign-artifacts</id>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
</execution>
</executions>
</plugin>

</plugins>
</build>

<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>net.minidev</groupId>
<artifactId>json-smart</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase.version}</version>
</dependency>
<dependency> <!-- Needed for using MapReduce w/ HBase -->
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>${hbase.version}</version>
</dependency>
<dependency>
<groupId>args4j</groupId>
<artifactId>args4j</artifactId>
<version>2.33</version>
</dependency>
<dependency>
<groupId>tl.lin</groupId>
<artifactId>lintools-datatypes</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>net.sf.jung</groupId>
<artifactId>jung-api</artifactId>
<version>2.1.1</version>
</dependency>
<dependency>
<groupId>net.sf.jung</groupId>
<artifactId>jung-algorithms</artifactId>
<version>2.1.1</version>
</dependency>
<dependency>
<groupId>net.sf.jung</groupId>
<artifactId>jung-graph-impl</artifactId>
<version>2.1.1</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>23.6-jre</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_2.11</artifactId>
<version>3.0.4</version>
</dependency>
<dependency>
<groupId>org.rogach</groupId>
<artifactId>scallop_2.11</artifactId>
<version>3.1.1</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.11</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.11</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>io.anserini</groupId>
<artifactId>anserini</artifactId>
<version>0.2.0</version>
</dependency>
</dependencies>

</project>
Oops, something went wrong.

0 comments on commit 6cd59c3

Please sign in to comment.