Add method for determining binary file extension. (#349)

This PR implements the strategy described in the discussion of the above issue to get an extension for a file described by a URL and a MIME type. It creates a GetExtensionMime object in the matchbox. This PR also removes most of the filtering by URL from the image, audio, video, presentation, spreadsheet, and word processor document extraction methods, since these were returning false positives. (CSV and TSV files are a special case, since Tika detects them as "text/plain" based on content.) Finally, I have inserted toLowerCase into the getUrl.endsWith() filter tests, which could possibly bring in some more CSV and TSV files * Adds method for getting a file extension from a MIME type. * Add getExtensions method to DetectMimeTypeTika. * Matchbox object to get extension of URL * Use GetExtensionMime for extraction methods; minor fixes. * Remove tika-parsers classifier * Remove most filtering by file extension from binary extraction methods; add CSV/TSV special cases. * Fix GetExtensionMime case where URL has no extension but a MIME type is detected * Insert `toLowerCase` into `getUrl.endsWith()` calls in io.archivesunleashed.packages; apply to `FilenameUtils.getExtension` in `GetExtensionMime`. * Remove filtering on URL for audio, video, and images. * Remove filtering on URL for images; add DF fields to image extraction * Remove saveImageToDisk and its test * Remove robots.txt check and extraneous imports * Close files so we don't get too many files open again. * Add GetExtensionMimeTest * Resolve #343
archivesunleashed · Aug 18, 2019 · 448601eaf9fb0f072f432d4dff30885aa939152d · 448601e
1 parent 018527a
commit 448601eaf9fb0f072f432d4dff30885aa939152d
diff --git a/src/main/scala/io/archivesunleashed/df/package.scala b/src/main/scala/io/archivesunleashed/df/package.scala
@@ -22,9 +22,7 @@ import io.archivesunleashed.matchbox.{ComputeMD5, ExtractDomain, RemoveHTML}
 import org.apache.spark.sql.functions.udf
 import org.apache.spark.sql.DataFrame
 import java.io.ByteArrayInputStream
-  
-import java.io.File
 import java.io.FileOutputStream
-  
-import javax.imageio.{ImageIO, ImageReader}
 import java.util.Base64
 /**
@@ -47,40 +45,6 @@ package object df {
   */
  implicit class SaveBytes(df: DataFrame) {
-  
-    /**
-  
-     * @param bytesColumnName the name of the column containing the image bytes
-  
-     * @param fileName the name of the file to save the images to (without extension)
-  
-     * e.g. fileName = "foo" => images are saved as "foo-[MD5 hash].jpg"
-  
-     */
-  
-    def saveImageToDisk(bytesColumnName: String, fileName: String): Unit = {
-  
-      df.select(bytesColumnName).foreach(row => {
-  
-        try {
-  
-          // Assumes the bytes are base64 encoded already as returned by ExtractImageDetails.
-  
-          val encodedBytes: String = row.getAs(bytesColumnName);
-  
-          val bytes = Base64.getDecoder.decode(encodedBytes);
-  
-          val in = new ByteArrayInputStream(bytes);
-  
-
-  
-          val input = ImageIO.createImageInputStream(in);
-  
-          val readers = ImageIO.getImageReaders(input);
-  
-          if (readers.hasNext()) {
-  
-            val reader = readers.next()
-  
-            reader.setInput(input)
-  
-            val image = reader.read(0)
-  
-
-  
-            val format = reader.getFormatName()
-  
-            val suffix = ComputeMD5(bytes)
-  
-            val file = new File(fileName + "-" + suffix + "." + format);
-  
-            if (image != null) {
-  
-              ImageIO.write(image, format, file);
-  
-            }
-  
-          }
-  
-        } catch {
-  
-          case e: Throwable => {
-  
-          }
-  
-        }
-  
-      })
-  
-    }
-  
-
    /**
      * @param bytesColumnName the name of the column containing the bytes
      * @param fileName the name of the file to save the binary file to (without extension)
@@ -99,6 +63,7 @@ package object df {
          val suffix = ComputeMD5(bytes)
          val file = new FileOutputStream(fileName + "-" + suffix + "." + extension.toLowerCase)
          IOUtils.copy(in, file)
+  
+          file.close()
        } catch {
          case e: Throwable => {
          }

diff --git a/src/main/scala/io/archivesunleashed/matchbox/DetectMimeTypeTika.scala b/src/main/scala/io/archivesunleashed/matchbox/DetectMimeTypeTika.scala
@@ -16,9 +16,11 @@
 */
 package io.archivesunleashed.matchbox
+  
+import scala.collection.JavaConverters._
 import org.apache.tika.Tika
 import org.apache.tika.detect.DefaultDetector
 import org.apache.tika.io.TikaInputStream
+  
+import org.apache.tika.mime.MimeTypes
 import org.apache.tika.parser.AutoDetectParser
 /** Detect MIME type using Apache Tika. */
@@ -27,6 +29,8 @@ object DetectMimeTypeTika {
  val parser = new AutoDetectParser(detector)
  val tika = new Tika(detector, parser)
+  
+  val allMimeTypes = MimeTypes.getDefaultMimeTypes();
+  
+
  /** Detect MIME type from an input string.
   *
   * @param content a byte array of content for which to detect the MimeType
@@ -42,4 +46,25 @@ object DetectMimeTypeTika {
      mimetype
    }
  }
+  
+
+  
+  /** Return the best guess at a file extension from a MIME type string
+  
+   *
+  
+   * @param mimeType string representation of the MimeType
+  
+   * @return file extension (e.g. ".jpg" for "image/jpeg").
+  
+   */
+  
+  def getExtension(mimeType: String): String = {
+  
+    val regMimeType = allMimeTypes.forName(mimeType)
+  
+    regMimeType.getExtension
+  
+  }
+  
+
+  
+  /** Return the list of all known file extensions for a MIME type string
+  
+   *
+  
+   * @param mimeType string representation of the MimeType
+  
+   * @return list of file extensions (e.g. ".jpg" for "image/jpeg").
+  
+   */
+  
+  def getExtensions(mimeType: String): List[String] = {
+  
+    val regMimeType = allMimeTypes.forName(mimeType)
+  
+    regMimeType.getExtensions.asScala.toList
+  
+  }
+  
+
 }
diff --git a/src/main/scala/io/archivesunleashed/matchbox/ExtractImageDetails.scala b/src/main/scala/io/archivesunleashed/matchbox/ExtractImageDetails.scala
@@ -18,10 +18,8 @@ package io.archivesunleashed.matchbox
 import java.security.MessageDigest
 import java.util.Base64
-  
-import java.nio.charset.StandardCharsets
 import org.apache.commons.codec.binary.Hex
-  
-
 /** Information about an image. e.g. width, height. */
 class ImageDetails(imageUrl: String, imageType: String, bytes: Array[Byte]) {
  val dimensions = ComputeImageSize(bytes);
@@ -33,7 +31,7 @@ class ImageDetails(imageUrl: String, imageType: String, bytes: Array[Byte]) {
  val body: String = Base64.getEncoder.encodeToString(bytes)
 }
-  
-/** Extracts image details given raw bytes (using Apache Tika). */
+  
+/** Extracts image details given raw bytes. */
 object ExtractImageDetails {
  /**

diff --git a/src/main/scala/io/archivesunleashed/matchbox/GetExtensionMime.scala b/src/main/scala/io/archivesunleashed/matchbox/GetExtensionMime.scala
@@ -0,0 +1,58 @@
+  
+/*
+  
+ * Archives Unleashed Toolkit (AUT):
+  
+ * An open-source toolkit for analyzing web archives.
+  
+ *
+  
+ * Licensed under the Apache License, Version 2.0 (the "License");
+  
+ * you may not use this file except in compliance with the License.
+  
+ * You may obtain a copy of the License at
+  
+ *
+  
+ *     http://www.apache.org/licenses/LICENSE-2.0
+  
+ *
+  
+ * Unless required by applicable law or agreed to in writing, software
+  
+ * distributed under the License is distributed on an "AS IS" BASIS,
+  
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  
+ * See the License for the specific language governing permissions and
+  
+ * limitations under the License.
+  
+ */
+  
+package io.archivesunleashed.matchbox
+  
+
+  
+import org.apache.commons.io.FilenameUtils
+  
+
+  
+/** Get file extension using MIME type, then URL extension. */
+  
+// scalastyle:off object.name
+  
+object GetExtensionMime {
+  
+// scalastyle:on object.name
+  
+
+  
+  /** Returns the extension of a file specified by URL
+  
+    *
+  
+    * @param url string
+  
+    * @param mimeType string
+  
+    * @return string
+  
+    */
+  
+  def apply(url: String, mimeType: String): String = {
+  
+    val tikaExtensions = DetectMimeTypeTika.getExtensions(mimeType)
+  
+    var ext = "unknown"
+  
+    // Determine extension from MIME type
+  
+    if (tikaExtensions.size == 1) {
+  
+      ext = tikaExtensions(0).substring(1)
+  
+    } else {
+  
+      // Get extension from URL
+  
+      val urlExt = FilenameUtils.getExtension(url).toLowerCase
+  
+      if (urlExt != null) {
+  
+        // Reconcile MIME-based and URL extension, preferring MIME-based
+  
+        if (tikaExtensions.size > 1) {
+  
+          if (tikaExtensions.contains("." + urlExt)) {
+  
+            ext = urlExt
+  
+          } else {
+  
+            ext = tikaExtensions(0).substring(1)
+  
+          }
+  
+        } else { // tikaExtensions.size == 0
+  
+          if (!urlExt.isEmpty) {
+  
+            ext = urlExt
+  
+          } // urlExt = "" => ext = "unknown"
+  
+        }
+  
+      }
+  
+    }
+  
+    ext
+  
+  }
+  
+}