Add language detection column to webpages. (#403)

- Addresses #402
archivesunleashed · Jan 12, 2020 · bc0d663fb42125af2ec9326c71a08e53f8a86457 · bc0d663
1 parent 0ecc4f8
commit bc0d663fb42125af2ec9326c71a08e53f8a86457
Unified Split

Showing with 6 additions and 3 deletions.

+6 −3 src/main/scala/io/archivesunleashed/package.scala
diff --git a/src/main/scala/io/archivesunleashed/package.scala b/src/main/scala/io/archivesunleashed/package.scala
@@ -27,7 +27,7 @@ import io.archivesunleashed.df.{DetectLanguageDF, DetectMimeTypeTikaDF, ExtractD
 
 import io.archivesunleashed.matchbox.{DetectLanguageRDD, DetectMimeTypeTika, ExtractDateRDD,
                                       ExtractDomainRDD, ExtractImageDetails, ExtractImageLinksRDD,
-                                      ExtractLinksRDD, GetExtensionMimeRDD, RemoveHTMLRDD}
+                                      ExtractLinksRDD, GetExtensionMimeRDD, RemoveHTMLRDD, RemoveHTTPHeaderRDD}
 import io.archivesunleashed.matchbox.ExtractDateRDD.DateComponent
 import io.archivesunleashed.matchbox.ExtractDateRDD.DateComponent.DateComponent
 import java.net.URI
@@ -340,13 +340,16 @@ package object archivesunleashed {
     def webpages(): DataFrame = {
       val records = rdd.keepValidPages()
         .map(r => Row(r.getCrawlDate, r.getUrl, r.getMimeType,
-          DetectMimeTypeTika(r.getBinaryBytes), r.getContentString))
+          DetectMimeTypeTika(r.getBinaryBytes),
+          DetectLanguageRDD(RemoveHTMLRDD(RemoveHTTPHeaderRDD(r.getContentString))),
+          r.getContentString))
 
       val schema = new StructType()
         .add(StructField("crawl_date", StringType, true))
         .add(StructField("url", StringType, true))
         .add(StructField("mime_type_web_server", StringType, true))
         .add(StructField("mime_type_tika", StringType, true))
+        .add(StructField("language", StringType, true))
         .add(StructField("content", StringType, true))
 
       val sqlContext = SparkSession.builder()
@@ -899,4 +902,4 @@ package object archivesunleashed {
       rdd.filter(r => !lang.contains(DetectLanguageRDD(RemoveHTMLRDD(r.getContentString))))
     }
   }
-}
+}