import org.apache.spark.sql.functions.lit
import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.select($"url",$"http_status_code")
			.filter(hasHttpStatusDF($"http_status_code", lit(Array("200","000"))))
			.show(10,false)

import org.apache.spark.sql.functions.lit
import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.select($"content")
			.filter(hasContentDF($"content", lit(Array("Content-Length: [0-9]{4}"))))
			.show(3,false)

import org.apache.spark.sql.functions.lit
import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.select($"url")
			.filter(hasUrlPatternsDF($"url", lit(Array(".*images.*"))))
			.show(10,false)

import org.apache.spark.sql.functions.lit
import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.select(DetectLanguageDF(RemoveHTMLDF($"content")).as("language"))
			.filter(hasLanguagesDF(DetectLanguageDF(RemoveHTMLDF($"content")), lit(Array("de","ht"))))
			.show(10,false)

import org.apache.spark.sql.functions.lit
import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.select($"mime_type_web_server")
			.filter(hasMimeTypesDF($"mime_type_web_server", lit(Array("text/html"))))
			.show(10,false)

import org.apache.spark.sql.functions.lit
import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.select($"url")
			.filter(hasUrlsDF($"url", lit(Array("http://www.archive.org/","http://www.archive.org/robots.txt"))))
			.show(10,false)

import org.apache.spark.sql.functions.lit
import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.select($"url")
			.filter(hasDomainsDF(ExtractDomainDF($"url"), lit(Array("www.archive.org"))))
			.show(10,false)