import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.discardHttpStatusDF(Set("000"))
			.select($"url",$"crawl_date",$"HttpStatus")
			.show(10,false)

import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.keepMimeTypesDF(Set("text/html"))
			.select($"mime_type_web_server")
			.show(10,false)

import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.keepMimeTypesTikaDF(Set("image/jpeg"))
			.select($"mime_type_tika")
			.show(10,false)