import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.select($"content")
			.discardContentDF(Set("Content-Length: [0-9]{4}".r))
			.write
  			.option("header","true")
  			.csv("content_after_discard")

import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.select($"content")
			.keepContentDF(Set("Content-Length: [0-9]{4}".r))
			.write
  			.option("header","true")
  			.csv("content_kept")

import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.select($"url")
			.discardUrlPatternsDF(Set(".*images.*".r))
			.show(10,false)

import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.select($"url")
			.keepUrlPatternsDF(Set(".*images.*".r))
			.show(10,false)

import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.select(DetectLanguageDF(RemoveHTMLDF($"content")).as("language"))
			.discardLanguagesDF(Set("th","de","ht"))
			.show(10,false)

import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.select(DetectLanguageDF(RemoveHTMLDF($"content")).as("language"))
			.keepLanguagesDF(Set("th","de","ht"))
			.show(10,false)

import io.archivesunleashed._
import io.archivesunleashed.df._

RecordLoader.loadArchives("./src/test/resources/arc/example.arc.gz",sc)
			.all()
			.keepImagesDF()
			.select($"mime_type_tika")
			.show(10,false)