case class GoogleRaw(content: String, crawlId: String)
case class NaverRaw(content: String)
case class GoogleParsed(sections: Seq[String], crawlId: String)
case class NaverParsed(sections: Seq[String])
trait Cralwer {
type Raw
type Parsed
def fetch(url: String): Raw
def parse(raw: Raw): Parsed
def save(raw: Raw, parsed: Parsed): Int
}
object GoogleCrawler extends Cralwer {
type Raw = GoogleRaw
type Parsed = GoogleParsed
override def fetch(url: String): GoogleRaw = GoogleRaw("g", "id123")
override def parse(raw: GoogleRaw): GoogleParsed = GoogleParsed(Seq(raw.content), raw.crawlId)
override def save(raw: GoogleRaw, parsed: GoogleParsed): Int = 1
}
object NaverCrawler extends Cralwer {
type Raw = NaverRaw
type Parsed = NaverParsed
override def fetch(url: String): NaverRaw = NaverRaw("n")
override def parse(raw: NaverRaw): NaverParsed = NaverParsed(Seq(raw.content))
override def save(raw: NaverRaw, parsed: NaverParsed): Int = 2
}
@main def main(): Unit = {
// val cralwer: Cralwer = GoogleCrawler
val cralwer: Cralwer = NaverCrawler
val content: cralwer.Raw = cralwer.fetch("http://www.some.com")
val parsed: cralwer.Parsed = cralwer.parse(content)
parsed match {
case GoogleParsed(_, id) => println(id)
case NaverParsed(content) => println(content)
}
val savedCnt = cralwer.save(content, parsed)
println(savedCnt)
}