<commit_before>package ru.wordmetrix.webcrawler

import scala.actors.Actor

class Storage()(implicit val cfg : CFG) extends Actor {
    def act() = loop {
        react {
            case seed : WebCrawler.Seed => {
                //TODO: mark page as significant
                
            }
            
            case (seed : WebCrawler.Seed, intell : WebCrawler.Intell) => {
                //TODO : save page in storage
            }<commit_msg>Storage was implemented by a simple manner: it saves data into plain files and moves the significant files to another directory.<commit_after>package ru.wordmetrix.webcrawler

import scala.actors.Actor
import ActorDebug._
import java.io.File
import java.io.FileOutputStream
import java.io.OutputStreamWriter
import java.io.FileInputStream

class Storage()(implicit val cfg: CFG) extends Actor {

    def seedToFilename(seed: WebCrawler.Seed) = """[/:\\]""".r.replaceAllIn("""https?://""".r.replaceFirstIn(seed.toString, ""), "-") match {
        case x if x.length > 120 => x.slice(0, 120) +
            x.slice(0, 120).hashCode.toString
        case x => x
    }

    def act() = loop {
        react {
            case seed: WebCrawler.Seed => {
                log("Store marks datum for %s as significant", seed)
                val name = seedToFilename(seed)
                val fin = new FileInputStream(new File(new File(cfg.path,"tmp"),name))
                val buf = new Array[Byte](fin.available())
                fin.read(buf)
                fin.close
                val fout = new FileOutputStream(new File(cfg.path,name))
                fout.write(buf)
                fout.close()
            }

            case (seed: WebCrawler.Seed, intell: WebCrawler.Intell) => {
                log("Store data for %s", seed)

                val file = new OutputStreamWriter(
                    new FileOutputStream(
                        new File(new File(cfg.path,"tmp"), seedToFilename(seed))))

                file.write(intell)
                file.close()
            }

            case x => log("Store got something strange: %s", x)
{'fraction_non_alphanumeric': 0.10990052108005685, 'fraction_numerical': 0.005210800568450971, 'mean_word_length': 1.8387096774193548, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1073066', 'n_tokens_mistral': 615, 'n_tokens_neox': 575, 'n_words': 158}