<commit_before>package ru.wordmetrix.webcrawler

import java.net.URI
import scala.Option.option2Iterable
import scala.actors.Actor
import org.ccil.cowan.tagsoup.jaxp.SAXFactoryImpl
import org.xml.sax.InputSource
import scala.xml.parsing.NoBindingFactoryAdapter
import java.io.CharArrayReader
import ActorDebug._
import scala.xml.Node
import ru.wordmetrix.webcrawler.LinkContext.Feature
/*
 * Gather analyzes a page and elicits links and useful load.
 */
class Gather(storage: Storage, queue: Actor, sample : SampleHierarchy2Priority)(implicit val cfg: CFG)
        extends Actor with CFGAware {
    override val name = "Gather"

    val map = scala.collection.mutable.Set[(String, String)]()

    def page2xml_whole(page: WebCrawler.Page) =
        (new NoBindingFactoryAdapter).loadXML(
            new InputSource(new CharArrayReader(page.toArray)),
            new SAXFactoryImpl().newSAXParser())

    def page2xml(page: WebCrawler.Page): scala.xml.NodeSeq =
        ((page2xml_whole(page) \\ "div").
            filter(
                x => x.attribute("id").getOrElse("").toString ==
                    "mw-content-text"))

    def xml2seeds(xml: scala.xml.NodeSeq, base: URI) = (xml \\ "a").
        map(x => x.attribute("href")).flatten.
        map(x => base.resolve(x.toString)).
        filter(x => x.getHost() == "en.wikipedia.org").
        filter(x => {
            val id = (x.getHost(), x.getPath())
            if (map contains id) {
                false
            } else {
                map.+=(id)
                true
            }
        }).toSet

    def xml2vector(xml: scala.xml.NodeSeq) = Vector(
        "\\s+".r.split(xml.text).groupBy(x => x).map({
            case (x, y) => (x -> y.length.toDouble)
        }).toList)

    def xml2intell(xml: scala.xml.NodeSeq) = xml.text

    def act() = loop {
        react {
            case (seed: URI, page: WebCrawler.Page) => {
                this.debug("Gather data from %s", seed)
                try {
                    val xml = page2xml(page)
                    //this.debug("%s",xml.map(x => new LinkContext().extract( x )).reduce(_ ++ _))//(x : Map[WebCrawler.Seed,Vector[Feature]],y :Map[WebCrawler.Seed,Vector[Feature]]) => x ++ x))
                    
                    //this.debug("%s",new LinkContext().extract(page2xml_whole(page)))
                    sample ! new LinkContext().extract(page2xml_whole(page))
                    storage ! ((seed, xml2intell(xml)))
                    queue ! ((xml2seeds(xml, seed), seed, xml2vector(xml)))
                } catch {
                    case x => log("Gathering failed on %s: %s", seed, x)
                }
            }
        }
    }
}<commit_msg>Wikipedia-specific filter is temporarily disabled.<commit_after>package ru.wordmetrix.webcrawler

import java.net.URI
import scala.Option.option2Iterable
import scala.actors.Actor
import org.ccil.cowan.tagsoup.jaxp.SAXFactoryImpl
import org.xml.sax.InputSource
import scala.xml.parsing.NoBindingFactoryAdapter
import java.io.CharArrayReader
import ActorDebug._
import scala.xml.Node
import ru.wordmetrix.webcrawler.LinkContext.Feature
/*
 * Gather analyzes a page and elicits links and useful load.
 */
class Gather(storage: Storage, queue: Actor, sample : SampleHierarchy2Priority)(implicit val cfg: CFG)
        extends Actor with CFGAware {
    override val name = "Gather"

    val map = scala.collection.mutable.Set[(String, String)]()

    def page2xml_whole(page: WebCrawler.Page) =
        (new NoBindingFactoryAdapter).loadXML(
            new InputSource(new CharArrayReader(page.toArray)),
            new SAXFactoryImpl().newSAXParser())

    def page2xml(page: WebCrawler.Page): scala.xml.NodeSeq =
        (page2xml_whole(page) \\ "body")
//        ((page2xml_whole(page) \\ "div").
//            filter(
//                x => x.attribute("id").getOrElse("").toString ==
//                    "mw-content-text"))

    def xml2seeds(xml: scala.xml.NodeSeq, base: URI) = (xml \\ "a").
        map(x => x.attribute("href")).flatten.
        map(x => base.resolve(x.toString)).
        filter(x => x.getHost() == "en.wikipedia.org").
        filter(x => {
            val id = (x.getHost(), x.getPath())
            if (map contains id) {
                false
            } else {
                map.+=(id)
                true
            }
        }).toSet

    def xml2vector(xml: scala.xml.NodeSeq) = Vector(
        "\\s+".r.split(xml.text).groupBy(x => x).map({
            case (x, y) => (x -> y.length.toDouble)
        }).toList)

    def xml2intell(xml: scala.xml.NodeSeq) = xml.text

    def act() = loop {
        react {
            case (seed: URI, page: WebCrawler.Page) => {
                this.debug("Gather data from %s", seed)
                try {
                    val xml = page2xml(page)
                    //this.debug("%s",xml.map(x => new LinkContext().extract( x )).reduce(_ ++ _))//(x : Map[WebCrawler.Seed,Vector[Feature]],y :Map[WebCrawler.Seed,Vector[Feature]]) => x ++ x))
                    
                    //this.debug("%s",new LinkContext().extract(page2xml_whole(page)))
                    sample ! new LinkContext(seed).extract(page2xml_whole(page))
                    storage ! ((seed, xml2intell(xml)))
                    queue ! ((xml2seeds(xml, seed), seed, xml2vector(xml)))
                } catch {
                    case x => log("Gathering failed on %s: %s", seed, x)
                }
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.14275309541150766, 'fraction_numerical': 0.00528040786598689, 'mean_word_length': 2.028114663726571, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 61}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4140855', 'n_tokens_mistral': 1749, 'n_tokens_neox': 1679, 'n_words': 344}