<commit_before>package org.diverse.pcm.io.wikipedia

import java.io.StringReader

import org.ccil.cowan.tagsoup.jaxp.SAXFactoryImpl
import org.diverse.pcm.io.wikipedia.parser.{PageVisitor, PreprocessVisitor}
import org.diverse.pcm.io.wikipedia.pcm.{Cell, Matrix, Page}
import org.sweble.wikitext.`lazy`.{LazyParser, LazyPreprocessor}
import org.sweble.wikitext.engine.utils.SimpleWikiConfiguration
import org.xml.sax.InputSource

import scala.xml.Node
import scala.xml.parsing.NoBindingFactoryAdapter
import scalaj.http.{Http, HttpOptions}

/**
 * Created by gbecan on 13/10/14.
 */
class WikipediaPageMiner {

  private val config = new SimpleWikiConfiguration()
  private val parser = new LazyParser(config)
  private val preprocessor = new LazyPreprocessor(config)

  /**
   * Retrieve Wikitext code of an article on Wikipedia servers
   * @param title : title of the article on English version of Wikipedia
   */
  def getPageCodeFromWikipedia(title : String): String = {
    val editPage = Http("http://en.wikipedia.org/w/index.php")
      .params("title" -> title.replaceAll(" ", "_"), "action" -> "edit")
      .option(HttpOptions.connTimeout(1000))
      .option(HttpOptions.readTimeout(30000))
      .asString
    val xml = parseHTMLAsXML(editPage)
    val code = (xml \\ "textarea").text
    code
  }

  /**
   * Clean HTML to get strict XML
   */
  private def parseHTMLAsXML(htmlCode : String) : Node = {
    val adapter = new NoBindingFactoryAdapter
    val htmlParser = (new SAXFactoryImpl).newSAXParser()
    val xml = adapter.loadXML(new InputSource(new StringReader(htmlCode)), htmlParser)
    xml
  }

  /**
   * Preprocess Wikitext
   * Remove templates with the help of https://en.wikipedia.org/wiki/Special:ExpandTemplates
   */<commit_msg>Bugfix while checking for a redirected page (ex: Comparison_of_Tryton_and_Open_ERP => Comparison_of_Tryton_and_Odoo)<commit_after>package org.diverse.pcm.io.wikipedia

import java.io.StringReader

import org.ccil.cowan.tagsoup.jaxp.SAXFactoryImpl
import org.diverse.pcm.io.wikipedia.parser.{PageVisitor, PreprocessVisitor}
import org.diverse.pcm.io.wikipedia.pcm.{Cell, Matrix, Page}
import org.sweble.wikitext.`lazy`.{LazyParser, LazyPreprocessor}
import org.sweble.wikitext.engine.utils.SimpleWikiConfiguration
import org.xml.sax.InputSource

import scala.xml.Node
import scala.xml.parsing.NoBindingFactoryAdapter
import scalaj.http.{Http, HttpOptions}

/**
 * Created by gbecan on 13/10/14.
 */
class WikipediaPageMiner {

  private val config = new SimpleWikiConfiguration()
  private val parser = new LazyParser(config)
  private val preprocessor = new LazyPreprocessor(config)

  /**
   * Retrieve Wikitext code of an article on Wikipedia servers
   * @param title : title of the article on English version of Wikipedia
   */
  def getPageCodeFromWikipedia(title : String): String = {
    val editPage = Http("http://en.wikipedia.org/w/index.php")
      .params("title" -> title.replaceAll(" ", "_"), "action" -> "edit")
      .option(HttpOptions.connTimeout(1000))
      .option(HttpOptions.readTimeout(30000))
      .asString
    val xml = parseHTMLAsXML(editPage)
    var code = (xml \\ "textarea").text
    // Manage the page redirection if any
    // TODO: arrange regex to get off the replace calls
    if (code.contains("#REDIRECT")) {
      val title = """\[\[(.*?)\]\]""".r findFirstIn code
      code = getPageCodeFromWikipedia(title.get.replace("[", "").replace("]", ""))
    }
    code
  }

  /**
   * Clean HTML to get strict XML
   */
  private def parseHTMLAsXML(htmlCode : String) : Node = {
    val adapter = new NoBindingFactoryAdapter
    val htmlParser = (new SAXFactoryImpl).newSAXParser()
    val xml = adapter.loadXML(new InputSource(new StringReader(htmlCode)), htmlParser)
    xml
  }

  /**
   * Preprocess Wikitext
   * Remove templates with the help of https://en.wikipedia.org/wiki/Special:ExpandTemplates
   */
{'fraction_non_alphanumeric': 0.11748144356283594, 'fraction_numerical': 0.007678525723061172, 'mean_word_length': 3.9281210592686002, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5147767', 'n_tokens_mistral': 1250, 'n_tokens_neox': 1209, 'n_words': 286}