<commit_before>package org.scalawiki

import akka.actor.ActorSystem
import akka.event.LoggingAdapter
import akka.io.IO
import akka.pattern.ask
import org.scalawiki.dto.{LoginResponse, Page, Site}
import org.scalawiki.dto.cmd.Action
import org.scalawiki.http.{HttpClient, HttpClientBee, HttpClientSpray}
import org.scalawiki.json.MwReads._
import org.scalawiki.query.{DslQuery, PageQuery, SinglePageQuery}
import play.api.libs.json._
import spray.can.Http
import spray.http._
import spray.util._

import scala.concurrent.duration._
import scala.concurrent.{Await, Future}

trait MwBot {

  def host: String

  def login(user: String, password: String): Future[String]

  def run(action: Action): Future[Seq[Page]]

  def get(params: Map[String, String]): Future[String]

  def post(params: Map[String, String]): Future[String]

  def getByteArray(url: String): Future[Array[Byte]]

  def post[T](reads: Reads[T], params: (String, String)*): Future[T]

  def post[T](reads: Reads[T], params: Map[String, String]): Future[T]

  def postMultiPart[T](reads: Reads[T], params: Map[String, String]): Future[T]

  def postFile[T](reads: Reads[T], params: Map[String, String], fileParam: String, filename: String): Future[T]

  def page(title: String): SinglePageQuery

  def page(id: Long): SinglePageQuery

  def pageText(title: String): Future[String]

  def token: String

  def await[T](future: Future[T]): T

  def system: ActorSystem

  def log: LoggingAdapter
}

class MwBotImpl(val site: Site,
                val http: HttpClient = new HttpClientSpray(MwBot.system),
                val system: ActorSystem = MwBot.system
               ) extends MwBot {

  def this(host: String) = this(Site.host(host))

  def this(host: String, http: HttpClient) = this(Site.host(host), http)

  implicit val sys = system

  import system.dispatcher

  def host = site.domain

  val baseUrl: String = site.protocol + "://" + host + site.scriptPath

  val indexUrl = baseUrl + "/index.php"

  val apiUrl = baseUrl + "/api.php"

  def encodeTitle(title: String): String = MwUtils.normalize(title)

  override def log = system.log

  override def login(user: String, password: String): Future[String] = {
    require(user != null, "User is null")
    require(password != null, "Password is null")

    log.info(s"$host login user: $user")

    tryLogin(user, password).flatMap {
      loginResponse =>
        tryLogin(user, password, loginResponse.token).map(_.result)
    }
  }

  def tryLogin(user: String, password: String, token: Option[String] = None): Future[LoginResponse] = {
    val loginParams = Map(
      "action" -> "login", "lgname" -> user, "lgpassword" -> password, "format" -> "json"
    ) ++ token.map("lgtoken" -> _)
    http.post(apiUrl, loginParams).map { resp =>
      val body = http.getBody(resp)
      val jsResult = Json.parse(body).validate(loginResponseReads)
      if (jsResult.isError) {
        throw new RuntimeException("Parsing failed: " + jsResult)
      } else {
        jsResult.get
      }
    }
  }

  override lazy val token = await(getToken)

  def getToken = get(tokenReads, "action" -> "query", "meta" -> "tokens")

  def getTokens = get(tokensReads, "action" -> "tokens")

  override def run(action: Action): Future[Seq[Page]] = {
    new DslQuery(action, this).run()
  }

  def get[T](reads: Reads[T], params: (String, String)*): Future[T] =
    http.get(getUri(params: _*)) map {
      body =>
        Json.parse(body).validate(reads).get
    }

  override def getByteArray(url: String): Future[Array[Byte]] =
    http.getResponse(url) map {
      response => response.entity.data.toByteArray
    }

  override def post[T](reads: Reads[T], params: (String, String)*): Future[T] =
    post(reads, params.toMap)

  override def post[T](reads: Reads[T], params: Map[String, String]): Future[T] =
    http.post(apiUrl, params) map http.getBody map {
      body =>
        val result = Json.parse(body).validate(reads).get
        println(result)
        result
    }

  override def postMultiPart[T](reads: Reads[T], params: Map[String, String]): Future[T] =
    http.postMultiPart(apiUrl, params) map http.getBody map {
      body =>
        val json = Json.parse(body)
        val response = json.validate(reads)
        //        response.fold[T](err => {
        //          json.validate(errorReads)
        //        },
        //          success => success
        //        )
        val result = response.get
        println(result)
        result
    }

  override def postFile[T](reads: Reads[T], params: Map[String, String], fileParam: String, filename: String): Future[T] =
    http.postFile(apiUrl, params, fileParam, filename) map http.getBody map {
      body =>
        val json = Json.parse(body)
        val response = json.validate(reads)
        //        response.fold[T](err => {
        //          json.validate(errorReads)
        //        },
        //          success => success
        //        )
        response.get
    }

  def pagesByTitle(titles: Set[String]) = PageQuery.byTitles(titles, this)

  def pagesById(ids: Set[Long]) = PageQuery.byIds(ids, this)

  override def page(title: String) = PageQuery.byTitle(title, this)

  override def page(id: Long) = PageQuery.byId(id, this)

  override def pageText(title: String): Future[String] = {
    val url = getIndexUri("title" -> encodeTitle(title), "action" -> "raw")
    http.get(url)
  }

  def getIndexUri(params: (String, String)*) =
    Uri(indexUrl) withQuery (params ++ Seq("format" -> "json"): _*)

  def getUri(params: (String, String)*) =
    Uri(apiUrl) withQuery (params ++ Seq("format" -> "json"): _*)

  override def get(params: Map[String, String]): Future[String] = {
    val uri: Uri = getUri(params)

    log.info(s"$host GET url: $uri")
    http.get(uri)
  }

  override def post(params: Map[String, String]): Future[String] = {
    val uri: Uri = Uri(apiUrl)
    log.info(s"$host POST url: $uri, params: $params")
    http.post(uri, params ++ Map("format" -> "json")) map http.getBody
  }

  def getUri(params: Map[String, String]) =
    Uri(apiUrl) withQuery (params ++ Map("format" -> "json"))

  def shutdown(): Unit = {
    IO(Http).ask(Http.CloseAll)(1.second).await
    system.shutdown()
  }

  override def await[T](future: Future[T]) = Await.result(future, http.timeout)
}

object MwBot {

  import spray.caching.{Cache, LruCache}

  import scala.concurrent.ExecutionContext.Implicits.global
  import scala.concurrent._

  val system = ActorSystem()

  def create(site: Site): MwBot = {
    val bot = new MwBotImpl(site)

    bot.await(bot.login(LoginInfo.login, LoginInfo.password))
    bot
  }

  val cache: Cache[MwBot] = LruCache()

  def get(host: String): MwBot = {
    get(Site.host(host))
  }

  def get(site: Site): MwBot = {
    Await.result(cache(site.domain) {
      Future {
        create(site)
      }
    }, 1.minute)
  }

  val commons = Site.commons.domain

  val ukWiki = Site.ukWiki.domain
}


<commit_msg>extract parseResponse<commit_after>package org.scalawiki

import akka.actor.ActorSystem
import akka.event.LoggingAdapter
import akka.io.IO
import akka.pattern.ask
import org.scalawiki.dto.{LoginResponse, Page, Site}
import org.scalawiki.dto.cmd.Action
import org.scalawiki.http.{HttpClient, HttpClientBee, HttpClientSpray}
import org.scalawiki.json.MwReads._
import org.scalawiki.query.{DslQuery, PageQuery, SinglePageQuery}
import play.api.libs.json._
import spray.can.Http
import spray.http._
import spray.util._

import scala.concurrent.duration._
import scala.concurrent.{Await, Future}

trait MwBot {

  def host: String

  def login(user: String, password: String): Future[String]

  def run(action: Action): Future[Seq[Page]]

  def get(params: Map[String, String]): Future[String]

  def post(params: Map[String, String]): Future[String]

  def getByteArray(url: String): Future[Array[Byte]]

  def post[T](reads: Reads[T], params: (String, String)*): Future[T]

  def post[T](reads: Reads[T], params: Map[String, String]): Future[T]

  def postMultiPart[T](reads: Reads[T], params: Map[String, String]): Future[T]

  def postFile[T](reads: Reads[T], params: Map[String, String], fileParam: String, filename: String): Future[T]

  def page(title: String): SinglePageQuery

  def page(id: Long): SinglePageQuery

  def pageText(title: String): Future[String]

  def token: String

  def await[T](future: Future[T]): T

  def system: ActorSystem

  def log: LoggingAdapter
}

class MwBotImpl(val site: Site,
                val http: HttpClient = new HttpClientSpray(MwBot.system),
                val system: ActorSystem = MwBot.system
               ) extends MwBot {

  def this(host: String) = this(Site.host(host))

  def this(host: String, http: HttpClient) = this(Site.host(host), http)

  implicit val sys = system

  import system.dispatcher

  def host = site.domain

  val baseUrl: String = site.protocol + "://" + host + site.scriptPath

  val indexUrl = baseUrl + "/index.php"

  val apiUrl = baseUrl + "/api.php"

  def encodeTitle(title: String): String = MwUtils.normalize(title)

  override def log = system.log

  override def login(user: String, password: String): Future[String] = {
    require(user != null, "User is null")
    require(password != null, "Password is null")

    log.info(s"$host login user: $user")

    tryLogin(user, password).flatMap {
      loginResponse =>
        tryLogin(user, password, loginResponse.token).map(_.result)
    }
  }

  def tryLogin(user: String, password: String, token: Option[String] = None): Future[LoginResponse] = {
    val loginParams = Map(
      "action" -> "login", "lgname" -> user, "lgpassword" -> password, "format" -> "json"
    ) ++ token.map("lgtoken" -> _)
    http.post(apiUrl, loginParams).map { resp =>
      val body = http.getBody(resp)
      val jsResult = Json.parse(body).validate(loginResponseReads)
      if (jsResult.isError) {
        throw new RuntimeException("Parsing failed: " + jsResult)
      } else {
        jsResult.get
      }
    }
  }

  override lazy val token = await(getToken)

  def getToken = get(tokenReads, "action" -> "query", "meta" -> "tokens")

  def getTokens = get(tokensReads, "action" -> "tokens")

  override def run(action: Action): Future[Seq[Page]] = {
    new DslQuery(action, this).run()
  }

  def get[T](reads: Reads[T], params: (String, String)*): Future[T] =
    http.get(getUri(params: _*)) map {
      body =>
        parseJson(reads, body).get
    }

  def parseJson[T](reads: Reads[T], body: String): JsResult[T] =
    Json.parse(body).validate(reads)

  def parseResponse[T](reads: Reads[T], response: Future[HttpResponse]): Future[T] =
    response map http.getBody map {
      body =>
        parseJson(reads, body).get
    }

  override def getByteArray(url: String): Future[Array[Byte]] =
    http.getResponse(url) map {
      response => response.entity.data.toByteArray
    }

  override def post[T](reads: Reads[T], params: (String, String)*): Future[T] =
    post(reads, params.toMap)

  override def post[T](reads: Reads[T], params: Map[String, String]): Future[T] =
    parseResponse(reads, http.post(apiUrl, params))

  override def postMultiPart[T](reads: Reads[T], params: Map[String, String]): Future[T] =
    parseResponse(reads, http.postMultiPart(apiUrl, params))

  override def postFile[T](reads: Reads[T], params: Map[String, String], fileParam: String, filename: String): Future[T] =
    parseResponse(reads, http.postFile(apiUrl, params, fileParam, filename))

  def pagesByTitle(titles: Set[String]) = PageQuery.byTitles(titles, this)

  def pagesById(ids: Set[Long]) = PageQuery.byIds(ids, this)

  override def page(title: String) = PageQuery.byTitle(title, this)

  override def page(id: Long) = PageQuery.byId(id, this)

  override def pageText(title: String): Future[String] = {
    val url = getIndexUri("title" -> encodeTitle(title), "action" -> "raw")
    http.get(url)
  }

  def getIndexUri(params: (String, String)*) =
    Uri(indexUrl) withQuery (params ++ Seq("format" -> "json"): _*)

  def getUri(params: (String, String)*) =
    Uri(apiUrl) withQuery (params ++ Seq("format" -> "json"): _*)

  override def get(params: Map[String, String]): Future[String] = {
    val uri: Uri = getUri(params)

    log.info(s"$host GET url: $uri")
    http.get(uri)
  }

  override def post(params: Map[String, String]): Future[String] = {
    val uri: Uri = Uri(apiUrl)
    log.info(s"$host POST url: $uri, params: $params")
    http.post(uri, params ++ Map("format" -> "json")) map http.getBody
  }

  def getUri(params: Map[String, String]) =
    Uri(apiUrl) withQuery (params ++ Map("format" -> "json"))

  def shutdown(): Unit = {
    IO(Http).ask(Http.CloseAll)(1.second).await
    system.shutdown()
  }

  override def await[T](future: Future[T]) = Await.result(future, http.timeout)
}

object MwBot {

  import spray.caching.{Cache, LruCache}

  import scala.concurrent.ExecutionContext.Implicits.global
  import scala.concurrent._

  val system = ActorSystem()

  def create(site: Site): MwBot = {
    val bot = new MwBotImpl(site)

    bot.await(bot.login(LoginInfo.login, LoginInfo.password))
    bot
  }

  val cache: Cache[MwBot] = LruCache()

  def get(host: String): MwBot = {
    get(Site.host(host))
  }

  def get(site: Site): MwBot = {
    Await.result(cache(site.domain) {
      Future {
        create(site)
      }
    }, 1.minute)
  }

  val commons = Site.commons.domain

  val ukWiki = Site.ukWiki.domain
}



{'fraction_non_alphanumeric': 0.14518045447794445, 'fraction_numerical': 0.0002970444081390168, 'mean_word_length': 3.4845154845154847, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '10515', 'n_tokens_mistral': 4597, 'n_tokens_neox': 4392, 'n_words': 1058}