<commit_before>package controllers

import play.api._
import play.api.libs.json.Json
import play.api.mvc._
import services._
import scala.io.Source
import java.io.File

object Application extends Controller {

  private def loadCsv(name: String, filename: String) = {
    val unigramsFile = new File(filename)
    if (!unigramsFile.exists()) sys error s"unable to load input CSV file: $filename"

    Logger.info(s"Loading $name from file: $filename")
    new BinarySearchCSV(unigramsFile)
  }

  val cppExecutable = System.getProperty("anagrams-cpp", "/Users/markhatton/Src/Mark/anagrams-cpp/anagrams")

  val unigramsFile = System.getProperty("unigrams", "/Users/markhatton/unigrams")

  val bigramsFile = System.getProperty("bigrams", "/Users/markhatton/bigrams")

  val unigrams = loadCsv("unigrams", unigramsFile)

  val bigrams = loadCsv("bigrams", bigramsFile)

  val anagramSolver: AnagramSolver = System.getProperty("solver", "native") match {
    case "external" =>
      Logger.info(s"Using external anagrams solver: $cppExecutable")
      ExternalAnagramSolver

    case "native" =>
      Logger.info(s"Using native anagrams solver")

      val dictionaryFile = System.getProperty("dictionary", "/usr/share/dict/words")

      if (!new File(dictionaryFile).exists) sys error s"unable to load input dictionary file: $dictionaryFile"
      val dictionary = {
        Logger.info(s"Loading dictionary from file: $dictionaryFile")
        val dictionary = Source.fromFile(dictionaryFile).getLines()
        dictionary.filter{s => s.length > ShortWords.maxLength || ShortWords.toSet.contains(s)}.toSet
      }
      new NativeAnagramSolver(dictionary, unigrams)
  }

  val sorter = {
    val presenter = new AnagramPresenter(unigrams, bigrams)
    new AnagramSorter(unigrams, presenter)
  }
  
  
  private val BotUserAgent = """^[A-Za-z0-9]+[Bb][Oo][Tt]/.*""".r

  private def showIndex = Ok(views.html.index())

  def index = Action { request =>
    request.headers.get("User-Agent") match {
      case Some(BotUserAgent()) =>
        request.getQueryString("s") match {
          case Some(s) if s.trim.nonEmpty =>
            Ok(views.html.solve(anagramSolver, sorter, s))  // non-JS route for bots
          case _ =>
            showIndex
        }
      case _ =>
        showIndex
    }
  }
<commit_msg>improved bot useragent regex<commit_after>package controllers

import play.api._
import play.api.libs.json.Json
import play.api.mvc._
import services._
import scala.io.Source
import java.io.File

object Application extends Controller {

  private def loadCsv(name: String, filename: String) = {
    val unigramsFile = new File(filename)
    if (!unigramsFile.exists()) sys error s"unable to load input CSV file: $filename"

    Logger.info(s"Loading $name from file: $filename")
    new BinarySearchCSV(unigramsFile)
  }

  val cppExecutable = System.getProperty("anagrams-cpp", "/Users/markhatton/Src/Mark/anagrams-cpp/anagrams")

  val unigramsFile = System.getProperty("unigrams", "/Users/markhatton/unigrams")

  val bigramsFile = System.getProperty("bigrams", "/Users/markhatton/bigrams")

  val unigrams = loadCsv("unigrams", unigramsFile)

  val bigrams = loadCsv("bigrams", bigramsFile)

  val anagramSolver: AnagramSolver = System.getProperty("solver", "native") match {
    case "external" =>
      Logger.info(s"Using external anagrams solver: $cppExecutable")
      ExternalAnagramSolver

    case "native" =>
      Logger.info(s"Using native anagrams solver")

      val dictionaryFile = System.getProperty("dictionary", "/usr/share/dict/words")

      if (!new File(dictionaryFile).exists) sys error s"unable to load input dictionary file: $dictionaryFile"
      val dictionary = {
        Logger.info(s"Loading dictionary from file: $dictionaryFile")
        val dictionary = Source.fromFile(dictionaryFile).getLines()
        dictionary.filter{s => s.length > ShortWords.maxLength || ShortWords.toSet.contains(s)}.toSet
      }
      new NativeAnagramSolver(dictionary, unigrams)
  }

  val sorter = {
    val presenter = new AnagramPresenter(unigrams, bigrams)
    new AnagramSorter(unigrams, presenter)
  }
  
  
  private val BotUserAgent = """^.*\([^;]*;.*[A-Za-z0-9]+[Bb][Oo][Tt]/.*""".r

  private def showIndex = Ok(views.html.index())

  def index = Action { request =>
    request.headers.get("User-Agent") match {
      case Some(BotUserAgent()) =>
        request.getQueryString("s") match {
          case Some(s) if s.trim.nonEmpty =>
            Ok(views.html.solve(anagramSolver, sorter, s))  // non-JS route for bots
          case _ =>
            showIndex
        }
      case _ =>
        showIndex
    }
  }

{'fraction_non_alphanumeric': 0.12309671885052541, 'fraction_numerical': 0.0008578168561012224, 'mean_word_length': 3.446139180171592, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3724820', 'n_tokens_mistral': 1551, 'n_tokens_neox': 1453, 'n_words': 327}