<commit_before>package lv.addresses.indexer

trait AddressResolver { this: AddressFinder =>
  /** Address format:
        <IEL> <NLT> - <DZI>, <CIEM>, <PAG>|<PIL>, <NOV>
        Examples:
          	Ancīši, Ancene, Asares pag., Aknīstes nov.
            Vīlandes iela 18 - 1, Rīga

  */
  def resolve(resolvable: String): ResolvedAddress = {
    import Constants._
    case class Basta(resolved: Option[Address]) extends Exception
    def to_str(addr: Address) = addr.address.toLowerCase.replace("\n", ", ")
    def all_words_match(str: String, addr: Address) =
      (str.split(SEPARATOR_REGEXP) zip addr.address.toLowerCase.split(SEPARATOR_REGEXP))
        .forall {case (s1, s2) => s1 == s2}
    def full_resolve(addressString: String): Option[Address] = search(addressString)(2) match {
      case Array(address) if to_str(address) startsWith addressString =>
        Some(address) //only one match take that if beginning matches
      case Array(address, _) if addressString == to_str(address) =>
        Some(address) //exact match
      case Array(a1, a2) if all_words_match(addressString, a1) && !all_words_match(addressString, a2) =>
        Some(a1) //first match is better than second, so choose first
      case _ =>
        None
    }
    val addressString = resolvable.toLowerCase
    ResolvedAddress(
      resolvable,
      full_resolve(addressString).orElse {
        try addressString
          .split(",")
          .map(_.trim)
          .foldRight(None: Option[Address]) { (a, resolvedAddr) =>
            full_resolve(a + resolvedAddr
              .map(", " + _.address.replace("\n", ", "))
              .mkString
              .toLowerCase
            ).orElse(throw Basta(resolvedAddr))
          }
        catch {
          case Basta(resolved) => resolved
        }
      }
    )
  }
}<commit_msg>eliminate quotes resolving addresses<commit_after>package lv.addresses.indexer

trait AddressResolver { this: AddressFinder =>
  /** Address format:
        <IEL> <NLT> - <DZI>, <CIEM>, <PAG>|<PIL>, <NOV>
        Examples:
          	Ancīši, Ancene, Asares pag., Aknīstes nov.
            Vīlandes iela 18 - 1, Rīga

  */
  def resolve(resolvable: String): ResolvedAddress = {
    import Constants._
    case class Basta(resolved: Option[Address]) extends Exception
    def to_str(addr: Address) = addr.address.toLowerCase.replace("\n", ", ").replace("\"", "")
    def all_words_match(str: String, addr: Address) =
      (str.split(SEPARATOR_REGEXP) zip addr.address.toLowerCase.split(SEPARATOR_REGEXP))
        .forall {case (s1, s2) => s1 == s2}
    def full_resolve(addressString: String): Option[Address] = search(addressString)(2) match {
      case Array(address) if to_str(address) startsWith addressString =>
        Some(address) //only one match take that if beginning matches
      case Array(address, _) if addressString == to_str(address) =>
        Some(address) //exact match
      case Array(a1, a2) if all_words_match(addressString, a1) && !all_words_match(addressString, a2) =>
        Some(a1) //first match is better than second, so choose first
      case _ =>
        None
    }
    val addressString = resolvable.toLowerCase.replace("\"", "")
    ResolvedAddress(
      resolvable,
      full_resolve(addressString).orElse {
        try addressString
          .split(",")
          .map(_.trim)
          .foldRight(None: Option[Address]) { (a, resolvedAddr) =>
            full_resolve(a + resolvedAddr
              .map(", " + _.address.replace("\n", ", "))
              .mkString
              .toLowerCase
            ).orElse(throw Basta(resolvedAddr))
          }
        catch {
          case Basta(resolved) => resolved
        }
      }
    )
  }
}
{'fraction_non_alphanumeric': 0.12510110541925049, 'fraction_numerical': 0.00700997573469938, 'mean_word_length': 2.519924098671727, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7649706', 'n_tokens_mistral': 1137, 'n_tokens_neox': 1140, 'n_words': 266}