<commit_before>package foodtruck.schedule;

import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.annotation.Nullable;

import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;

import foodtruck.model.Truck;

/**
 * @author <EMAIL>
 * @since Jul 20, 2011
 */
public class AddressExtractor {
  private final List<PatternTransform> patterns;
  private final static String INTERSECTION_PARTIAL =
      "(N|E|W|S\\s+)?[A-Z0-9][a-zA-Z0-9]+(\\s+(st|St|Drive|Dr|Buren|BUREN|Blvd|Ave)\\.?)?";
  private final static String INTERSECTION_AND =
      "((\\s+(and|at|near|n)\\s+)|(\\s*(\\&|\\\\|\\/)\\s*))";

  private static Function<String, String> keyword(final String toWhat) {
    return new Function<String, String>() {
      @Override public String apply(@Nullable String input) {
        return toWhat;
      }
    };
  }

  public AddressExtractor() {
    Function<String, String> cityAppender = new Function<String, String>() {
      public String apply(String input) {
        return
            input.replaceAll("(&|b\\/w|\\/)", " and ").replace(" at ", " and ").replace("  and  ",
                " and ").replace(" between ", " and ").replace(" in", "").replace(" n ", " and ")
                .replace(" on ", " and ").replace(" near ", " and ")
                .trim() +
                ", Chicago, IL";
      }
    };
    Function<String, String> keywordReplace = new Function<String, String>() {
      final ImmutableMap<String, String> keywords = ImmutableMap.of("@wttw", "WTTW", "harpo",
          "Harpo Studios", "grant park", "<NAME>",
          "presidential towers", "Presidential Towers");

      public String apply(String input) {
        return keywords.get(input.toLowerCase());
      }
    };
    final Pattern others = Pattern.compile(" w/ \\d+ others");
    Function<String, String> foursquareMassage = new Function<String, String>() {
      @Override public String apply(String input) {
        Matcher m = others.matcher(input);
        if (m.find()) {
          return input.substring(0, m.start());
        }
        return input;
      }
    };
    patterns = ImmutableList.of(
        // foursquare format
        new PatternTransform(Pattern.compile("\\(@ (.*)\\)"), foursquareMassage, true, 1),
        // University of Chicago
        new PatternTransform(
            Pattern.compile("U of Chicago|UofC|UChicago|Ellis \\(57 / 58\\)|The Reg\\b",
                Pattern.CASE_INSENSITIVE),
            keyword("57th and Ellis, Chicago, IL"), true, 0),
        new PatternTransform(Pattern.compile("\\bjeff/jack\\b", Pattern.CASE_INSENSITIVE),
            keyword("Jefferson and Jackson, Chicago, IL"), false, 0),
        // Harpo Studios
        new PatternTransform(
            Pattern.compile("(harpo|rosie)\\b", Pattern.CASE_INSENSITIVE), keyword("Harpo Studios"),
            true,
            0),
        new PatternTransform(Pattern.compile("Trump", Pattern.CASE_INSENSITIVE),
            keyword("400 North Wabash, Chicago, IL"), true, 0),
        // AON
        new PatternTransform(Pattern.compile("@aon| aon|#aon", Pattern.CASE_INSENSITIVE),
            keyword("Randolph and Columbus, Chicago, IL"), true, 0),
        // Old Town
        new PatternTransform(Pattern.compile("old town", Pattern.CASE_INSENSITIVE),
            keyword("North and Wells, Chicago, IL"), true, 0),
        // Lincoln Park Zoo
        new PatternTransform(Pattern.compile("lincoln park zoo", Pattern.CASE_INSENSITIVE),
            keyword("Lincoln Park Zoo, Chicago, IL"), true, 0),
        // Northeastern
        new PatternTransform(Pattern.compile("\\bNEIU\\b", Pattern.CASE_INSENSITIVE),
            keyword("5500 North Saint Louis Avenue, Chicago, IL"), true, 0),
        // address format
        new PatternTransform(Pattern.compile("(^:)*\\d+\\s*[NnSsEeWw]\\.?\\s+\\w+"), cityAppender,
            true, 0),
        // Willis Tower
        new PatternTransform(Pattern.compile("sears|willis", Pattern.CASE_INSENSITIVE),
            keyword("Wacker and Adams, Chicago, IL"), true, 0),
        // Rush Medical
        new PatternTransform(Pattern
            .compile("\\buic\\b(.*)\\brush\\b|\\brush\\b(.*)\\buic\\b|\\brush medical\\b",
                Pattern.CASE_INSENSITIVE),
            keyword("600 South Paulina, Chicago, IL"), true, 0),
        // UIC Medical
        new PatternTransform(Pattern
            .compile("\\buic medical\\b", Pattern.CASE_INSENSITIVE),
            keyword("<NAME>, Chicago, IL"), true, 0),
        // UIC
        new PatternTransform(Pattern.compile("UIC\\b", Pattern.CASE_INSENSITIVE),
            keyword("Vernon Park Circle, Chicago, IL"), true, 0),
        // tamale spaceship format
        new PatternTransform(Pattern.compile("<<(.*)>>"), null, true, 1),
        // Between two streets
        new PatternTransform(Pattern.compile(
            "between " + INTERSECTION_PARTIAL + INTERSECTION_AND + "(" +
                INTERSECTION_PARTIAL + " on " + INTERSECTION_PARTIAL + ")"),
            cityAppender, true, 9),
        // Between two streets
        new PatternTransform(Pattern.compile(
            "(" + INTERSECTION_PARTIAL + " ((in )?between|b/w) " + INTERSECTION_PARTIAL +
                ")((\\s+(and|at)\\s+)|(\\s*(\\&|\\\\|\\/)\\s*))" + INTERSECTION_PARTIAL),
            cityAppender, true, 1),
        // Between two streets
        new PatternTransform(Pattern.compile(
            "(" + INTERSECTION_PARTIAL + " ((in )?between|b/w) " + INTERSECTION_PARTIAL +
                ")((\\s+(and|at)\\s+)|(\\s*(\\&|\\\\|\\/)\\s*))" + INTERSECTION_PARTIAL,
            Pattern.CASE_INSENSITIVE),
            cityAppender, true, 1, ImmutableSet.of("steakwch")),
        // intersection format preceded by at
        new PatternTransform(Pattern.compile(
            "[A-Z0-9][a-zA-Z0-9]+\\s+at\\s+(" + INTERSECTION_PARTIAL +
                "((\\s+(and|at|n)\\s+)|(\\s*(\\&|\\\\|\\/)\\s*))" + INTERSECTION_PARTIAL + ")"),
            cityAppender, true, 1),
        // intersection format
        new PatternTransform(Pattern.compile(
            INTERSECTION_PARTIAL + INTERSECTION_AND + INTERSECTION_PARTIAL),
            cityAppender, false, 0),
        // case-insensitive intersection
        new PatternTransform(Pattern.compile(
            "at (" + INTERSECTION_PARTIAL + INTERSECTION_AND + INTERSECTION_PARTIAL + ")",
            Pattern.CASE_INSENSITIVE),
            cityAppender, false, 1,
            ImmutableSet.of("steakwch", "rzjp6cakes", "flirtycupcakes", "theslideride")),
        // special case intersections
        // Merchandise Mart
        new PatternTransform(
            Pattern.compile("Merch(andise)? mart|#MerchandiseMart", Pattern.CASE_INSENSITIVE),
            keyword("Merchandise Mart"), true, 0),<commit_msg>Added caponiesexp to the list of lower-case-allowed trucks<commit_after>package foodtruck.schedule;

import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.annotation.Nullable;

import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;

import foodtruck.model.Truck;

/**
 * @author <EMAIL>
 * @since Jul 20, 2011
 */
public class AddressExtractor {
  private final List<PatternTransform> patterns;
  private final static String INTERSECTION_PARTIAL =
      "(N|E|W|S\\s+)?[A-Z0-9][a-zA-Z0-9]+(\\s+(st|St|Drive|Dr|Buren|BUREN|Blvd|Ave)\\.?)?";
  private final static String INTERSECTION_AND =
      "((\\s+(and|at|near|n)\\s+)|(\\s*(\\&|\\\\|\\/)\\s*))";

  private static Function<String, String> keyword(final String toWhat) {
    return new Function<String, String>() {
      @Override public String apply(@Nullable String input) {
        return toWhat;
      }
    };
  }

  public AddressExtractor() {
    Function<String, String> cityAppender = new Function<String, String>() {
      public String apply(String input) {
        return
            input.replaceAll("(&|b\\/w|\\/)", " and ").replace(" at ", " and ").replace("  and  ",
                " and ").replace(" between ", " and ").replace(" in", "").replace(" n ", " and ")
                .replace(" on ", " and ").replace(" near ", " and ")
                .trim() +
                ", Chicago, IL";
      }
    };
    Function<String, String> keywordReplace = new Function<String, String>() {
      final ImmutableMap<String, String> keywords = ImmutableMap.of("@wttw", "WTTW", "harpo",
          "Harpo Studios", "grant park", "Grant Park",
          "presidential towers", "Presidential Towers");

      public String apply(String input) {
        return keywords.get(input.toLowerCase());
      }
    };
    final Pattern others = Pattern.compile(" w/ \\d+ others");
    Function<String, String> foursquareMassage = new Function<String, String>() {
      @Override public String apply(String input) {
        Matcher m = others.matcher(input);
        if (m.find()) {
          return input.substring(0, m.start());
        }
        return input;
      }
    };
    patterns = ImmutableList.of(
        // foursquare format
        new PatternTransform(Pattern.compile("\\(@ (.*)\\)"), foursquareMassage, true, 1),
        // University of Chicago
        new PatternTransform(
            Pattern.compile("U of Chicago|UofC|UChicago|Ellis \\(57 / 58\\)|The Reg\\b",
                Pattern.CASE_INSENSITIVE),
            keyword("57th and Ellis, Chicago, IL"), true, 0),
        new PatternTransform(Pattern.compile("\\bjeff/jack\\b", Pattern.CASE_INSENSITIVE),
            keyword("Jefferson and Jackson, Chicago, IL"), false, 0),
        // <NAME>
        new PatternTransform(
            Pattern.compile("(harpo|rosie)\\b", Pattern.CASE_INSENSITIVE), keyword("<NAME>"),
            true,
            0),
        new PatternTransform(Pattern.compile("Trump", Pattern.CASE_INSENSITIVE),
            keyword("400 North Wabash, Chicago, IL"), true, 0),
        // AON
        new PatternTransform(Pattern.compile("@aon| aon|#aon", Pattern.CASE_INSENSITIVE),
            keyword("Randolph and Columbus, Chicago, IL"), true, 0),
        // Old Town
        new PatternTransform(Pattern.compile("old town", Pattern.CASE_INSENSITIVE),
            keyword("North and Wells, Chicago, IL"), true, 0),
        // Lincoln Park Zoo
        new PatternTransform(Pattern.compile("lincoln park zoo", Pattern.CASE_INSENSITIVE),
            keyword("Lincoln Park Zoo, Chicago, IL"), true, 0),
        // Northeastern
        new PatternTransform(Pattern.compile("\\bNEIU\\b", Pattern.CASE_INSENSITIVE),
            keyword("5500 North Saint Louis Avenue, Chicago, IL"), true, 0),
        // address format
        new PatternTransform(Pattern.compile("(^:)*\\d+\\s*[NnSsEeWw]\\.?\\s+\\w+"), cityAppender,
            true, 0),
        // Willis Tower
        new PatternTransform(Pattern.compile("sears|willis", Pattern.CASE_INSENSITIVE),
            keyword("Wacker and Adams, Chicago, IL"), true, 0),
        // Rush Medical
        new PatternTransform(Pattern
            .compile("\\buic\\b(.*)\\brush\\b|\\brush\\b(.*)\\buic\\b|\\brush medical\\b",
                Pattern.CASE_INSENSITIVE),
            keyword("600 South Paulina, Chicago, IL"), true, 0),
        // UIC Medical
        new PatternTransform(Pattern
            .compile("\\buic medical\\b", Pattern.CASE_INSENSITIVE),
            keyword("<NAME>, Chicago, IL"), true, 0),
        // UIC
        new PatternTransform(Pattern.compile("UIC\\b", Pattern.CASE_INSENSITIVE),
            keyword("Vernon Park Circle, Chicago, IL"), true, 0),
        // tamale spaceship format
        new PatternTransform(Pattern.compile("<<(.*)>>"), null, true, 1),
        // Between two streets
        new PatternTransform(Pattern.compile(
            "between " + INTERSECTION_PARTIAL + INTERSECTION_AND + "(" +
                INTERSECTION_PARTIAL + " on " + INTERSECTION_PARTIAL + ")"),
            cityAppender, true, 9),
        // Between two streets
        new PatternTransform(Pattern.compile(
            "(" + INTERSECTION_PARTIAL + " ((in )?between|b/w) " + INTERSECTION_PARTIAL +
                ")((\\s+(and|at)\\s+)|(\\s*(\\&|\\\\|\\/)\\s*))" + INTERSECTION_PARTIAL),
            cityAppender, true, 1),
        // Between two streets
        new PatternTransform(Pattern.compile(
            "(" + INTERSECTION_PARTIAL + " ((in )?between|b/w) " + INTERSECTION_PARTIAL +
                ")((\\s+(and|at)\\s+)|(\\s*(\\&|\\\\|\\/)\\s*))" + INTERSECTION_PARTIAL,
            Pattern.CASE_INSENSITIVE),
            cityAppender, true, 1, ImmutableSet.of("steakwch")),
        // intersection format preceded by at
        new PatternTransform(Pattern.compile(
            "[A-Z0-9][a-zA-Z0-9]+\\s+at\\s+(" + INTERSECTION_PARTIAL +
                "((\\s+(and|at|n)\\s+)|(\\s*(\\&|\\\\|\\/)\\s*))" + INTERSECTION_PARTIAL + ")"),
            cityAppender, true, 1),
        // intersection format
        new PatternTransform(Pattern.compile(
            INTERSECTION_PARTIAL + INTERSECTION_AND + INTERSECTION_PARTIAL),
            cityAppender, false, 0),
        // case-insensitive intersection
        new PatternTransform(Pattern.compile(
            "at (" + INTERSECTION_PARTIAL + INTERSECTION_AND + INTERSECTION_PARTIAL + ")",
            Pattern.CASE_INSENSITIVE),
            cityAppender, false, 1,
            ImmutableSet
                .of("steakwch", "rzjp6cakes", "flirtycupcakes", "theslideride", "caponiesexp")),
        // special case intersections
        // <NAME>
        new PatternTransform(
            Pattern.compile("Merch(andise)? mart|#MerchandiseMart", Pattern.CASE_INSENSITIVE),
            keyword("<NAME>"), true, 0),
{'fraction_non_alphanumeric': 0.15892665474060824, 'fraction_numerical': 0.007728085867620751, 'mean_word_length': 2.4671297444802778, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1375126', 'n_tokens_mistral': 4366, 'n_tokens_neox': 4111, 'n_words': 875}