<commit_before>global:
    cldr_country_probability: 0.5
    localized_name_probability: 0.7
    iso_alpha_2_code_probability: 0.2
    iso_alpha_3_code_probability: 0.1

    abbreviate_street_probability: 0.3
    separate_street_probability: 0.2
    abbreviate_unit_probability: 0.3
    separate_unit_probability: 0.2

    fields: &default_fields
        - field_name: NUMBER
          component: house_number
        - field_name: STREET
          component: road
        - field_name: UNIT
          component: unit
        - field_name: CITY
          component: city
        - field_name: POSTCODE
          component: postcode

    fields_no_postcode: &fields_no_postcode
        - field_name: NUMBER
          component: house_number
        - field_name: STREET
          component: road
        - field_name: UNIT
          component: unit
        - field_name: CITY
          component: city

    fields_with_unit: &fields_with_unit
        - field_name: NUMBER
          component: house_number
        - field_name: STREET
          component: road
        - field_name: UNIT
          component: unit
        - field_name: CITY
          component: city
        - field_name: POSTCODE
          component: postcode

    fields_with_state_and_unit: &fields_with_state_and_unit
        - field_name: NUMBER
          component: house_number
        - field_name: STREET
          component: road
        - field_name: UNIT
          component: unit
        - field_name: CITY
          component: city
        - field_name: REGION
          component: state
        - field_name: POSTCODE
          component: postcode


    # Turned off by default to avoid performance penalty for reverse geocoding
    add_osm_boundaries: false
    add_osm_neighborhoods: true

    # Units have strong restrictions, have to be a number or hyphenated number
    non_numeric_units: false

    # This is fine for many countries
    numeric_postcodes_only: true

countries:
    au:
        fields: *fields_with_state_and_unit
        files:
            - filename: countrywide.csv
              non_numeric_units: true
    nz:
        files:
            - filename: countrywide.csv
            - filename: city_of_palmerston_north.csv

    be:
        files:
            - filename: flanders.csv
              language: nl
        subdirs:
            wa:
                files:
                    - filename: brussels-fr.csv
                      language: fr
                    - filename: brussels-nl.csv
                      language: nl

    ca:
        cldr_country_probability: 0.3
        ignore_fields_containing:
            city:
                - "\\bcounty\\b"
            road:
                - "\\bp\\.?\\s*o\\.?\\s*box\\b"
        subdirs:
            ab:
                add:
                    state: AB
                files:
                    - filename: airdrie.csv
                      add_osm_boundaries: true
                    - filename: banff.csv
                      add_osm_boundaries: true
                    - filename: calgary.csv
                      add_osm_boundaries: true
                    - filename: grande_prairie.csv
                      add_osm_boundaries: true
                    - filename: strathcona-county.csv

            bc:
                add:
                    state: BC
                files:
                    - filename: kamloops.csv
                      add_osm_boundaries: true
                    - filename: kelowna.csv
                      add_osm_boundaries: true
                    - filename: langley.csv
                      add_osm_boundaries: true
                    - filename: nanaimo.csv
                      add_osm_boundaries: true
                    - filename: north_cowichan.csv
                    - filename: prince_george.csv
                    - filename: province.csv
                    - filename: thompson-nicola-regional-district.csv
                    - filename: vancouver.csv
                      add_osm_boundaries: true
                    - filename: victoria.csv
                      add_osm_boundaries: true
                    - filename: west_kelowna.csv
                      add_osm_boundaries: true

    us:
        cldr_country_probability: 0.05
        ignore_fields_containing:
            city:
                - "\\bcounty\\b"
            road:
                - "\\bp\\.?\\s*o\\.?\\s*box\\b"
        subdirs:
            ak:
                add:
                    state: AK
                files:
                    - filename: anchorage.csv
                      add_osm_boundaries: true
                    - filename: fairbanks_north_star_borough.csv
                      add_osm_boundaries: true
                    - filename: kenai_peninsula_borough.csv
                    - filename: kodiak_island_borough.csv
                      add_osm_boundaries: true
                    - filename: matanuska_susitna_borough.csv
                      add_osm_boundaries: true<commit_msg>[openaddresses] adding Manitoba, Canada<commit_after>global:
    cldr_country_probability: 0.5
    localized_name_probability: 0.7
    iso_alpha_2_code_probability: 0.2
    iso_alpha_3_code_probability: 0.1

    abbreviate_street_probability: 0.3
    separate_street_probability: 0.2
    abbreviate_unit_probability: 0.3
    separate_unit_probability: 0.2

    fields: &default_fields
        - field_name: NUMBER
          component: house_number
        - field_name: STREET
          component: road
        - field_name: UNIT
          component: unit
        - field_name: CITY
          component: city
        - field_name: POSTCODE
          component: postcode

    fields_no_postcode: &fields_no_postcode
        - field_name: NUMBER
          component: house_number
        - field_name: STREET
          component: road
        - field_name: UNIT
          component: unit
        - field_name: CITY
          component: city

    fields_with_unit: &fields_with_unit
        - field_name: NUMBER
          component: house_number
        - field_name: STREET
          component: road
        - field_name: UNIT
          component: unit
        - field_name: CITY
          component: city
        - field_name: POSTCODE
          component: postcode

    fields_with_state_and_unit: &fields_with_state_and_unit
        - field_name: NUMBER
          component: house_number
        - field_name: STREET
          component: road
        - field_name: UNIT
          component: unit
        - field_name: CITY
          component: city
        - field_name: REGION
          component: state
        - field_name: POSTCODE
          component: postcode


    # Turned off by default to avoid performance penalty for reverse geocoding
    add_osm_boundaries: false
    add_osm_neighborhoods: true

    # Units have strong restrictions, have to be a number or hyphenated number
    non_numeric_units: false

    # This is fine for many countries
    numeric_postcodes_only: true

countries:
    au:
        fields: *fields_with_state_and_unit
        files:
            - filename: countrywide.csv
              non_numeric_units: true
    nz:
        files:
            - filename: countrywide.csv
            - filename: city_of_palmerston_north.csv

    be:
        files:
            - filename: flanders.csv
              language: nl
        subdirs:
            wa:
                files:
                    - filename: brussels-fr.csv
                      language: fr
                    - filename: brussels-nl.csv
                      language: nl

    ca:
        cldr_country_probability: 0.3
        ignore_fields_containing:
            city:
                - "\\bcounty\\b"
            road:
                - "\\bp\\.?\\s*o\\.?\\s*box\\b"
        subdirs:
            ab:
                add:
                    state: AB
                files:
                    - filename: airdrie.csv
                      add_osm_boundaries: true
                    - filename: banff.csv
                      add_osm_boundaries: true
                    - filename: calgary.csv
                      add_osm_boundaries: true
                    - filename: grande_prairie.csv
                      add_osm_boundaries: true
                    - filename: strathcona-county.csv

            bc:
                add:
                    state: BC
                files:
                    - filename: kamloops.csv
                      add_osm_boundaries: true
                    - filename: kelowna.csv
                      add_osm_boundaries: true
                    - filename: langley.csv
                      add_osm_boundaries: true
                    - filename: nanaimo.csv
                      add_osm_boundaries: true
                    - filename: north_cowichan.csv
                    - filename: prince_george.csv
                    - filename: province.csv
                    - filename: thompson-nicola-regional-district.csv
                    - filename: vancouver.csv
                      add_osm_boundaries: true
                    - filename: victoria.csv
                      add_osm_boundaries: true
                    - filename: west_kelowna.csv
                      add_osm_boundaries: true

            mb:
                add:
                    state: MB
                files:
                    - filename: brandon.csv

    us:
        cldr_country_probability: 0.05
        ignore_fields_containing:
            city:
                - "\\bcounty\\b"
            road:
                - "\\bp\\.?\\s*o\\.?\\s*box\\b"
        subdirs:
            ak:
                add:
                    state: AK
                files:
                    - filename: anchorage.csv
                      add_osm_boundaries: true
                    - filename: fairbanks_north_star_borough.csv
                      add_osm_boundaries: true
                    - filename: kenai_peninsula_borough.csv
                    - filename: kodiak_island_borough.csv
                      add_osm_boundaries: true
                    - filename: matanuska_susitna_borough.csv
                      add_osm_boundaries: true
{'fraction_non_alphanumeric': 0.05909492490428978, 'fraction_numerical': 0.004515559045842741, 'mean_word_length': 1.2037637897469176, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '361524', 'n_tokens_mistral': 2890, 'n_tokens_neox': 2652, 'n_words': 567}