<commit_before>/*
 * Copyright (C) 2009 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.i18n.phonenumbers;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.i18n.phonenumbers.Phonemetadata.NumberFormat;
import com.google.i18n.phonenumbers.Phonemetadata.PhoneMetadata;
import com.google.i18n.phonenumbers.Phonemetadata.PhoneMetadataCollection;
import com.google.i18n.phonenumbers.Phonemetadata.PhoneNumberDesc;
import com.google.i18n.phonenumbers.Phonenumber.PhoneNumber;
import com.google.i18n.phonenumbers.Phonenumber.PhoneNumber.CountryCodeSource;
import com.google.protobuf.MessageLite;

import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Utility for international phone numbers. Functionality includes formatting, parsing and
 * validation.
 *
 * @author <NAME>
 * @author <NAME>
 */
public class PhoneNumberUtil {
  // The minimum and maximum length of the national significant number.
  private static final int MIN_LENGTH_FOR_NSN = 3;
  private static final int MAX_LENGTH_FOR_NSN = 15;
  private static final String META_DATA_FILE =
      "/com/google/i18n/phonenumbers/src/generated_files/PhoneNumberMetadataProto";
  private static final Logger LOGGER = Logger.getLogger(PhoneNumberUtil.class.getName());

  // A mapping from a country code to a region code which denotes the country/region
  // represented by that country code. Note countries under NANPA share the country code 1,
  // Russia and Kazakhstan share the country code 7, and many French territories in the Indian
  // Ocean share the country code 262. Under this map, 1 is mapped to US, 7 is mapped to RU,
  // and 262 is mapped to RE.
  private final Map<Integer, String> countryCodeToRegionCodeMap = new HashMap<Integer, String>();

  // The set of countries that share country code 1.
  private final Set<String> nanpaCountries = new HashSet<String>();
  private static final int NANPA_COUNTRY_CODE = 1;

  // The set of countries that share country code 7.
  private final Set<String> russiaFederationCountries = new HashSet<String>(2);
  private static final int RUSSIAN_FED_COUNTRY_CODE = 7;

  // The set of countries that share country code 262.
  private final Set<String> frenchIndianOceanTerritories = new HashSet<String>(6);

  private static final int FRENCH_INDIAN_OCEAN_COUNTRY_CODE = 262;

  // The PLUS_SIGN signifies the international prefix.
  static final char PLUS_SIGN = '+';

  // These mappings map a character (key) to a specific digit that should replace it for
  // normalization purposes. Non-European digits that may be used in phone numbers are mapped to a
  // European equivalent.
  static final Map<Character, Character> DIGIT_MAPPINGS =
      new ImmutableMap.Builder<Character, Character>()
      .put('0', '0')
      .put('\uFF10', '0')  // Fullwidth digit 0
      .put('\u0660', '0')  // Arabic-indic digit 0
      .put('1', '1')
      .put('\uFF11', '1')  // Fullwidth digit 1
      .put('\u0661', '1')  // Arabic-indic digit 1
      .put('2', '2')
      .put('\uFF12', '2')  // Fullwidth digit 2
      .put('\u0662', '2')  // Arabic-indic digit 2
      .put('3', '3')
      .put('\uFF13', '3')  // Fullwidth digit 3
      .put('\u0663', '3')  // Arabic-indic digit 3
      .put('4', '4')
      .put('\uFF14', '4')  // Fullwidth digit 4
      .put('\u0664', '4')  // Arabic-indic digit 4
      .put('5', '5')
      .put('\uFF15', '5')  // Fullwidth digit 5
      .put('\u0665', '5')  // Arabic-indic digit 5
      .put('6', '6')
      .put('\uFF16', '6')  // Fullwidth digit 6
      .put('\u0666', '6')  // Arabic-indic digit 6
      .put('7', '7')
      .put('\uFF17', '7')  // Fullwidth digit 7
      .put('\u0667', '7')  // Arabic-indic digit 7
      .put('8', '8')
      .put('\uFF18', '8')  // Fullwidth digit 8
      .put('\u0668', '8')  // Arabic-indic digit 8
      .put('9', '9')
      .put('\uFF19', '9')  // Fullwidth digit 9
      .put('\u0669', '9')  // Arabic-indic digit 9
      .build();

  // Only upper-case variants of alpha characters are stored.
  private static final Map<Character, Character> ALPHA_MAPPINGS =
      new ImmutableMap.Builder<Character, Character>()
      .put('A', '2')
      .put('B', '2')
      .put('C', '2')
      .put('D', '3')
      .put('E', '3')
      .put('F', '3')
      .put('G', '4')
      .put('H', '4')
      .put('I', '4')
      .put('J', '5')
      .put('K', '5')
      .put('L', '5')
      .put('M', '6')
      .put('N', '6')
      .put('O', '6')
      .put('P', '7')
      .put('Q', '7')
      .put('R', '7')
      .put('S', '7')
      .put('T', '8')
      .put('U', '8')
      .put('V', '8')
      .put('W', '9')
      .put('X', '9')
      .put('Y', '9')
      .put('Z', '9')
      .build();

  // For performance reasons, amalgamate both into one map.
  private static final Map<Character, Character> ALL_NORMALIZATION_MAPPINGS =
      new ImmutableMap.Builder<Character, Character>()
      .putAll(ALPHA_MAPPINGS)
      .putAll(DIGIT_MAPPINGS)
      .build();

  // A list of all country codes where national significant numbers (excluding any national prefix)
  // exist that start with a leading zero.
  private static final Set<Integer> LEADING_ZERO_COUNTRIES =
      new ImmutableSet.Builder<Integer>()
      .add(39)  // Italy
      .add(225)  // Cte d'Ivoire
      .add(227)  // Niger
      .add(228)  // Togo
      .add(240)  // Equatorial Guinea
      .add(241)  // Gabon
      .build();

  // Pattern that makes it easy to distinguish whether a country has a unique international dialing
  // prefix or not. If a country has a unique international prefix (e.g. 011 in USA), it will be
  // represented as a string that contains a sequence of ASCII digits. If there are multiple
  // available international prefixes in a country, they will be represented as a regex string that
  // always contains character(s) other than ASCII digits.
  // Note this regex also includes tilde, which signals waiting for the tone.
  private static final Pattern UNIQUE_INTERNATIONAL_PREFIX =
      Pattern.compile("[\\d]+(?:[~\u2053\u223C\uFF5E][\\d]+)?");

  // Regular expression of acceptable punctuation found in phone numbers. This excludes punctuation
  // found as a leading character only.
  // This consists of dash characters, white space characters, full stops, slashes,
  // square brackets, parentheses and tildes. It also includes the letter 'x' as that is found as a
  // placeholder for carrier information in some phone numbers.<commit_msg>Remove non-Ascii character from source code.<commit_after>/*
 * Copyright (C) 2009 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.i18n.phonenumbers;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.i18n.phonenumbers.Phonemetadata.NumberFormat;
import com.google.i18n.phonenumbers.Phonemetadata.PhoneMetadata;
import com.google.i18n.phonenumbers.Phonemetadata.PhoneMetadataCollection;
import com.google.i18n.phonenumbers.Phonemetadata.PhoneNumberDesc;
import com.google.i18n.phonenumbers.Phonenumber.PhoneNumber;
import com.google.i18n.phonenumbers.Phonenumber.PhoneNumber.CountryCodeSource;
import com.google.protobuf.MessageLite;

import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Utility for international phone numbers. Functionality includes formatting, parsing and
 * validation.
 *
 * @author <NAME>
 * @author <NAME>
 */
public class PhoneNumberUtil {
  // The minimum and maximum length of the national significant number.
  private static final int MIN_LENGTH_FOR_NSN = 3;
  private static final int MAX_LENGTH_FOR_NSN = 15;
  private static final String META_DATA_FILE =
      "/com/google/i18n/phonenumbers/src/generated_files/PhoneNumberMetadataProto";
  private static final Logger LOGGER = Logger.getLogger(PhoneNumberUtil.class.getName());

  // A mapping from a country code to a region code which denotes the country/region
  // represented by that country code. Note countries under NANPA share the country code 1,
  // Russia and Kazakhstan share the country code 7, and many French territories in the Indian
  // Ocean share the country code 262. Under this map, 1 is mapped to US, 7 is mapped to RU,
  // and 262 is mapped to RE.
  private final Map<Integer, String> countryCodeToRegionCodeMap = new HashMap<Integer, String>();

  // The set of countries that share country code 1.
  private final Set<String> nanpaCountries = new HashSet<String>();
  private static final int NANPA_COUNTRY_CODE = 1;

  // The set of countries that share country code 7.
  private final Set<String> russiaFederationCountries = new HashSet<String>(2);
  private static final int RUSSIAN_FED_COUNTRY_CODE = 7;

  // The set of countries that share country code 262.
  private final Set<String> frenchIndianOceanTerritories = new HashSet<String>(6);

  private static final int FRENCH_INDIAN_OCEAN_COUNTRY_CODE = 262;

  // The PLUS_SIGN signifies the international prefix.
  static final char PLUS_SIGN = '+';

  // These mappings map a character (key) to a specific digit that should replace it for
  // normalization purposes. Non-European digits that may be used in phone numbers are mapped to a
  // European equivalent.
  static final Map<Character, Character> DIGIT_MAPPINGS =
      new ImmutableMap.Builder<Character, Character>()
      .put('0', '0')
      .put('\uFF10', '0')  // Fullwidth digit 0
      .put('\u0660', '0')  // Arabic-indic digit 0
      .put('1', '1')
      .put('\uFF11', '1')  // Fullwidth digit 1
      .put('\u0661', '1')  // Arabic-indic digit 1
      .put('2', '2')
      .put('\uFF12', '2')  // Fullwidth digit 2
      .put('\u0662', '2')  // Arabic-indic digit 2
      .put('3', '3')
      .put('\uFF13', '3')  // Fullwidth digit 3
      .put('\u0663', '3')  // Arabic-indic digit 3
      .put('4', '4')
      .put('\uFF14', '4')  // Fullwidth digit 4
      .put('\u0664', '4')  // Arabic-indic digit 4
      .put('5', '5')
      .put('\uFF15', '5')  // Fullwidth digit 5
      .put('\u0665', '5')  // Arabic-indic digit 5
      .put('6', '6')
      .put('\uFF16', '6')  // Fullwidth digit 6
      .put('\u0666', '6')  // Arabic-indic digit 6
      .put('7', '7')
      .put('\uFF17', '7')  // Fullwidth digit 7
      .put('\u0667', '7')  // Arabic-indic digit 7
      .put('8', '8')
      .put('\uFF18', '8')  // Fullwidth digit 8
      .put('\u0668', '8')  // Arabic-indic digit 8
      .put('9', '9')
      .put('\uFF19', '9')  // Fullwidth digit 9
      .put('\u0669', '9')  // Arabic-indic digit 9
      .build();

  // Only upper-case variants of alpha characters are stored.
  private static final Map<Character, Character> ALPHA_MAPPINGS =
      new ImmutableMap.Builder<Character, Character>()
      .put('A', '2')
      .put('B', '2')
      .put('C', '2')
      .put('D', '3')
      .put('E', '3')
      .put('F', '3')
      .put('G', '4')
      .put('H', '4')
      .put('I', '4')
      .put('J', '5')
      .put('K', '5')
      .put('L', '5')
      .put('M', '6')
      .put('N', '6')
      .put('O', '6')
      .put('P', '7')
      .put('Q', '7')
      .put('R', '7')
      .put('S', '7')
      .put('T', '8')
      .put('U', '8')
      .put('V', '8')
      .put('W', '9')
      .put('X', '9')
      .put('Y', '9')
      .put('Z', '9')
      .build();

  // For performance reasons, amalgamate both into one map.
  private static final Map<Character, Character> ALL_NORMALIZATION_MAPPINGS =
      new ImmutableMap.Builder<Character, Character>()
      .putAll(ALPHA_MAPPINGS)
      .putAll(DIGIT_MAPPINGS)
      .build();

  // A list of all country codes where national significant numbers (excluding any national prefix)
  // exist that start with a leading zero.
  private static final Set<Integer> LEADING_ZERO_COUNTRIES =
      new ImmutableSet.Builder<Integer>()
      .add(39)  // Italy
      .add(225)  // Cote d'Ivoire
      .add(227)  // Niger
      .add(228)  // Togo
      .add(240)  // Equatorial Guinea
      .add(241)  // Gabon
      .build();

  // Pattern that makes it easy to distinguish whether a country has a unique international dialing
  // prefix or not. If a country has a unique international prefix (e.g. 011 in USA), it will be
  // represented as a string that contains a sequence of ASCII digits. If there are multiple
  // available international prefixes in a country, they will be represented as a regex string that
  // always contains character(s) other than ASCII digits.
  // Note this regex also includes tilde, which signals waiting for the tone.
  private static final Pattern UNIQUE_INTERNATIONAL_PREFIX =
      Pattern.compile("[\\d]+(?:[~\u2053\u223C\uFF5E][\\d]+)?");

  // Regular expression of acceptable punctuation found in phone numbers. This excludes punctuation
  // found as a leading character only.
  // This consists of dash characters, white space characters, full stops, slashes,
  // square brackets, parentheses and tildes. It also includes the letter 'x' as that is found as a
  // placeholder for carrier information in some phone numbers.
{'fraction_non_alphanumeric': 0.1321662434852332, 'fraction_numerical': 0.029800881999198184, 'mean_word_length': 3.107299670691548, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '675199', 'n_tokens_mistral': 4949, 'n_tokens_neox': 4677, 'n_words': 1550}