<commit_before>package org.elasticsearch.index.analysis;


import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.math.NumberUtils;
import  org.apache.lucene.analysis.Tokenizer;
import  org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.elasticsearch.common.lang3.StringUtils;

import com.google.i18n.phonenumbers.NumberParseException;
import com.google.i18n.phonenumbers.PhoneNumberUtil;
import com.google.i18n.phonenumbers.Phonenumber.PhoneNumber;

import  java.io.IOException;
import  java.io.Reader;
import java.util.ArrayList;
import java.util.List;

public class PhoneTokenizer extends Tokenizer {

	// The raw input
	private String stringToTokenize = null;

	// Position in the tokens array. We build all the tokens and return them one at a time as incrementToken gets called. 
	private int position = 0;

	/**
	 * The tokens are determined on the first iteration and then returned one at a time
	 * thereafter.
	 */

	private List<String> tokens = null;

	// The base class grabs the charTermAttribute each time incrementToken returns
	protected CharTermAttribute charTermAttribute = addAttribute(CharTermAttribute.class);

	public PhoneTokenizer(Reader reader) {
		super(reader);
	}

	@Override
	public final boolean incrementToken() throws IOException {
		// Clear anything that is already saved in this.charTermAttribute
		this.charTermAttribute.setEmpty();

		if(tokens == null) {
			// It's the 1st iteration, chop it up into tokens. 
			generateTokens();
		}

		// Return those tokens
		return returnTokensOneAtATime();
	}

	private boolean returnTokensOneAtATime() {
		// Token have already been generated. Return them 1 at a time
		if(tokens != null) {
			if(this.position == tokens.size()) {
				// No more tokens
				return false;
			}

			// return each token, 1 at a time
			this.charTermAttribute.append(tokens.get(this.position));
			this.position += 1;
			return true;
		}
		return false;
	}

	private void generateTokens() {
		String uri = getStringToTokenize();

		tokens = new ArrayList<String>();
		tokens.add(getStringToTokenize());

		// Rip off the "tel:" or "sip:" prefix
		if (uri.indexOf("tel:") != -1 || uri.indexOf("sip:") != -1) {
			uri = uri.substring(4);
		} else {
			// If it's not formatted at least this correctly then the whole string is 1 token. Sorry, put a tel: or sip: at the beginning so we know how to treat it
			tokens.add(getStringToTokenize());
			return;
		}

		// Drop anything after @. Most likely there's nothing of interest
		String[] parts = StringUtils.split(uri, "@");
		if(parts.length == 0) {
			return;
		}
		
		String number = parts[0];

		// Add a token for the raw unmanipulated address. Note this could be a username (sip) instead of telephone number so take it as is
		tokens.add(number);	

		// Let google's libphone try to parse it
		PhoneNumberUtil phoneUtil = PhoneNumberUtil.getInstance();
		PhoneNumber numberProto = null;
		String countryCode = null;
		try{
			// ZZ is the generic "I don't know the country code" region. Google's libphone library will try to infer it.
			numberProto = phoneUtil.parse(number, "ZZ");
			if(numberProto != null) {
				// Libphone likes it! 
				countryCode = String.valueOf(numberProto.getCountryCode());
				number = String.valueOf(numberProto.getNationalNumber());

				// Add Country code, extension, and the number as tokens
				tokens.add(countryCode);
				if(!StringUtils.isEmpty(numberProto.getExtension())) {
					tokens.add(numberProto.getExtension());	
				}

				tokens.add(number);
			} 
		} catch(NumberParseException e) {
			// Libphone didn't like it, no biggie. We'll just ngram the number as it is. 
		}	

		// ngram the phone number EG 19198243333 produces 9, 91, 919, etc 
		if(NumberUtils.isNumber(number)) {
			for(int count = 1; count <= number.length(); count++) {
				String token = number.substring(0, count);
				tokens.add(token);
				if(countryCode != null) {<commit_msg>AR-4917 seems it's possible for google libphone to throw a StringIndexOutOfBoundsException parsing certain strings. Instead of failing to index, use the existing fallback of prefix ngram tokenization<commit_after>package org.elasticsearch.index.analysis;


import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.math.NumberUtils;
import  org.apache.lucene.analysis.Tokenizer;
import  org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.elasticsearch.common.lang3.StringUtils;

import com.google.i18n.phonenumbers.NumberParseException;
import com.google.i18n.phonenumbers.PhoneNumberUtil;
import com.google.i18n.phonenumbers.Phonenumber.PhoneNumber;

import  java.io.IOException;
import  java.io.Reader;
import java.util.ArrayList;
import java.util.List;

public class PhoneTokenizer extends Tokenizer {

	// The raw input
	private String stringToTokenize = null;

	// Position in the tokens array. We build all the tokens and return them one at a time as incrementToken gets called. 
	private int position = 0;

	/**
	 * The tokens are determined on the first iteration and then returned one at a time
	 * thereafter.
	 */

	private List<String> tokens = null;

	// The base class grabs the charTermAttribute each time incrementToken returns
	protected CharTermAttribute charTermAttribute = addAttribute(CharTermAttribute.class);

	public PhoneTokenizer(Reader reader) {
		super(reader);
	}

	@Override
	public final boolean incrementToken() throws IOException {
		// Clear anything that is already saved in this.charTermAttribute
		this.charTermAttribute.setEmpty();

		if(tokens == null) {
			// It's the 1st iteration, chop it up into tokens. 
			generateTokens();
		}

		// Return those tokens
		return returnTokensOneAtATime();
	}

	private boolean returnTokensOneAtATime() {
		// Token have already been generated. Return them 1 at a time
		if(tokens != null) {
			if(this.position == tokens.size()) {
				// No more tokens
				return false;
			}

			// return each token, 1 at a time
			this.charTermAttribute.append(tokens.get(this.position));
			this.position += 1;
			return true;
		}
		return false;
	}

	private void generateTokens() {
		String uri = getStringToTokenize();

		tokens = new ArrayList<String>();
		tokens.add(getStringToTokenize());

		// Rip off the "tel:" or "sip:" prefix
		if (uri.indexOf("tel:") != -1 || uri.indexOf("sip:") != -1) {
			uri = uri.substring(4);
		} else {
			// If it's not formatted at least this correctly then the whole string is 1 token. Sorry, put a tel: or sip: at the beginning so we know how to treat it
			tokens.add(getStringToTokenize());
			return;
		}

		// Drop anything after @. Most likely there's nothing of interest
		String[] parts = StringUtils.split(uri, "@");
		if(parts.length == 0) {
			return;
		}
		
		String number = parts[0];

		// Add a token for the raw unmanipulated address. Note this could be a username (sip) instead of telephone number so take it as is
		tokens.add(number);	

		// Let google's libphone try to parse it
		PhoneNumberUtil phoneUtil = PhoneNumberUtil.getInstance();
		PhoneNumber numberProto = null;
		String countryCode = null;
		try{
			// ZZ is the generic "I don't know the country code" region. Google's libphone library will try to infer it.
			numberProto = phoneUtil.parse(number, "ZZ");
			if(numberProto != null) {
				// Libphone likes it! 
				countryCode = String.valueOf(numberProto.getCountryCode());
				number = String.valueOf(numberProto.getNationalNumber());

				// Add Country code, extension, and the number as tokens
				tokens.add(countryCode);
				if(!StringUtils.isEmpty(numberProto.getExtension())) {
					tokens.add(numberProto.getExtension());	
				}

				tokens.add(number);
			} 
		} catch(NumberParseException e) {
			// Libphone didn't like it, no biggie. We'll just ngram the number as it is. 
		} catch(StringIndexOutOfBoundsException e) {
		    // Libphone didn't like it, no biggie. We'll just ngram the number as it is.
		}

		// ngram the phone number EG 19198243333 produces 9, 91, 919, etc 
		if(NumberUtils.isNumber(number)) {
			for(int count = 1; count <= number.length(); count++) {
				String token = number.substring(0, count);
				tokens.add(token);
				if(countryCode != null) {
{'fraction_non_alphanumeric': 0.10393566467649568, 'fraction_numerical': 0.009747776288534179, 'mean_word_length': 3.7417677642980935, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3956535', 'n_tokens_mistral': 2747, 'n_tokens_neox': 2371, 'n_words': 790}