<commit_before>/*
 * Portions Copyright (c) 2009-10 Zepheira LLC and <NAME>, Some Rights Reserved
 * Portions Copyright (c) 2010-11 Talis Inc, Some Rights Reserved 
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package org.callimachusproject.behaviours;

import java.util.HashSet;
import java.util.Set;

import org.apache.commons.codec.language.Soundex;
import org.callimachusproject.traits.SoundexTrait;
import org.openrdf.repository.object.annotations.triggeredBy;

/**
 * Inserts calli:soundex triples for every label that is inserted and provides
 * methods to help lookup the labels.
 * 
 * @author <NAME>
 * 
 */
public abstract class SoundexLabelSupport implements SoundexTrait {
	private static Soundex soundex = new Soundex();

	public String asSoundex(String label) {
		if (label == null || label.length() == 0)
			return "";
		String ex = soundex.encode(clean(label));
		if (ex == null || ex.length() < 3)
			return "";
		return ex.substring(0, 3) + "0";
	}

	public String regexStartsWith(String string) {
		if (string == null)
			return "^";
		return "^" + string.replaceAll("[^a-zA-Z0-9\\s]", ".");
	}

	@triggeredBy( { "http://www.w3.org/2000/01/rdf-schema#label",
			"http://www.w3.org/2004/02/skos/core#prefLabel",
			"http://www.w3.org/2004/02/skos/core#altLabel",
			"http://www.w3.org/2004/02/skos/core#hiddenLabel" })
	public void addSoundexForLabel(String label) {
		Set<String> phones = new HashSet<String>();
		String previous = null;
		int same = 0;
		for (int i = 1; i <= label.length(); i++) {
			String prefix = label.substring(0, i);
			String phone = asSoundex(prefix);
			if (phone.equals(previous)) {
				same++;
			} else {
				same = 0;
			}
			if (same > 6)
				break;
			previous = phone;
			phones.add(phone);
		}
		getSoundexes().addAll(phones);
	}

	/**
	 * This removes non-english letters.
	 */
	private String clean(String str) {
		if (str == null || str.length() == 0)
			return str;
		int len = str.length();
		char[] chars = new char[len];
		int count = 0;
		for (int i = 0; i < len; i++) {
			char ch = str.charAt(i);
			if ('A' <= ch && ch <= 'Z') {
				chars[count++] = ch;
			} else if ('a' <= ch && ch <= 'z') {
				chars[count++] = Character.toUpperCase(ch);
			}
		}
		if (count == len)
			return str.toUpperCase(java.util.Locale.ENGLISH);
		return new String(chars, 0, count);
	}
}
<commit_msg>Ensure the full soundex is included<commit_after>/*
 * Portions Copyright (c) 2009-10 Zepheira LLC and <NAME>, Some Rights Reserved
 * Portions Copyright (c) 2010-11 Talis Inc, Some Rights Reserved 
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package org.callimachusproject.behaviours;

import java.util.HashSet;
import java.util.Set;

import org.apache.commons.codec.language.Soundex;
import org.callimachusproject.traits.SoundexTrait;
import org.openrdf.repository.object.annotations.triggeredBy;

/**
 * Inserts calli:soundex triples for every label that is inserted and provides
 * methods to help lookup the labels.
 * 
 * @author <NAME>
 * 
 */
public abstract class SoundexLabelSupport implements SoundexTrait {
	private static Soundex soundex = new Soundex();

	public String asSoundex(String label) {
		if (label == null || label.length() == 0)
			return "";
		String ex = soundex.encode(clean(label));
		if (ex == null || ex.length() < 3)
			return "";
		return ex.substring(0, 3) + "0";
	}

	public String regexStartsWith(String string) {
		if (string == null)
			return "^";
		return "^" + string.replaceAll("[^a-zA-Z0-9\\s]", ".");
	}

	@triggeredBy( { "http://www.w3.org/2000/01/rdf-schema#label",
			"http://www.w3.org/2004/02/skos/core#prefLabel",
			"http://www.w3.org/2004/02/skos/core#altLabel",
			"http://www.w3.org/2004/02/skos/core#hiddenLabel" })
	public void addSoundexForLabel(String label) {
		Set<String> phones = new HashSet<String>();
		String clean = clean(label);
		String full = soundex.encode(clean);
		if (full == null || full.length() < 3)
			return;
		full = full.substring(0, 3);
		int same = 0;
		String previous = null;
		for (int i = 1, n = clean.length(); i <= n; i++) {
			String prefix = clean.substring(0, i);
			String phone = soundex.encode(prefix).substring(0, 3);
			if (phone.equals(previous)) {
				same++;
			} else {
				same = 0;
				previous = phone;
				phones.add(phone + '0');
			}
			if (same > 3 && previous.equals(full))
				break;
		}
		getSoundexes().addAll(phones);
	}

	/**
	 * This removes non-english letters.
	 */
	private String clean(String str) {
		if (str == null || str.length() == 0)
			return str;
		int len = str.length();
		char[] chars = new char[len];
		int count = 0;
		for (int i = 0; i < len; i++) {
			char ch = str.charAt(i);
			if ('A' <= ch && ch <= 'Z') {
				chars[count++] = ch;
			} else if ('a' <= ch && ch <= 'z') {
				chars[count++] = Character.toUpperCase(ch);
			}
		}
		if (count == len)
			return str.toUpperCase(java.util.Locale.ENGLISH);
		return new String(chars, 0, count);
	}
}

{'fraction_non_alphanumeric': 0.14728552278820375, 'fraction_numerical': 0.02111260053619303, 'mean_word_length': 3.441220238095238, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 10, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3749494', 'n_tokens_mistral': 2229, 'n_tokens_neox': 1960, 'n_words': 589}