<commit_before>package de.netpioneer.socialmedia.wordprocessing.service;

import java.text.BreakIterator;
import java.util.ArrayList;
import java.util.List;

import org.springframework.stereotype.Service;

@Service
public class WordSplittingService {

	public List<String> getWordsFromText(String text) {
		List<String> words = new ArrayList<>();
		BreakIterator breakIterator = BreakIterator.getWordInstance();
		breakIterator.setText(text);
		int start = breakIterator.first();
		for (int end = breakIterator.next(); end != BreakIterator.DONE; start = end, end = breakIterator
				.next()) {
			String word = text.substring(start, end).toLowerCase();
			if (Character.isAlphabetic(word.codePointAt(0))) {
				words.add(word);
			}
		}
		return words;
	}
	
}
<commit_msg>Don't depend on system's default locale.<commit_after>package de.netpioneer.socialmedia.wordprocessing.service;

import java.text.BreakIterator;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;

import org.springframework.stereotype.Service;

@Service
public class WordSplittingService {

	public List<String> getWordsFromText(String text) {
		List<String> words = new ArrayList<>();
		BreakIterator breakIterator = BreakIterator.getWordInstance(Locale.ENGLISH);
		breakIterator.setText(text);
		int start = breakIterator.first();
		for (int end = breakIterator.next(); end != BreakIterator.DONE; start = end, end = breakIterator
				.next()) {
			String word = text.substring(start, end).toLowerCase();
			if (Character.isAlphabetic(word.codePointAt(0))) {
				words.add(word);
			}
		}
		return words;
	}
	
}

{'fraction_non_alphanumeric': 0.12236024844720497, 'fraction_numerical': 0.0012422360248447205, 'mean_word_length': 5.0111940298507465, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5456096', 'n_tokens_mistral': 532, 'n_tokens_neox': 485, 'n_words': 105}