<commit_before>package com.github.ktrnka.droidling;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;

/**
 * Code to load a data file and identify the language of
 * a piece of text.
 * @author keith.trnka
 *
 */
public class LanguageIdentifier
	{
	private ArrayList<LIDModel> models;
	
	private class LIDModel
		{
		/**
		 * Note: Each String has only one character, but it's a String
		 * because the memory overhead isn't much different than Character
		 * and Character would require an additional conversion.
		 */
		private HashSet<String> chars;
		private HashSet<String> discriminativeChars;
		private HashSet<String> charPairs;
		private HashSet<String> words;
		private HashSet<String> discriminativeWords;
		
		String languageName;
		String languageCode2;
		
		LIDModel(BufferedReader in) throws IOException
			{
			languageCode2 = in.readLine();
			languageName = in.readLine();

			words = new HashSet<String>();
			String[] tokens = in.readLine().split(" ");
			for (String token : tokens)
				words.add(token);
			<commit_msg>Filling out the language ID code.<commit_after>package com.github.ktrnka.droidling;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;

/**
 * Code to load a data file and identify the language of
 * a unigram model.
 * @author keith.trnka
 *
 */
public class LanguageIdentifier
	{
	private ArrayList<LIDModel> models;
	
	public LanguageIdentifier(InputStream in) throws IOException
		{
		BufferedReader textIn = new BufferedReader(new InputStreamReader(in, "UTF-8"));
		
		models = new ArrayList<LIDModel>();
		
		while (true)
			{
			try
				{
				LIDModel model = new LIDModel(textIn);
				models.add(model);
				}
			catch (IOException e)
				{
				// This code is terribly ugly; the LIDModel constructor should really be replaced by a factory method
				break;
				}
			}
		textIn.close();
		}
	
	public Identification identify(HashMap<String,int[]> unigrams)
	{
		Identification ident = new Identification();
		
		for (LIDModel model : models)
			ident.scores.put(model, new double[] { model.score(unigrams) });
		
		return ident;
	}
	
	public class Identification
		{
		public HashMap<LIDModel,double[]> scores;
		
		Identification()
			{
			scores = new HashMap<LIDModel,double[]>();
			}
		
		public String findBest()
			{
			LIDModel best = null;
			for (LIDModel model : scores.keySet())
				{
				if (best == null || scores.get(model)[0] > scores.get(best)[0])
					best = model;
				}
			
			if (best != null)
				return best.languageName;
			
			return "unknown";
			}
		}
	
	public class LIDModel
		{
		/**
		 * Note: Each String has only one character, but it's a String
		 * because the memory overhead isn't much different than Character
		 * and Character would require an additional conversion.
		 */
		private HashSet<String> chars;
		private HashSet<String> discriminativeChars;
		private HashSet<String> charPairs;
		private HashSet<String> words;
		private HashSet<String> discriminativeWords;
		
		public final String languageName;
		public final String languageCode2;
		
		LIDModel(BufferedReader in) throws IOException
			{
			languageCode2 = in.readLine();
			languageName = in.readLine();

			words = new HashSet<String>();
			String[] tokens = in.readLine().split(" ");
			for (String token : tokens)
				words.add(token);
			
{'fraction_non_alphanumeric': 0.10153846153846154, 'fraction_numerical': 0.001958041958041958, 'mean_word_length': 3.6381322957198443, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1213739', 'n_tokens_mistral': 1268, 'n_tokens_neox': 1049, 'n_words': 315}