<commit_before>package org.uom.raavana.ananya.service.endpoints.gazetteer;

import java.util.*;

import ananya.utils.gazetteer.Gazetteer;
import com.google.gson.Gson;

import corpus.sinhala.SinhalaTokenizer;
import corpus.sinhala.SinhalaVowelLetterFixer;
import org.uom.raavana.ananya.service.endpoints.gazetteer.EntityObject;


public class AnnotationService {
	
	private static String fixedText;
	private static HashMap<String, String> Gazzatters = new HashMap<String, String>();
	private static Map<String, EntityObject> entityObjectMap = new HashMap<String, EntityObject>();

	public static ArrayList<int[]> getCharacterOffsetArray(String checkText){

		int Stringlength = checkText.length();

		ArrayList<int[]> offsetList = new ArrayList<int[]>();

		int [] offsetPair ;

		for (int i = -1; (i = fixedText.indexOf(checkText, i + 1)) != -1; ) {
			offsetPair = new int[2];
			offsetPair[0] = i;
			offsetPair[1] = i+Stringlength;
			offsetList.add(offsetPair);
		}

		return offsetList;
	}
	
	public static LinkedList<String> tokenizeText(){
		
		SinhalaTokenizer tokenizer = new SinhalaTokenizer();
	    LinkedList<String> tokenizedwords = tokenizer.splitWords(fixedText);
	    
	    return tokenizedwords;		
			
	}
	
	public static void cleanText(String originText){
		
		SinhalaVowelLetterFixer vowelFixer = new SinhalaVowelLetterFixer();
	    fixedText = vowelFixer.fixText(originText, false);
			
	}
	
	public String getFixedText(){
		return fixedText;
	}
	
	public static void generateEtities(){

//		Gazetteer gazetteer = Gazetteer.getInstance("");


		LinkedList<String> tokenizedWords = tokenizeText();
		ArrayList<int[]> characterOffsetArray = new ArrayList<int[]>();
		String currentIteration;
		String tag;
		String [] arrayOfTexts = new String[1];
		int entityCount = 0;
		
		ListIterator<String> listIterator = tokenizedWords.listIterator();
        while (listIterator.hasNext()) {
        	currentIteration = listIterator.next();
        	if(Gazzatters.containsKey(currentIteration)){            	
        		        		
        		characterOffsetArray = getCharacterOffsetArray(currentIteration);

//				tag= gazetteer.findNamedEntityTag("අතපත්තු");
//				EntityObject entity = new EntityObject(tag, currentIteration, charOffset);

				arrayOfTexts[0] = currentIteration;
        		EntityObject entity = new EntityObject(Gazzatters.get(currentIteration), arrayOfTexts, characterOffsetArray);<commit_msg>Text error fix<commit_after>package org.uom.raavana.ananya.service.endpoints.gazetteer;

import java.util.*;

import ananya.utils.gazetteer.Gazetteer;
import com.google.gson.Gson;

import corpus.sinhala.SinhalaTokenizer;
import corpus.sinhala.SinhalaVowelLetterFixer;
import org.uom.raavana.ananya.service.endpoints.gazetteer.EntityObject;


public class AnnotationService {
	
	private static String fixedText;
	private static HashMap<String, String> Gazzatters = new HashMap<String, String>();
	private static Map<String, EntityObject> entityObjectMap = new HashMap<String, EntityObject>();

	public static ArrayList<int[]> getCharacterOffsetArray(String checkText){

		int Stringlength = checkText.length();

		ArrayList<int[]> offsetList = new ArrayList<int[]>();

		int [] offsetPair ;

		for (int i = -1; (i = fixedText.indexOf(checkText, i + 1)) != -1; ) {
			offsetPair = new int[2];
			offsetPair[0] = i;
			offsetPair[1] = i+Stringlength;
			offsetList.add(offsetPair);
		}

		return offsetList;
	}
	
	public static LinkedList<String> tokenizeText(){
		
		SinhalaTokenizer tokenizer = new SinhalaTokenizer();
	    LinkedList<String> tokenizedwords = tokenizer.splitWords(fixedText);
	    
	    return tokenizedwords;		
			
	}
	
	public static void cleanText(String originText){
		
		SinhalaVowelLetterFixer vowelFixer = new SinhalaVowelLetterFixer();
	    fixedText = vowelFixer.fixText(originText, false);
			
	}
	
	public String getFixedText(){
		return fixedText;
	}
	
	public static void generateEtities(){

//		Gazetteer gazetteer = Gazetteer.getInstance("");


		LinkedList<String> tokenizedWords = tokenizeText();
		ArrayList<int[]> characterOffsetArray = new ArrayList<int[]>();
		String currentIteration;
		String tag;
		String [] arrayOfTexts;
		int entityCount = 0;
		
		ListIterator<String> listIterator = tokenizedWords.listIterator();
        while (listIterator.hasNext()) {
        	currentIteration = listIterator.next();
        	if(Gazzatters.containsKey(currentIteration)){            	
        		        		
        		characterOffsetArray = getCharacterOffsetArray(currentIteration);

//				tag= gazetteer.findNamedEntityTag("අතපත්තු");
//				EntityObject entity = new EntityObject(tag, currentIteration, charOffset);
				arrayOfTexts = new String[1];
				arrayOfTexts[0] = currentIteration;
        		EntityObject entity = new EntityObject(Gazzatters.get(currentIteration), arrayOfTexts, characterOffsetArray);
{'fraction_non_alphanumeric': 0.10715754833401893, 'fraction_numerical': 0.003702180172768408, 'mean_word_length': 4.373480662983425, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5137387', 'n_tokens_mistral': 1686, 'n_tokens_neox': 1511, 'n_words': 304}