<commit_before>package net.cherokeedictionary.lyx;

import org.apache.commons.lang3.StringUtils;

public class EnglishCherokee implements Comparable<EnglishCherokee>{
	private String english;
	public EnglishCherokee() {
	}
	public EnglishCherokee(EnglishCherokee ec) {
		this.english=ec.english;
		this.pronounce=ec.pronounce;
		this.syllabary=ec.syllabary;
		this.toLabel=ec.toLabel;
	}
	public void setEnglish(String english) {
		this.english = english;
	}
	public String getDefinition() {
		String eng=english;
		if (eng.endsWith(".")) {
			eng=StringUtils.left(eng, eng.length()-1);
		}
		eng = transform(eng);		
		if (eng.startsWith("(")) {
			String sub = StringUtils.substringBetween(eng, "(", ")");
			eng = StringUtils.substringAfter(eng, "("+sub+")");
			eng += " "+"("+sub+")";
		}
		eng=transform(eng);
		if (eng.endsWith(",")) {
			eng=StringUtils.left(eng, eng.length()-1);
		}		
		return eng;
	}
	public String syllabary;
	public String pronounce;
	public int toLabel;

	public String getLyxCode(boolean bold) {
		StringBuilder sb = new StringBuilder();
		String eng = StringUtils.strip(getDefinition().replace("\\n", " "));		
		if (bold) {
			sb.append("\\begin_layout Standard\n");
			sb.append("\\series bold\n");
			sb.append(eng);
			sb.append("\n");
			sb.append("\\series default\n");
		} else {
			sb.append("\\begin_layout Standard\n");
			sb.append(eng);
		}		
		
		sb.append(": ");
		sb.append(syllabary);			
		sb.append(" [");
		sb.append(pronounce);
		sb.append("]");
		sb.append(" (page ");
		sb.append("\\begin_inset CommandInset ref\n" + 
				"LatexCommand pageref\n" + 
				"reference \"");
		sb.append(""+toLabel);
		sb.append("\"\n" + 
				"\\end_inset\n"); 
		sb.append(")\n");
		sb.append("\\end_layout\n\n");		
		return sb.toString();
	}

	private String transform(String eng) {		
		eng = StringUtils.strip(eng);
		String lc = eng.toLowerCase();
		if (lc.startsWith("n.")) {
			eng = StringUtils.substring(eng, 2);
			eng = StringUtils.strip(eng);
			lc = eng.toLowerCase();
		}
		if (lc.startsWith("v. t.")) {
			eng = StringUtils.substring(eng, 5);
			eng = StringUtils.strip(eng);
			lc = eng.toLowerCase();
		}
		if (lc.startsWith("v.t.")) {
			eng = StringUtils.substring(eng, 4);
			eng = StringUtils.strip(eng);
			lc = eng.toLowerCase();
		}
		if (lc.startsWith("v. i.")) {
			eng = StringUtils.substring(eng, 5);
			eng = StringUtils.strip(eng);
			lc = eng.toLowerCase();
		}
		if (lc.startsWith("v.i.")) {
			eng = StringUtils.substring(eng, 4);
			eng = StringUtils.strip(eng);
			lc = eng.toLowerCase();
		}
		if (lc.startsWith("adv.")) {
			eng = StringUtils.substring(eng, 4);
			eng = StringUtils.strip(eng);
			lc = eng.toLowerCase();
		}
		if (lc.startsWith("adj.")) {
			eng = StringUtils.substring(eng, 4);
			eng = StringUtils.strip(eng);
			lc = eng.toLowerCase();
		}
		if (lc.contains(".") && lc.indexOf(".")<4 && !lc.startsWith("1")) {
			System.err.println("WARNING: BAD DEFINITION! => "+eng);
		}
		if (lc.startsWith("becoming ")) {
			eng = StringUtils.substring(eng, 9)+" (becoming)";
			lc = eng.toLowerCase();
		}
		chopper: {			
			if (lc.startsWith("at the ")) {
				eng = StringUtils.substring(eng, 7)+" (at the)";
				break chopper;
			}
			if (lc.startsWith("at a ")) {
				eng = StringUtils.substring(eng, 5)+" (at a)";
				break chopper;
			}
			if (lc.startsWith("at ")) {
				eng = StringUtils.substring(eng, 3)+" (at)";
				break chopper;
			}
			if (lc.startsWith("in the ")) {
				eng = StringUtils.substring(eng, 7)+" (in the)";
				break chopper;
			}
			if (lc.startsWith("in a ")) {<commit_msg>more definition munging ...<commit_after>package net.cherokeedictionary.lyx;

import org.apache.commons.lang3.StringUtils;

public class EnglishCherokee implements Comparable<EnglishCherokee>{
	private String english;
	public EnglishCherokee() {
	}
	public EnglishCherokee(EnglishCherokee ec) {
		this.english=ec.english;
		this.pronounce=ec.pronounce;
		this.syllabary=ec.syllabary;
		this.toLabel=ec.toLabel;
	}
	public void setEnglish(String english) {
		this.english = english;
	}
	public String getDefinition() {
		String eng=english;
		if (eng.endsWith(".")) {
			eng=StringUtils.left(eng, eng.length()-1);
		}
		eng = transform(eng);		
		if (eng.startsWith("(")) {
			String sub = StringUtils.substringBetween(eng, "(", ")");
			eng = StringUtils.substringAfter(eng, "("+sub+")");
			eng += " "+"("+sub+")";
		}
		eng=transform(eng);
		if (eng.endsWith(",")) {
			eng=StringUtils.left(eng, eng.length()-1);
		}		
		return eng;
	}
	public String syllabary;
	public String pronounce;
	public int toLabel;

	public String getLyxCode(boolean bold) {
		StringBuilder sb = new StringBuilder();
		String eng = StringUtils.strip(getDefinition().replace("\\n", " "));		
		if (bold) {
			sb.append("\\begin_layout Standard\n");
			sb.append("\\series bold\n");
			sb.append(eng);
			sb.append("\n");
			sb.append("\\series default\n");
		} else {
			sb.append("\\begin_layout Standard\n");
			sb.append(eng);
		}		
		
		sb.append(": ");
		sb.append(syllabary);			
		sb.append(" [");
		sb.append(pronounce);
		sb.append("]");
		sb.append(" (page ");
		sb.append("\\begin_inset CommandInset ref\n" + 
				"LatexCommand pageref\n" + 
				"reference \"");
		sb.append(""+toLabel);
		sb.append("\"\n" + 
				"\\end_inset\n"); 
		sb.append(")\n");
		sb.append("\\end_layout\n\n");		
		return sb.toString();
	}

	private String transform(String eng) {		
		eng = StringUtils.strip(eng);
		String lc = eng.toLowerCase();
		if (lc.startsWith("n.")) {
			eng = StringUtils.substring(eng, 2);
			eng = StringUtils.strip(eng);
			lc = eng.toLowerCase();
		}
		if (lc.startsWith("v. t.")) {
			eng = StringUtils.substring(eng, 5);
			eng = StringUtils.strip(eng);
			lc = eng.toLowerCase();
		}
		if (lc.startsWith("v.t.")) {
			eng = StringUtils.substring(eng, 4);
			eng = StringUtils.strip(eng);
			lc = eng.toLowerCase();
		}
		if (lc.startsWith("v. i.")) {
			eng = StringUtils.substring(eng, 5);
			eng = StringUtils.strip(eng);
			lc = eng.toLowerCase();
		}
		if (lc.startsWith("v.i.")) {
			eng = StringUtils.substring(eng, 4);
			eng = StringUtils.strip(eng);
			lc = eng.toLowerCase();
		}
		if (lc.startsWith("adv.")) {
			eng = StringUtils.substring(eng, 4);
			eng = StringUtils.strip(eng);
			lc = eng.toLowerCase();
		}
		if (lc.startsWith("adj.")) {
			eng = StringUtils.substring(eng, 4);
			eng = StringUtils.strip(eng);
			lc = eng.toLowerCase();
		}
		if (lc.contains(".") && lc.indexOf(".")<4 && !lc.startsWith("1")) {
			System.err.println("WARNING: BAD DEFINITION! => "+eng);
		}
		if (lc.startsWith("becoming ")) {
			eng = StringUtils.substring(eng, 9)+" (becoming)";
			lc = eng.toLowerCase();
		}
		chopper: {
			if (lc.startsWith("they're ")) {
				eng = StringUtils.substring(eng, 8)+" (they are)";
				break chopper;
			}
			if (lc.startsWith("they are ")) {
				eng = StringUtils.substring(eng, 9)+" (they are)";
				break chopper;
			}
			if (lc.startsWith("at the ")) {
				eng = StringUtils.substring(eng, 7)+" (at the)";
				break chopper;
			}
			if (lc.startsWith("at a ")) {
				eng = StringUtils.substring(eng, 5)+" (at a)";
				break chopper;
			}
			if (lc.startsWith("at ")) {
				eng = StringUtils.substring(eng, 3)+" (at)";
				break chopper;
			}
			if (lc.startsWith("in the ")) {
				eng = StringUtils.substring(eng, 7)+" (in the)";
				break chopper;
			}
			if (lc.startsWith("in a ")) {
{'fraction_non_alphanumeric': 0.19329548526628818, 'fraction_numerical': 0.004866180048661801, 'mean_word_length': 3.3806986382474835, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6200578', 'n_tokens_mistral': 3174, 'n_tokens_neox': 2768, 'n_words': 504}