<commit_before>package yuku.alkitabconverter.util;

import java.io.PrintStream;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;

import yuku.alkitab.base.model.Ari;



public class TextDb {
	public static final String TAG = TextDb.class.getSimpleName();
	
	public interface TextProcessor {
		void process(int ari, VerseState verseState);
	}
	
	public static class VerseState {
		// was: "public int menjorok;" but no longer used
		public String text;
	}
	
	TreeMap<Integer, VerseState> map = new TreeMap<Integer, VerseState>();
	
	public String append(int bookId, int chapter_1, int verse_1, String s, int currentIndent) {
		return append(Ari.encode(bookId, chapter_1, verse_1), s, currentIndent);
	}
	
	public String append(int bookId, int chapter_1, int verse_1, String s, int currentIndent, String separatorWhenExisting) {
		return append(Ari.encode(bookId, chapter_1, verse_1), s, currentIndent, separatorWhenExisting);
	}
	
	/**
	 * @param currentIndent if -1, don't write anything. 
	 * @return
	 */
	public String append(int ari, String s, int currentIndent) {
		return append(ari, s, currentIndent, null);
	}

	/**
	 * @param separatorWhenExisting if the text is appended to an ari that has already some text, append this first, then the text. 
	 * @param currentIndent if -1, don't write anything. 
	 * @return
	 */
	public String append(int ari, String text, int currentIndent, String separatorWhenExisting) {
		VerseState as = map.get(ari);
		boolean isNew = false;
		if (as == null) {
			as = new VerseState();
			as.text = "";
			map.put(ari, as);
			isNew = true;
		}
		
		boolean writtenParaMarker = false;
		
		if (currentIndent != -1) {
			if (currentIndent == -2) {
				as.text += "@^";
			} else if (currentIndent < 0 || currentIndent > 4) {
				throw new RuntimeException("menjorok ngaco: " + currentIndent);
			} else {
				as.text += "@" + String.valueOf(currentIndent);
			}
			
			writtenParaMarker = true;
			
			// was: "update menjoroknya ayatstate" but no longer used
//			for (int i = 0; i < as.isi.length(); i++) {
//				if (as.isi.charAt(i) == '@' && as.isi.charAt(i+1) >= '0' && as.isi.charAt(i+1) <= '4') {
//					as.menjorok = as.isi.charAt(i+1) - '0';
//				}
//			}
		}
		
		if (!isNew) {<commit_msg>Fix bug adding string "null" because of the separator<commit_after>package yuku.alkitabconverter.util;

import java.io.PrintStream;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;

import yuku.alkitab.base.model.Ari;



public class TextDb {
	public static final String TAG = TextDb.class.getSimpleName();
	
	public interface TextProcessor {
		void process(int ari, VerseState verseState);
	}
	
	public static class VerseState {
		// was: "public int menjorok;" but no longer used
		public String text;
	}
	
	TreeMap<Integer, VerseState> map = new TreeMap<Integer, VerseState>();
	
	public String append(int bookId, int chapter_1, int verse_1, String s, int currentIndent) {
		return append(Ari.encode(bookId, chapter_1, verse_1), s, currentIndent);
	}
	
	public String append(int bookId, int chapter_1, int verse_1, String s, int currentIndent, String separatorWhenExisting) {
		return append(Ari.encode(bookId, chapter_1, verse_1), s, currentIndent, separatorWhenExisting);
	}
	
	/**
	 * @param currentIndent if -1, don't write anything. 
	 * @return
	 */
	public String append(int ari, String s, int currentIndent) {
		return append(ari, s, currentIndent, null);
	}

	/**
	 * @param separatorWhenExisting if the text is appended to an ari that has already some text, append this first, then the text. 
	 * @param currentIndent if -1, don't write anything. 
	 * @return
	 */
	public String append(int ari, String text, int currentIndent, String separatorWhenExisting) {
		VerseState as = map.get(ari);
		boolean isNew = false;
		if (as == null) {
			as = new VerseState();
			as.text = "";
			map.put(ari, as);
			isNew = true;
		}
		
		boolean writtenParaMarker = false;
		
		if (currentIndent != -1) {
			if (currentIndent == -2) {
				as.text += "@^";
			} else if (currentIndent < 0 || currentIndent > 4) {
				throw new RuntimeException("menjorok ngaco: " + currentIndent);
			} else {
				as.text += "@" + String.valueOf(currentIndent);
			}
			
			writtenParaMarker = true;
			
			// was: "update menjoroknya ayatstate" but no longer used
//			for (int i = 0; i < as.isi.length(); i++) {
//				if (as.isi.charAt(i) == '@' && as.isi.charAt(i+1) >= '0' && as.isi.charAt(i+1) <= '4') {
//					as.menjorok = as.isi.charAt(i+1) - '0';
//				}
//			}
		}
		
		if (!isNew && separatorWhenExisting != null) {
{'fraction_non_alphanumeric': 0.1328398108163685, 'fraction_numerical': 0.00863664404688464, 'mean_word_length': 3.5671361502347416, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5626607', 'n_tokens_mistral': 1864, 'n_tokens_neox': 1639, 'n_words': 466}