<commit_before>package congress;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;

import model.*;

import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.apache.poi.xwpf.usermodel.XWPFRun;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;

public class ParseCongress {

	static ArrayList<Congress> notes = new ArrayList<Congress>();

	static Congress currentCongress;
	static Session currentSession;
	static State currentState;
	static Chamber currentChamber;

	static boolean nextLineCongressDate = false;
	static boolean nextLineSessionDate = false;

	static String currentString = null;
	static String lastString = null;


	static ArrayList<String> states = new ArrayList<String>();

	public static void main(String... args) throws Exception {
		XWPFDocument document = new XWPFDocument(new FileInputStream("c:/users/karl/downloads/GPO-CDOC-108hdoc222-3.docx"));

		StringBuilder builder = new StringBuilder();

		addStates();
		
		
		int line = 1;
		BufferedWriter  debugFile = new BufferedWriter( 
				new OutputStreamWriter( 
					new FileOutputStream("c:/users/karl/downloads/debugfile.txt"), Charset.forName("UTF-8") ) );
		Iterator<XWPFParagraph> i = document.getParagraphsIterator();
		while(i.hasNext()) {
			XWPFParagraph paragraph = i.next();

			// Do the paragraph text
			for(XWPFRun run : paragraph.getRuns()) {
				String str = run.toString();
				if ( str.length() == 0 ) continue;
				builder.append(str);
				if ( !str.endsWith(",") && !str.endsWith(", ") && !str.equals(" ")  ) {
					lastString = currentString;
					currentString = builder.toString();
					String debugString = currentString;
					if ( !searchCurrents(currentString) ) {
						if ( !allUpper(str)) {
							debugFile.write(line + ": " + debugString);
							debugFile.newLine();
							line++;
							builder = new StringBuilder();							
						}
					} else {
						debugFile.write(line + ": " + debugString);
						debugFile.newLine();
						line++;
						builder = new StringBuilder();
					}
				}
			}

			// Do endnotes and footnotes
			String footnameText = paragraph.getFootnoteText();
			if(footnameText != null && footnameText.length() > 0) {
//				note.footnote = footnameText;
//				String[] splits = footnameText.split("\\[[0-9][0-9][0-9][0-9]:");
				String[] splits = footnameText.split("\\[?[0-9]*:");
				
				if ( splits.length > 2 ) {
//					System.out.println("** DOUBLE **" + splits.length +":" + lastString);
					int start0 = 0;
					int start1 = footnameText.indexOf( splits[splits.length-2] )
							+ splits[splits.length-2].length();

					Note note = new Note(
						lastString, 							
						footnameText.substring(
							start0, start1
						), 
						line
					);
					addNote( note);
					note = new Note(
						currentString, 
						footnameText.substring(
							start1
						), 
						line
					);
					addNote( note);
				} else {
					Note note = new Note(
						currentString, 
						footnameText, 
						line
					);
					addNote( note);
				}
			}

		}
	
	    BufferedWriter bWriter = new BufferedWriter( new OutputStreamWriter(new FileOutputStream("c:/users/karl/downloads/congress.json"), Charset.forName("UTF-8")));
		// write out json
		ObjectMapper mapper = new ObjectMapper();
		ObjectWriter writer = mapper.writer().withDefaultPrettyPrinter();
		bWriter.write( writer.writeValueAsString(notes) );
//		mapper.writeValue(new File("c:/users/karl/downloads/congress.json"), notes);
		debugFile.close();
	}
	
	private static void addNote(Note note) {
		Congress lCongress = null;
		if ( notes.size() != 0 ) lCongress = notes.get(notes.size()-1);
		if ( lCongress != currentCongress || lCongress == null) {
			currentCongress.sessions = new ArrayList<Session>();
			lCongress = currentCongress;  <commit_msg>Close output file, and flush.<commit_after>package congress;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;

import model.*;

import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.apache.poi.xwpf.usermodel.XWPFRun;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;

public class ParseCongress {

	static ArrayList<Congress> notes = new ArrayList<Congress>();

	static Congress currentCongress;
	static Session currentSession;
	static State currentState;
	static Chamber currentChamber;

	static boolean nextLineCongressDate = false;
	static boolean nextLineSessionDate = false;

	static String currentString = null;
	static String lastString = null;


	static ArrayList<String> states = new ArrayList<String>();

	public static void main(String... args) throws Exception {
		XWPFDocument document = new XWPFDocument(new FileInputStream("c:/users/karl/downloads/GPO-CDOC-108hdoc222-3.docx"));

		StringBuilder builder = new StringBuilder();

		addStates();
		
		
		int line = 1;
		BufferedWriter  debugFile = new BufferedWriter( 
				new OutputStreamWriter( 
					new FileOutputStream("c:/users/karl/downloads/debugfile.txt"), Charset.forName("UTF-8") ) );
		Iterator<XWPFParagraph> i = document.getParagraphsIterator();
		while(i.hasNext()) {
			XWPFParagraph paragraph = i.next();

			// Do the paragraph text
			for(XWPFRun run : paragraph.getRuns()) {
				String str = run.toString();
				if ( str.length() == 0 ) continue;
				builder.append(str);
				if ( !str.endsWith(",") && !str.endsWith(", ") && !str.equals(" ")  ) {
					lastString = currentString;
					currentString = builder.toString();
					String debugString = currentString;
					if ( !searchCurrents(currentString) ) {
						if ( !allUpper(str)) {
							debugFile.write(line + ": " + debugString);
							debugFile.newLine();
							line++;
							builder = new StringBuilder();							
						}
					} else {
						debugFile.write(line + ": " + debugString);
						debugFile.newLine();
						line++;
						builder = new StringBuilder();
					}
				}
			}

			// Do endnotes and footnotes
			String footnameText = paragraph.getFootnoteText();
			if(footnameText != null && footnameText.length() > 0) {
//				note.footnote = footnameText;
//				String[] splits = footnameText.split("\\[[0-9][0-9][0-9][0-9]:");
				String[] splits = footnameText.split("\\[?[0-9]*:");
				
				if ( splits.length > 2 ) {
//					System.out.println("** DOUBLE **" + splits.length +":" + lastString);
					int start0 = 0;
					int start1 = footnameText.indexOf( splits[splits.length-2] )
							+ splits[splits.length-2].length();

					Note note = new Note(
						lastString, 							
						footnameText.substring(
							start0, start1
						), 
						line
					);
					addNote( note);
					note = new Note(
						currentString, 
						footnameText.substring(
							start1
						), 
						line
					);
					addNote( note);
				} else {
					Note note = new Note(
						currentString, 
						footnameText, 
						line
					);
					addNote( note);
				}
			}

		}
	
	    BufferedWriter bWriter = new BufferedWriter( new OutputStreamWriter(new FileOutputStream("c:/users/karl/downloads/congress.json"), Charset.forName("UTF-8")));
		// write out json
		ObjectMapper mapper = new ObjectMapper();
		ObjectWriter writer = mapper.writer().withDefaultPrettyPrinter();
		bWriter.write( writer.writeValueAsString(notes) );
//		mapper.writeValue(new File("c:/users/karl/downloads/congress.json"), notes);
		bWriter.close();
		debugFile.close();
	}
	
	private static void addNote(Note note) {
		Congress lCongress = null;
		if ( notes.size() != 0 ) lCongress = notes.get(notes.size()-1);
		if ( lCongress != currentCongress || lCongress == null) {
			currentCongress.sessions = new ArrayList<Session>();
			lCongress = currentCongress;  
{'fraction_non_alphanumeric': 0.13267398383789653, 'fraction_numerical': 0.00796043903027379, 'mean_word_length': 3.3232533889468194, 'pattern_counts': {'":': 6, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1999380', 'n_tokens_mistral': 3282, 'n_tokens_neox': 2548, 'n_words': 543}