<commit_before>package org.xmlcml.ami2.plugins.word;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.Before;
import org.junit.Test;

public class TesseractTest {

	
	private static final Logger LOG = Logger.getLogger(TesseractTest.class);
	static {
		LOG.setLevel(Level.DEBUG);
	}
	
	private static final File TESSERACTED_DIR = new File("src/text/resources/org/xmlcml/ami2/word/peterijsem/tesseracted/");
	private static final File TESS_CLEAN_DIR = new File("examples/peterijsem/tessclean/");
//	private static final Pattern SPEC_STRAIN_ACCESS = Pattern.compile("([A-Z](?:\\.|[a-z]+)\\s*(.*)\\s*\\(([A-Z]{1,2}\\d{1,6})\\)\\s*");
	private static final Pattern SPEC_STRAIN_ACCESS = Pattern.compile("([A-Z](?:\\.|[a-z]+)\\s+[a-z]+)\\s*(.*)\\s+\\(([A-Z]{1,2}\\d{5,6})\\)\\s*");
	private List<List<String>> linesList;
	private List<File> txtFiles;
	

	@Before
	public void setup() throws IOException {
		if (linesList == null) {
			txtFiles = new ArrayList<File>(FileUtils.listFiles(TESSERACTED_DIR, new String[]{"txt"}, false));
			linesList = new ArrayList<List<String>>();
			for (File txtFile : txtFiles) {
				List<String> lines = FileUtils.readLines(txtFile, "iso-8859-1");
				lines = clean(lines);
				linesList.add(lines);
				String filename = FilenameUtils.getBaseName(txtFile.toString());
				File newFile = new File(TESS_CLEAN_DIR, filename+".txt");
				FileUtils.writeLines(newFile, lines);
			}
		}
	}

	private List<String> clean(List<String> lines) {
		List<String> newLines = new ArrayList<String>();
		for (String line : lines) {
			// remove leading numbers
			line = line.replaceAll("^[\\*\\-\\d\\.]+\\s*", "");
			// substitute slashes by "l"<commit_msg>fixed filename bug<commit_after>package org.xmlcml.ami2.plugins.word;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.Before;
import org.junit.Test;

public class TesseractTest {

	
	private static final Logger LOG = Logger.getLogger(TesseractTest.class);
	static {
		LOG.setLevel(Level.DEBUG);
	}
	
	private static final File TESSERACTED_DIR = new File("src/test/resources/org/xmlcml/ami2/word/peterijsem/tesseracted/");
	private static final File TESS_CLEAN_DIR = new File("examples/peterijsem/tessclean/");
//	private static final Pattern SPEC_STRAIN_ACCESS = Pattern.compile("([A-Z](?:\\.|[a-z]+)\\s*(.*)\\s*\\(([A-Z]{1,2}\\d{1,6})\\)\\s*");
	private static final Pattern SPEC_STRAIN_ACCESS = Pattern.compile("([A-Z](?:\\.|[a-z]+)\\s+[a-z]+)\\s*(.*)\\s+\\(([A-Z]{1,2}\\d{5,6})\\)\\s*");
	private List<List<String>> linesList;
	private List<File> txtFiles;
	

	@Before
	public void setup() throws IOException {
		if (linesList == null) {
			
			txtFiles = new ArrayList<File>(FileUtils.listFiles(TESSERACTED_DIR, new String[]{"txt"}, false));
			linesList = new ArrayList<List<String>>();
			for (File txtFile : txtFiles) {
				List<String> lines = FileUtils.readLines(txtFile, "iso-8859-1");
				lines = clean(lines);
				linesList.add(lines);
				String filename = FilenameUtils.getBaseName(txtFile.toString());
				File newFile = new File(TESS_CLEAN_DIR, filename+".txt");
				FileUtils.writeLines(newFile, lines);
			}
		}
	}

	private List<String> clean(List<String> lines) {
		List<String> newLines = new ArrayList<String>();
		for (String line : lines) {
			// remove leading numbers
			line = line.replaceAll("^[\\*\\-\\d\\.]+\\s*", "");
			// substitute slashes by "l"
{'fraction_non_alphanumeric': 0.18084291187739462, 'fraction_numerical': 0.008684546615581098, 'mean_word_length': 4.3717421124828535, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3018426', 'n_tokens_mistral': 1471, 'n_tokens_neox': 1365, 'n_words': 259}