<commit_before>package org.xmlcml.norma.pubstyle.bmc;

import java.awt.Dimension;
import java.io.File;
import java.io.FileOutputStream;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.io.FileUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import org.xmlcml.cmine.files.CMDir;
import org.xmlcml.euclid.Real2;
import org.xmlcml.graphics.svg.SVGElement;
import org.xmlcml.graphics.svg.SVGG;
import org.xmlcml.graphics.svg.SVGPath;
import org.xmlcml.graphics.svg.SVGSVG;
import org.xmlcml.graphics.svg.SVGText;
import org.xmlcml.graphics.svg.objects.SVGBoxChart;
import org.xmlcml.html.HtmlElement;
import org.xmlcml.html.HtmlFactory;
import org.xmlcml.html.util.HtmlUtil;
import org.xmlcml.norma.Fixtures;
import org.xmlcml.norma.Norma;
import org.xmlcml.norma.input.pdf.PDF2XHTMLConverter;
import org.xmlcml.xml.XMLUtil;

import com.google.common.collect.HashMultiset;
import com.google.common.collect.Multiset;
import com.google.common.collect.Multiset.Entry;

public class BMCTest {

	
	private static final String SRC_MAIN_RESOURCES = "src/main/resources";
	private static final Logger LOG = Logger.getLogger(BMCTest.class);
	static {
		LOG.setLevel(Level.DEBUG);
	}

	@Test
	@Ignore // too long
	public void readProvisionalPDFTest() throws Exception {
		PDF2XHTMLConverter converter = new PDF2XHTMLConverter();
		HtmlElement htmlElement = converter.readAndConvertToXHTML(Fixtures.BMC_0277_PDF);
		new File("target/bmc/").mkdirs();
		XMLUtil.debug(htmlElement, new FileOutputStream("target/bmc/0277.html"), 1);
		
	}
	

	@Test
	@Ignore // too long - creates the SVG // file missing
	public void readBMCTrialsProvisional() throws Exception {
		PDF2XHTMLConverter converter = new PDF2XHTMLConverter();
		HtmlElement htmlElement = converter.readAndConvertToXHTML(new File(Fixtures.TEST_BMC_DIR, "1745-6215-15-486.pdf"));
		new File("target/bmc/").mkdirs();
		XMLUtil.debug(htmlElement, new FileOutputStream("target/bmc/486.html"), 1);
	}
	
	@Test
//	@Ignore // too long - creates the SVG // file missing
	public void readBMCLions() throws Exception {
		PDF2XHTMLConverter converter = new PDF2XHTMLConverter();
		HtmlElement htmlElement = converter.readAndConvertToXHTML(new File(Fixtures.TEST_BMC_DIR, "1745-2148-14-70.pdf"));
		new File("target/bmc/").mkdirs();
		XMLUtil.debug(htmlElement, new FileOutputStream("target/bmc/14-170.html"), 1);
	}
	

	
	@Test
	// this one has outline glyphs... :-( // all papers in this journal do :-(
	// 4 boxes and three lines
	public void testExtractFlowChart() {
		SVGElement rawChart = SVGElement.readAndCreateSVG(new File(Fixtures.BMC_MISC_DIR, "1745-6215-15-486.29.0.svg"));
		SVGBoxChart boxChart = new SVGBoxChart(rawChart);
		boxChart.createChart();
		List<SVGPath> pathList = boxChart.getSVGPathList();
		Multiset<String> signatureSet = HashMultiset.create();
		Map<String, String> characterByDStringMap = new HashMap<String, String>();
		characterByDStringMap.put("MLCCCCCCLCCCCCCCCCCLLCCCCCCCCCCCCCCCCCCCCCCZMLLCCCCCCCCZ","a");
		characterByDStringMap.put("MCCCCCCCCLCCCCCCCCLCCCCCCCCLCCCCCCCCZMCCCCCCCCLCCCCCCZ","b");<commit_msg>reads PDF into cmdir<commit_after>package org.xmlcml.norma.pubstyle.bmc;

import java.awt.Dimension;
import java.io.File;
import java.io.FileOutputStream;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.io.FileUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import org.xmlcml.cmine.files.CMDir;
import org.xmlcml.euclid.Real2;
import org.xmlcml.graphics.svg.SVGElement;
import org.xmlcml.graphics.svg.SVGG;
import org.xmlcml.graphics.svg.SVGPath;
import org.xmlcml.graphics.svg.SVGSVG;
import org.xmlcml.graphics.svg.SVGText;
import org.xmlcml.graphics.svg.objects.SVGBoxChart;
import org.xmlcml.html.HtmlElement;
import org.xmlcml.html.HtmlFactory;
import org.xmlcml.html.util.HtmlUtil;
import org.xmlcml.norma.Fixtures;
import org.xmlcml.norma.Norma;
import org.xmlcml.norma.input.pdf.PDF2XHTMLConverter;
import org.xmlcml.xml.XMLUtil;

import com.google.common.collect.HashMultiset;
import com.google.common.collect.Multiset;
import com.google.common.collect.Multiset.Entry;

public class BMCTest {

	
	private static final String SRC_MAIN_RESOURCES = "src/main/resources";
	private static final Logger LOG = Logger.getLogger(BMCTest.class);
	static {
		LOG.setLevel(Level.DEBUG);
	}

	@Test
	@Ignore // too long
	public void readProvisionalPDFTest() throws Exception {
		PDF2XHTMLConverter converter = new PDF2XHTMLConverter();
		HtmlElement htmlElement = converter.readAndConvertToXHTML(Fixtures.BMC_0277_PDF);
		new File("target/bmc/").mkdirs();
		XMLUtil.debug(htmlElement, new FileOutputStream("target/bmc/0277.html"), 1);
		
	}
	

	@Test
	@Ignore // too long - creates the SVG // file missing
	public void readBMCTrialsProvisional() throws Exception {
		PDF2XHTMLConverter converter = new PDF2XHTMLConverter();
		HtmlElement htmlElement = converter.readAndConvertToXHTML(new File(Fixtures.TEST_BMC_DIR, "1745-6215-15-486.pdf"));
		new File("target/bmc/").mkdirs();
		XMLUtil.debug(htmlElement, new FileOutputStream("target/bmc/486.html"), 1);
	}
	
	@Test
	@Ignore // too long - creates the SVG // file missing
	public void readBMCLions() throws Exception {
		PDF2XHTMLConverter converter = new PDF2XHTMLConverter();
		HtmlElement htmlElement = converter.readAndConvertToXHTML(new File(Fixtures.TEST_BMC_DIR, "1745-2148-14-70.pdf"));
		new File("target/bmc/").mkdirs();
		XMLUtil.debug(htmlElement, new FileOutputStream("target/bmc/14-170.html"), 1);
	}
	

	
	@Test
	// this one has outline glyphs... :-( // all papers in this journal do :-(
	// 4 boxes and three lines
	public void testExtractFlowChart() {
		SVGElement rawChart = SVGElement.readAndCreateSVG(new File(Fixtures.BMC_MISC_DIR, "1745-6215-15-486.29.0.svg"));
		SVGBoxChart boxChart = new SVGBoxChart(rawChart);
		boxChart.createChart();
		List<SVGPath> pathList = boxChart.getSVGPathList();
		Multiset<String> signatureSet = HashMultiset.create();
		Map<String, String> characterByDStringMap = new HashMap<String, String>();
		characterByDStringMap.put("MLCCCCCCLCCCCCCCCCCLLCCCCCCCCCCCCCCCCCCCCCCZMLLCCCCCCCCZ","a");
		characterByDStringMap.put("MCCCCCCCCLCCCCCCCCLCCCCCCCCLCCCCCCCCZMCCCCCCCCLCCCCCCZ","b");
{'fraction_non_alphanumeric': 0.12102404965089217, 'fraction_numerical': 0.022032583397982933, 'mean_word_length': 5.420318725099602, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 34}, 'pii_count': 0, 'substrings_counts': 16, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5448303', 'n_tokens_mistral': 2429, 'n_tokens_neox': 2217, 'n_words': 408}