<commit_before>/**
 * Copyright 2011 The Open Source Research Group,
 *                University of Erlangen-Nürnberg
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.sweble.wikitext.parser;

import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;

import junit.framework.Assert;

import org.apache.commons.io.IOUtils;
import org.apache.commons.jxpath.JXPathContext;
import org.apache.commons.jxpath.ri.JXPathContextReferenceImpl;
import org.junit.Test;
import org.sweble.wikitext.parser.nodes.WtNode;
import org.sweble.wikitext.parser.utils.NonExpandingParser;
import org.sweble.wikitext.parser.utils.WtAstPrinter;

import xtc.parser.ParseException;
import de.fau.cs.osr.ptk.common.jxpath.AstNodePointerFactory;
import de.fau.cs.osr.ptk.common.test.TestResourcesFixture;
import de.fau.cs.osr.utils.StringUtils;

public class XPathTest
{
	private static final String PATH = "/xpath";
	
	private static final boolean WARNINGS_ENABLED = false;
	
	private static final boolean GATHER_RTD = true;
	
	private static final boolean AUTO_CORRECT = false;
	
	// =========================================================================
	
	@Test
	public void testFrance() throws IOException, ParseException
	{
		String title = "raw-France";
		
		WtNode ast = parse(title);
		
		JXPathContext context = JXPathContext.newContext(ast);
		
		StringBuilder b = new StringBuilder();
		
		doQuery(context, b, "/WtParagraph[3]");
		
		doQuery(context, b, "(//WtSection[@level=3])[1]");
		
		doQuery(context, b, "//WtTemplate[contains(name//WtText[@content],\"Infobox Country\")]//WtTemplateArgument[contains(name//WtText[@content],\"capital\")]/value");
		
		String actual = TestResourcesFixture.lineEndToUnix(b.toString());
		
		String expected = null;
		try
		{
			expected = TestResourcesFixture.lineEndToUnix(load(PATH + "/ast/" + title + ".ast"));
		}
		catch (IOException e)
		{
		}
		
		Assert.assertEquals(expected, actual);
	}
	
	// =========================================================================
	
	@Test
	public void testGermany() throws IOException, ParseException
	{
		String title = "raw-Germany";
		
		WtNode ast = parse(title);
		
		JXPathContext context = JXPathContext.newContext(ast);
		
		StringBuilder b = new StringBuilder();
		
		doQuery(context, b, "//WtTemplate[contains(name//WtText[@content],\"Infobox country\")]//WtTemplateArgument[contains(name//WtText[@content],\"capital\")]/value");
		
		String actual = TestResourcesFixture.lineEndToUnix(b.toString());
		
		String expected = null;
		try
		{
			expected = TestResourcesFixture.lineEndToUnix(load(PATH + "/ast/" + title + ".ast"));
		}
		catch (IOException e)
		{
		}
		
		Assert.assertEquals(expected, actual);
	}
	
	// =========================================================================
	
	private final NonExpandingParser parser;
	
	public XPathTest()
	{
		JXPathContextReferenceImpl.addNodePointerFactory(
				new AstNodePointerFactory());
		
		parser = new NonExpandingParser(WARNINGS_ENABLED, GATHER_RTD, AUTO_CORRECT);
	}
	
	// =========================================================================
	
	private WtNode parse(String title) throws IOException, ParseException
	{
		WtNode ast = parser.parseArticle(
				load(PATH + "/wikitext/" + title + ".wikitext"),
				title);
		
		return ast;
	}
	
	private String load(String path) throws IOException
	{
		InputStream in = getClass().getResourceAsStream(path);
		if (in == null)
			return null;
		return IOUtils.toString(in, "UTF-8");
	}
	
	private void doQuery(
			JXPathContext context,
			StringBuilder b,
			final String query)
	{
		b.append(StringUtils.strrep('-', 80));
		b.append("\n  ");
		b.append(query);
		b.append('\n');
		b.append(StringUtils.strrep('-', 80));
		b.append('\n');
		
		int j = 1;
		for (Iterator<?> i = context.iterate(query); i.hasNext();)<commit_msg>Improved on last patch: All text loaded from files is normalized to unix line ends<commit_after>/**
 * Copyright 2011 The Open Source Research Group,
 *                University of Erlangen-Nürnberg
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.sweble.wikitext.parser;

import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;

import junit.framework.Assert;

import org.apache.commons.io.IOUtils;
import org.apache.commons.jxpath.JXPathContext;
import org.apache.commons.jxpath.ri.JXPathContextReferenceImpl;
import org.junit.Test;
import org.sweble.wikitext.parser.nodes.WtNode;
import org.sweble.wikitext.parser.utils.NonExpandingParser;
import org.sweble.wikitext.parser.utils.WtAstPrinter;

import xtc.parser.ParseException;
import de.fau.cs.osr.ptk.common.jxpath.AstNodePointerFactory;
import de.fau.cs.osr.ptk.common.test.TestResourcesFixture;
import de.fau.cs.osr.utils.StringUtils;

public class XPathTest
{
	private static final String PATH = "/xpath";
	
	private static final boolean WARNINGS_ENABLED = false;
	
	private static final boolean GATHER_RTD = true;
	
	private static final boolean AUTO_CORRECT = false;
	
	// =========================================================================
	
	@Test
	public void testFrance() throws IOException, ParseException
	{
		String title = "raw-France";
		
		WtNode ast = parse(title);
		
		JXPathContext context = JXPathContext.newContext(ast);
		
		StringBuilder b = new StringBuilder();
		
		doQuery(context, b, "/WtParagraph[3]");
		
		doQuery(context, b, "(//WtSection[@level=3])[1]");
		
		doQuery(context, b, "//WtTemplate[contains(name//WtText[@content],\"Infobox Country\")]//WtTemplateArgument[contains(name//WtText[@content],\"capital\")]/value");
		
		String actual = TestResourcesFixture.lineEndToUnix(b.toString());
		
		String expected = null;
		try
		{
			expected = load(PATH + "/ast/" + title + ".ast");
		}
		catch (IOException e)
		{
		}
		
		Assert.assertEquals(expected, actual);
	}
	
	// =========================================================================
	
	@Test
	public void testGermany() throws IOException, ParseException
	{
		String title = "raw-Germany";
		
		WtNode ast = parse(title);
		
		JXPathContext context = JXPathContext.newContext(ast);
		
		StringBuilder b = new StringBuilder();
		
		doQuery(context, b, "//WtTemplate[contains(name//WtText[@content],\"Infobox country\")]//WtTemplateArgument[contains(name//WtText[@content],\"capital\")]/value");
		
		String actual = TestResourcesFixture.lineEndToUnix(b.toString());
		
		String expected = null;
		try
		{
			expected = load(PATH + "/ast/" + title + ".ast");
		}
		catch (IOException e)
		{
		}
		
		Assert.assertEquals(expected, actual);
	}
	
	// =========================================================================
	
	private final NonExpandingParser parser;
	
	public XPathTest()
	{
		JXPathContextReferenceImpl.addNodePointerFactory(
				new AstNodePointerFactory());
		
		parser = new NonExpandingParser(WARNINGS_ENABLED, GATHER_RTD, AUTO_CORRECT);
	}
	
	// =========================================================================
	
	private WtNode parse(String title) throws IOException, ParseException
	{
		WtNode ast = parser.parseArticle(
				load(PATH + "/wikitext/" + title + ".wikitext"),
				title);
		
		return ast;
	}
	
	private String load(String path) throws IOException
	{
		InputStream in = getClass().getResourceAsStream(path);
		if (in == null)
			return null;
		return TestResourcesFixture.lineEndToUnix(
				IOUtils.toString(in, "UTF-8"));
	}
	
	private void doQuery(
			JXPathContext context,
			StringBuilder b,
			final String query)
	{
		b.append(StringUtils.strrep('-', 80));
		b.append("\n  ");
		b.append(query);
		b.append('\n');
		b.append(StringUtils.strrep('-', 80));
		b.append('\n');
		
		int j = 1;
		for (Iterator<?> i = context.iterate(query); i.hasNext();)
{'fraction_non_alphanumeric': 0.18229403732362312, 'fraction_numerical': 0.0038689121529358215, 'mean_word_length': 4.3234403391883705, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2562930', 'n_tokens_mistral': 2971, 'n_tokens_neox': 2581, 'n_words': 634}