<commit_before>/*******************************************************************************
 * Copyright (c) 2015 Eclipse RDF4J contributors, Aduna, and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Distribution License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/org/documents/edl-v10.php.
 *******************************************************************************/
package org.eclipse.rdf4j.rio;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Random;

import org.apache.commons.io.IOUtils;
import org.eclipse.rdf4j.model.BNode;
import org.eclipse.rdf4j.model.IRI;
import org.eclipse.rdf4j.model.Literal;
import org.eclipse.rdf4j.model.Model;
import org.eclipse.rdf4j.model.Resource;
import org.eclipse.rdf4j.model.Statement;
import org.eclipse.rdf4j.model.Value;
import org.eclipse.rdf4j.model.ValueFactory;
import org.eclipse.rdf4j.model.impl.LinkedHashModel;
import org.eclipse.rdf4j.model.impl.SimpleValueFactory;
import org.eclipse.rdf4j.model.util.Models;
import org.eclipse.rdf4j.model.vocabulary.DC;
import org.eclipse.rdf4j.model.vocabulary.DCTERMS;
import org.eclipse.rdf4j.model.vocabulary.EARL;
import org.eclipse.rdf4j.model.vocabulary.FOAF;
import org.eclipse.rdf4j.model.vocabulary.OWL;
import org.eclipse.rdf4j.model.vocabulary.RDF;
import org.eclipse.rdf4j.model.vocabulary.RDFS;
import org.eclipse.rdf4j.model.vocabulary.SESAME;
import org.eclipse.rdf4j.model.vocabulary.SKOS;
import org.eclipse.rdf4j.model.vocabulary.SP;
import org.eclipse.rdf4j.model.vocabulary.SPIN;
import org.eclipse.rdf4j.rio.ParserConfig;
import org.eclipse.rdf4j.rio.RDFFormat;
import org.eclipse.rdf4j.rio.RDFHandlerException;
import org.eclipse.rdf4j.rio.RDFParseException;
import org.eclipse.rdf4j.rio.RDFParser;
import org.eclipse.rdf4j.rio.RDFParserFactory;
import org.eclipse.rdf4j.rio.RDFWriter;
import org.eclipse.rdf4j.rio.RDFWriterFactory;
import org.eclipse.rdf4j.rio.Rio;
import org.eclipse.rdf4j.rio.WriterConfig;
import org.eclipse.rdf4j.rio.helpers.BasicParserSettings;
import org.eclipse.rdf4j.rio.helpers.JSONLDMode;
import org.eclipse.rdf4j.rio.helpers.StatementCollector;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

/**
 * @author <NAME>
 * @author <NAME>
 */
public abstract class RDFWriterTest {

	@Rule
	public TemporaryFolder tempDir = new TemporaryFolder();

	/**
	 * One prng per testsuite run
	 */
	private static final Random prng = new SecureRandom();

	protected RDFWriterFactory rdfWriterFactory;

	protected RDFParserFactory rdfParserFactory;

	protected ValueFactory vf;

	private BNode bnode;

	private BNode bnodeEmpty;

	private BNode bnodeSingleLetter;

	private BNode bnodeDuplicateLetter;

	private BNode bnodeNumeric;

	private BNode bnodeDashes;

	private BNode bnodeSpecialChars;

	private BNode bnodeSingleUseSubject;

	private BNode bnodeSingleUseObject;

	private BNode bnodeUseAcrossContextsSubject;

	private BNode bnodeUseAcrossContextsSubjectAndObject;

	private BNode bnodeUseAcrossContextsObject;

	private IRI uri1;

	private IRI uri2;

	private IRI uri3;

	private IRI uri4;

	private IRI uri5;

	private Literal plainLit;

	private Literal dtLit;

	private Literal langLit;

	private Literal litWithNewlineAtEnd;

	private Literal litWithNewlineAtStart;

	private Literal litWithMultipleNewlines;

	private Literal litWithSingleQuotes;

	private Literal litWithDoubleQuotes;

	private Literal litBigPlaceholder;

	private String exNs;

	private List<Resource> potentialSubjects;

	private List<Value> potentialObjects;

	private List<IRI> potentialPredicates;

	protected RDFWriterTest(RDFWriterFactory writerF, RDFParserFactory parserF) {
		rdfWriterFactory = writerF;
		rdfParserFactory = parserF;

		vf = SimpleValueFactory.getInstance();

		exNs = "http://example.org/";

		bnode = vf.createBNode("anon");
		bnodeEmpty = vf.createBNode("");
		bnodeSingleLetter = vf.createBNode("a");
		bnodeDuplicateLetter = vf.createBNode("aa");
		bnodeNumeric = vf.createBNode("123");
		bnodeDashes = vf.createBNode("a-b");
		bnodeSpecialChars = vf.createBNode("$%^&*()!@#$a-b<>?\"'[]{}|\\");
		bnodeSingleUseSubject = vf.createBNode("bnodeSingleUseSubject");
		bnodeSingleUseObject = vf.createBNode("bnodeSingleUseObject");
		bnodeUseAcrossContextsSubject = vf.createBNode("bnodeUseAcrossContextsSubject");
		bnodeUseAcrossContextsSubjectAndObject = vf.createBNode("bnodeUseAcrossContextsSubjectAndObject");
		bnodeUseAcrossContextsObject = vf.createBNode("bnodeUseAcrossContextsObject");

		uri1 = vf.createIRI(exNs, "uri1");
		uri2 = vf.createIRI(exNs, "uri2");
		uri3 = vf.createIRI(exNs, "uri3.");
		uri4 = vf.createIRI(exNs, "uri4#");
		uri5 = vf.createIRI(exNs, "uri5/");
		plainLit = vf.createLiteral("plain");
		dtLit = vf.createLiteral(1);
		langLit = vf.createLiteral("test", "en");
		litWithNewlineAtEnd = vf.createLiteral("literal with newline at end\n");
		litWithNewlineAtStart = vf.createLiteral("\nliteral with newline at start");
		litWithMultipleNewlines = vf.createLiteral("\nliteral \nwith newline at start\n");
		litWithSingleQuotes = vf.createLiteral("'''some single quote text''' - abc");
		litWithDoubleQuotes = vf.createLiteral("\"\"\"some double quote text\"\"\" - abc");

		litBigPlaceholder = vf.createLiteral(prng.nextDouble());

		potentialSubjects = new ArrayList<Resource>();
		potentialSubjects.add(bnode);
		potentialSubjects.add(bnodeEmpty);
		potentialSubjects.add(bnodeSingleLetter);
		potentialSubjects.add(bnodeDuplicateLetter);
		potentialSubjects.add(bnodeNumeric);
		potentialSubjects.add(bnodeDashes);
		potentialSubjects.add(bnodeSpecialChars);
		potentialSubjects.add(uri1);
		potentialSubjects.add(uri2);
		potentialSubjects.add(uri3);
		potentialSubjects.add(uri4);
		potentialSubjects.add(uri5);
		for (int i = 0; i < 50; i++) {
			potentialSubjects.add(vf.createBNode());
		}
		for (int i = 0; i < 50; i++) {
			potentialSubjects.add(vf.createBNode(Integer.toHexString(i)));
		}
		for (int i = 1; i < 50; i++) {
			potentialSubjects.add(vf.createBNode("a" + Integer.toHexString(i).toUpperCase()));
		}
		for (int i = 1; i < 50; i++) {
			potentialSubjects.add(vf.createBNode("a" + Integer.toHexString(i).toLowerCase()));
		}
		for (int i = 0; i < 200; i++) {
			potentialSubjects.add(
					vf.createIRI(exNs + Integer.toHexString(i) + "/a" + Integer.toOctalString(i % 133)));
		}
		Collections.shuffle(potentialSubjects, prng);

		potentialObjects = new ArrayList<Value>();
		potentialObjects.addAll(potentialSubjects);
		potentialObjects.add(plainLit);
		potentialObjects.add(dtLit);
		potentialObjects.add(langLit);
		// FIXME: SES-879: The following break the RDF/XML parser/writer
		// combination in terms of getting the same number of triples back as we
		// start with

		if (rdfParserFactory.getRDFFormat().equals(RDFFormat.RDFXML)) {
			// System.out.println("FIXME: SES-879: RDFXML Parser does not
			// preserve literals starting or ending in newline character");
		}
		else {
			potentialObjects.add(litWithNewlineAtEnd);
			potentialObjects.add(litWithNewlineAtStart);
			potentialObjects.add(litWithMultipleNewlines);
		}
		potentialObjects.add(litWithSingleQuotes);
		potentialObjects.add(litWithDoubleQuotes);
		potentialObjects.add(litBigPlaceholder);
		Collections.shuffle(potentialObjects, prng);

		potentialPredicates = new ArrayList<IRI>();
		// In particular, the following fuzz tests the ability of the parser to
		// cater for rdf:type predicates with literal endings, in unknown
		// situations. All parsers/writers should preserve these statements,
		// even
		// if they have shortcuts for URIs
		potentialPredicates.add(RDF.TYPE);
		potentialPredicates.add(RDF.NIL);
		potentialPredicates.add(RDF.FIRST);
		potentialPredicates.add(RDF.REST);
		potentialPredicates.add(SKOS.ALT_LABEL);
		potentialPredicates.add(SKOS.PREF_LABEL);
		potentialPredicates.add(SKOS.BROADER_TRANSITIVE);
		potentialPredicates.add(OWL.ONTOLOGY);
		potentialPredicates.add(OWL.ONEOF);
		potentialPredicates.add(DC.TITLE);
		potentialPredicates.add(DCTERMS.ACCESS_RIGHTS);
		potentialPredicates.add(FOAF.KNOWS);
		potentialPredicates.add(EARL.SUBJECT);
		potentialPredicates.add(RDFS.LABEL);
		potentialPredicates.add(SP.DEFAULT_PROPERTY);
		potentialPredicates.add(SP.TEXT_PROPERTY);
		potentialPredicates.add(SP.BIND_CLASS);
		potentialPredicates.add(SP.DOCUMENT_PROPERTY);
		potentialPredicates.add(SPIN.LABEL_TEMPLATE_PROPERTY);
		potentialPredicates.add(SESAME.DIRECTTYPE);
		Collections.shuffle(potentialPredicates, prng);
	}

	/**
	 * Override this method to setup custom settings for WriterConfig needed to pass tests.
	 * <p>
	 * One example of this is that {@link JSONLDMode#EXPAND} does not preserve namespace prefixes, causing the
	 * tests here to be unnecessarily ignored. The fix for that is to override this method and set the mode to
	 * {@link JSONLDMode#COMPACT} that does preserve namespaces.
	 * 
	 * @param config
	 *        The config object to modify.
	 */
	protected void setupWriterConfig(WriterConfig config) {
	}

	/**
	 * Override this method to setup custom settings for ParserConfig needed to pass tests.
	 * 
	 * @param config
	 *        The config object to modify.
	 */
	protected void setupParserConfig(ParserConfig config) {
		config.set(BasicParserSettings.FAIL_ON_UNKNOWN_DATATYPES, true);
		config.set(BasicParserSettings.FAIL_ON_UNKNOWN_LANGUAGES, true);
	}

	@Test
	public void testRoundTripWithXSDString()
		throws RDFHandlerException, IOException, RDFParseException
	{
		roundTrip(true);
	}

	@Test
	public void testRoundTripWithoutXSDString()
		throws RDFHandlerException, IOException, RDFParseException
	{
		roundTrip(false);
	}

	private void roundTrip(boolean serialiseXSDString)
		throws RDFHandlerException, IOException, RDFParseException
	{
		testRoundTripInternal(false);
	}

	@Test
	public void testRoundTripPreserveBNodeIds()
		throws Exception
	{
		testRoundTripInternal(true);
	}

	private void testRoundTripInternal(boolean preserveBNodeIds)
		throws RDFHandlerException, IOException, RDFParseException
	{
		Statement st1 = vf.createStatement(bnode, uri1, plainLit);
		Statement st2 = vf.createStatement(bnodeEmpty, uri1, plainLit);
		Statement st3 = vf.createStatement(bnodeNumeric, uri1, plainLit);
		Statement st4 = vf.createStatement(bnodeDashes, uri1, plainLit);
		Statement st5 = vf.createStatement(bnodeSpecialChars, uri1, plainLit);
		Statement st6 = vf.createStatement(uri2, uri1, bnode);
		Statement st7 = vf.createStatement(uri2, uri1, bnodeEmpty);
		Statement st8 = vf.createStatement(uri2, uri1, bnodeNumeric);
		Statement st9 = vf.createStatement(uri2, uri1, bnodeDashes);
		Statement st10 = vf.createStatement(uri2, uri1, bnodeSpecialChars);
		Statement st11 = vf.createStatement(uri1, uri2, langLit, uri2);
		Statement st12 = vf.createStatement(uri1, uri2, dtLit);
		Statement st13 = vf.createStatement(uri1, uri2, litWithNewlineAtEnd);
		Statement st14 = vf.createStatement(uri1, uri2, litWithNewlineAtStart);
		Statement st15 = vf.createStatement(uri1, uri2, litWithMultipleNewlines);
		Statement st16 = vf.createStatement(uri1, uri2, litWithSingleQuotes);
		Statement st17 = vf.createStatement(uri1, uri2, litWithDoubleQuotes);
		Statement st18 = vf.createStatement(uri1, uri2, uri3);
		Statement st19 = vf.createStatement(uri2, uri3, uri1);
		Statement st20 = vf.createStatement(uri3, uri1, uri2);
		Statement st21 = vf.createStatement(bnodeSingleUseSubject, uri4, uri5);
		Statement st22 = vf.createStatement(uri4, uri5, bnodeSingleUseObject);
		// Blank node use across contexts, which is unique to TriG-1.1 in
		// interpretation
		Statement st23 = vf.createStatement(bnodeUseAcrossContextsSubject, uri4, uri3, uri1);
		Statement st24 = vf.createStatement(bnodeUseAcrossContextsSubject, uri4, uri3, uri2);
		Statement st25 = vf.createStatement(bnodeUseAcrossContextsSubjectAndObject, uri4, uri2, uri1);
		Statement st26 = vf.createStatement(uri4, uri3, bnodeUseAcrossContextsSubjectAndObject, uri3);
		Statement st27 = vf.createStatement(uri3, uri4, bnodeUseAcrossContextsObject, uri1);
		Statement st28 = vf.createStatement(uri3, uri4, bnodeUseAcrossContextsObject, uri2);
		Statement st29 = vf.createStatement(uri5, uri4, uri3, bnodeSpecialChars);

		ByteArrayOutputStream out = new ByteArrayOutputStream();
		RDFWriter rdfWriter = rdfWriterFactory.getWriter(out);
		setupWriterConfig(rdfWriter.getWriterConfig());
		rdfWriter.handleNamespace("ex", exNs);
		rdfWriter.startRDF();
		rdfWriter.handleStatement(st1);
		rdfWriter.handleStatement(st2);
		rdfWriter.handleStatement(st3);
		rdfWriter.handleStatement(st4);
		rdfWriter.handleStatement(st5);
		rdfWriter.handleStatement(st6);
		rdfWriter.handleStatement(st7);
		rdfWriter.handleStatement(st8);
		rdfWriter.handleStatement(st9);
		rdfWriter.handleStatement(st10);
		rdfWriter.handleStatement(st11);
		rdfWriter.handleStatement(st12);
		rdfWriter.handleStatement(st13);
		rdfWriter.handleStatement(st14);
		rdfWriter.handleStatement(st15);
		rdfWriter.handleStatement(st16);
		rdfWriter.handleStatement(st17);
		rdfWriter.handleStatement(st18);
		rdfWriter.handleStatement(st19);
		rdfWriter.handleStatement(st20);
		rdfWriter.handleStatement(st21);
		rdfWriter.handleStatement(st22);
		rdfWriter.handleStatement(st23);
		rdfWriter.handleStatement(st24);
		rdfWriter.handleStatement(st25);
		rdfWriter.handleStatement(st26);
		rdfWriter.handleStatement(st27);
		rdfWriter.handleStatement(st28);
		rdfWriter.handleStatement(st29);
		rdfWriter.endRDF();

		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
		RDFParser rdfParser = rdfParserFactory.getParser();
		setupParserConfig(rdfParser.getParserConfig());
		if (preserveBNodeIds) {
			rdfParser.getParserConfig().set(BasicParserSettings.PRESERVE_BNODE_IDS, true);
		}
		rdfParser.setValueFactory(vf);
		Model model = new LinkedHashModel();
		rdfParser.setRDFHandler(new StatementCollector(model));

		rdfParser.parse(in, "foo:bar");

		if (rdfParser.getRDFFormat().supportsContexts()) {
			assertEquals("Unexpected number of statements, found " + model.size(), 29, model.size());
		}
		else {
			// Two sets of two statements, st23/st24 and st27/st28 in the input set differ only on context
			// which isn't preserved by this format
			assertEquals("Unexpected number of statements, found " + model.size(), 27, model.size());
		}

		if (rdfParser.getRDFFormat().supportsNamespaces()) {
			assertTrue("Expected at least one namespace, found" + model.getNamespaces().size(),
					model.getNamespaces().size() >= 1);
			assertEquals(exNs, model.getNamespace("ex").get().getName());
		}

		// Test for four unique statements for blank nodes in subject position
		assertEquals(5, model.filter(null, uri1, plainLit).size());
		// Test for four unique statements for blank nodes in object position
		assertEquals(5, model.filter(uri2, uri1, null).size());
		if (rdfParser.getRDFFormat().supportsContexts()) {
			assertTrue("missing statement with language literal and context", model.contains(st11));
		}
		else {
			assertTrue("missing statement with language literal",
					model.contains(vf.createStatement(uri1, uri2, langLit)));
		}
		assertTrue("missing statement with datatype", model.contains(st12));
		if (rdfParser.getRDFFormat().equals(RDFFormat.RDFXML)) {
			System.out.println(
					"FIXME: SES-879: RDFXML Parser does not preserve literals starting or ending in newline character");
		}
		else {
			assertTrue("missing statement with literal ending with newline", model.contains(st13));
			assertTrue("missing statement with literal starting with newline", model.contains(st14));
			assertTrue("missing statement with literal containing multiple newlines", model.contains(st15));
		}
		assertTrue("missing statement with single quotes", model.contains(st16));
		assertTrue("missing statement with double quotes", model.contains(st17));
		assertTrue("missing statement with object URI ending in period", model.contains(st18));
		assertTrue("missing statement with predicate URI ending in period", model.contains(st19));
		assertTrue("missing statement with subject URI ending in period", model.contains(st20));

		if (rdfParser.getRDFFormat().supportsContexts()) {
			assertTrue("missing statement with blank node single use subject", model.contains(st21));
		}
		else {
			assertEquals("missing statement with blank node single use subject", 1,
					model.filter(null, uri4, uri5).size());
		}
		if (rdfParser.getRDFFormat().supportsContexts()) {
			assertTrue("missing statement with blank node single use object: st22", model.contains(st22));
		}
		else {
			assertEquals("missing statement with blank node single use object: st22", 1,
					model.filter(uri4, uri5, null).size());
		}
		if (rdfParser.getRDFFormat().supportsContexts()) {
			assertTrue("missing statement with blank node use across contexts subject: st23",
					model.contains(st23));
			assertTrue("missing statement with blank node use across contexts subject: st24",
					model.contains(st24));
		}
		else {
			assertEquals("missing statement with blank node use across contexts: st23/st24", 1,
					model.filter(null, uri4, uri3).size());
		}
		if (rdfParser.getRDFFormat().supportsContexts()) {
			assertTrue("missing statement with blank node use across contexts subject and object: st25",
					model.contains(st25));
			assertTrue("missing statement with blank node use across contexts subject and object: st26",
					model.contains(st26));
		}
		else {
			assertEquals("missing statement with blank node use across contexts subject and object: st25", 1,
					model.filter(null, uri4, uri2).size());
			assertEquals("missing statement with blank node use across contexts subject and object: st26", 1,
					model.filter(uri4, uri3, null).size());
		}
		if (rdfParser.getRDFFormat().supportsContexts()) {
			assertTrue("missing statement with blank node use across contexts subject: st27",
					model.contains(st27));
			assertTrue("missing statement with blank node use across contexts subject: st28",
					model.contains(st28));
		}
		else {
			assertEquals("missing statement with blank node use across contexts: object: st27/st28", 1,
					model.filter(uri3, uri4, null).size());
		}
		if (rdfParser.getRDFFormat().supportsContexts()) {
			assertTrue("missing statement with blank node context: st29", model.contains(st29));
		}
		else {
			assertEquals("missing statement with blank node context: st29", 1,
					model.filter(uri5, uri4, uri3).size());
		}
	}

	@Test
	public void testPrefixRedefinition()
		throws RDFHandlerException, RDFParseException, IOException
	{
		String ns1 = "a:";
		String ns2 = "b:";
		String ns3 = "c:";

		IRI uri1 = vf.createIRI(ns1, "r1");
		IRI uri2 = vf.createIRI(ns2, "r2");
		IRI uri3 = vf.createIRI(ns3, "r3");
		Statement st = vf.createStatement(uri1, uri2, uri3);

		ByteArrayOutputStream out = new ByteArrayOutputStream();
		RDFWriter rdfWriter = rdfWriterFactory.getWriter(out);<commit_msg>issue#207 : Fix statement pattern that was matching twice accidentally<commit_after>/*******************************************************************************
 * Copyright (c) 2015 Eclipse RDF4J contributors, Aduna, and others.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Distribution License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/org/documents/edl-v10.php.
 *******************************************************************************/
package org.eclipse.rdf4j.rio;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Random;

import org.apache.commons.io.IOUtils;
import org.eclipse.rdf4j.model.BNode;
import org.eclipse.rdf4j.model.IRI;
import org.eclipse.rdf4j.model.Literal;
import org.eclipse.rdf4j.model.Model;
import org.eclipse.rdf4j.model.Resource;
import org.eclipse.rdf4j.model.Statement;
import org.eclipse.rdf4j.model.Value;
import org.eclipse.rdf4j.model.ValueFactory;
import org.eclipse.rdf4j.model.impl.LinkedHashModel;
import org.eclipse.rdf4j.model.impl.SimpleValueFactory;
import org.eclipse.rdf4j.model.util.Models;
import org.eclipse.rdf4j.model.vocabulary.DC;
import org.eclipse.rdf4j.model.vocabulary.DCTERMS;
import org.eclipse.rdf4j.model.vocabulary.EARL;
import org.eclipse.rdf4j.model.vocabulary.FOAF;
import org.eclipse.rdf4j.model.vocabulary.OWL;
import org.eclipse.rdf4j.model.vocabulary.RDF;
import org.eclipse.rdf4j.model.vocabulary.RDFS;
import org.eclipse.rdf4j.model.vocabulary.SESAME;
import org.eclipse.rdf4j.model.vocabulary.SKOS;
import org.eclipse.rdf4j.model.vocabulary.SP;
import org.eclipse.rdf4j.model.vocabulary.SPIN;
import org.eclipse.rdf4j.rio.ParserConfig;
import org.eclipse.rdf4j.rio.RDFFormat;
import org.eclipse.rdf4j.rio.RDFHandlerException;
import org.eclipse.rdf4j.rio.RDFParseException;
import org.eclipse.rdf4j.rio.RDFParser;
import org.eclipse.rdf4j.rio.RDFParserFactory;
import org.eclipse.rdf4j.rio.RDFWriter;
import org.eclipse.rdf4j.rio.RDFWriterFactory;
import org.eclipse.rdf4j.rio.Rio;
import org.eclipse.rdf4j.rio.WriterConfig;
import org.eclipse.rdf4j.rio.helpers.BasicParserSettings;
import org.eclipse.rdf4j.rio.helpers.JSONLDMode;
import org.eclipse.rdf4j.rio.helpers.StatementCollector;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;

/**
 * @author <NAME>
 * @author <NAME>
 */
public abstract class RDFWriterTest {

	@Rule
	public TemporaryFolder tempDir = new TemporaryFolder();

	/**
	 * One prng per testsuite run
	 */
	private static final Random prng = new SecureRandom();

	protected RDFWriterFactory rdfWriterFactory;

	protected RDFParserFactory rdfParserFactory;

	protected ValueFactory vf;

	private BNode bnode;

	private BNode bnodeEmpty;

	private BNode bnodeSingleLetter;

	private BNode bnodeDuplicateLetter;

	private BNode bnodeNumeric;

	private BNode bnodeDashes;

	private BNode bnodeSpecialChars;

	private BNode bnodeSingleUseSubject;

	private BNode bnodeSingleUseObject;

	private BNode bnodeUseAcrossContextsSubject;

	private BNode bnodeUseAcrossContextsSubjectAndObject;

	private BNode bnodeUseAcrossContextsObject;

	private IRI uri1;

	private IRI uri2;

	private IRI uri3;

	private IRI uri4;

	private IRI uri5;

	private Literal plainLit;

	private Literal dtLit;

	private Literal langLit;

	private Literal litWithNewlineAtEnd;

	private Literal litWithNewlineAtStart;

	private Literal litWithMultipleNewlines;

	private Literal litWithSingleQuotes;

	private Literal litWithDoubleQuotes;

	private Literal litBigPlaceholder;

	private String exNs;

	private List<Resource> potentialSubjects;

	private List<Value> potentialObjects;

	private List<IRI> potentialPredicates;

	protected RDFWriterTest(RDFWriterFactory writerF, RDFParserFactory parserF) {
		rdfWriterFactory = writerF;
		rdfParserFactory = parserF;

		vf = SimpleValueFactory.getInstance();

		exNs = "http://example.org/";

		bnode = vf.createBNode("anon");
		bnodeEmpty = vf.createBNode("");
		bnodeSingleLetter = vf.createBNode("a");
		bnodeDuplicateLetter = vf.createBNode("aa");
		bnodeNumeric = vf.createBNode("123");
		bnodeDashes = vf.createBNode("a-b");
		bnodeSpecialChars = vf.createBNode("$%^&*()!@#$a-b<>?\"'[]{}|\\");
		bnodeSingleUseSubject = vf.createBNode("bnodeSingleUseSubject");
		bnodeSingleUseObject = vf.createBNode("bnodeSingleUseObject");
		bnodeUseAcrossContextsSubject = vf.createBNode("bnodeUseAcrossContextsSubject");
		bnodeUseAcrossContextsSubjectAndObject = vf.createBNode("bnodeUseAcrossContextsSubjectAndObject");
		bnodeUseAcrossContextsObject = vf.createBNode("bnodeUseAcrossContextsObject");

		uri1 = vf.createIRI(exNs, "uri1");
		uri2 = vf.createIRI(exNs, "uri2");
		uri3 = vf.createIRI(exNs, "uri3.");
		uri4 = vf.createIRI(exNs, "uri4#");
		uri5 = vf.createIRI(exNs, "uri5/");
		plainLit = vf.createLiteral("plain");
		dtLit = vf.createLiteral(1);
		langLit = vf.createLiteral("test", "en");
		litWithNewlineAtEnd = vf.createLiteral("literal with newline at end\n");
		litWithNewlineAtStart = vf.createLiteral("\nliteral with newline at start");
		litWithMultipleNewlines = vf.createLiteral("\nliteral \nwith newline at start\n");
		litWithSingleQuotes = vf.createLiteral("'''some single quote text''' - abc");
		litWithDoubleQuotes = vf.createLiteral("\"\"\"some double quote text\"\"\" - abc");

		litBigPlaceholder = vf.createLiteral(prng.nextDouble());

		potentialSubjects = new ArrayList<Resource>();
		potentialSubjects.add(bnode);
		potentialSubjects.add(bnodeEmpty);
		potentialSubjects.add(bnodeSingleLetter);
		potentialSubjects.add(bnodeDuplicateLetter);
		potentialSubjects.add(bnodeNumeric);
		potentialSubjects.add(bnodeDashes);
		potentialSubjects.add(bnodeSpecialChars);
		potentialSubjects.add(uri1);
		potentialSubjects.add(uri2);
		potentialSubjects.add(uri3);
		potentialSubjects.add(uri4);
		potentialSubjects.add(uri5);
		for (int i = 0; i < 50; i++) {
			potentialSubjects.add(vf.createBNode());
		}
		for (int i = 0; i < 50; i++) {
			potentialSubjects.add(vf.createBNode(Integer.toHexString(i)));
		}
		for (int i = 1; i < 50; i++) {
			potentialSubjects.add(vf.createBNode("a" + Integer.toHexString(i).toUpperCase()));
		}
		for (int i = 1; i < 50; i++) {
			potentialSubjects.add(vf.createBNode("a" + Integer.toHexString(i).toLowerCase()));
		}
		for (int i = 0; i < 200; i++) {
			potentialSubjects.add(
					vf.createIRI(exNs + Integer.toHexString(i) + "/a" + Integer.toOctalString(i % 133)));
		}
		Collections.shuffle(potentialSubjects, prng);

		potentialObjects = new ArrayList<Value>();
		potentialObjects.addAll(potentialSubjects);
		potentialObjects.add(plainLit);
		potentialObjects.add(dtLit);
		potentialObjects.add(langLit);
		// FIXME: SES-879: The following break the RDF/XML parser/writer
		// combination in terms of getting the same number of triples back as we
		// start with

		if (rdfParserFactory.getRDFFormat().equals(RDFFormat.RDFXML)) {
			// System.out.println("FIXME: SES-879: RDFXML Parser does not
			// preserve literals starting or ending in newline character");
		}
		else {
			potentialObjects.add(litWithNewlineAtEnd);
			potentialObjects.add(litWithNewlineAtStart);
			potentialObjects.add(litWithMultipleNewlines);
		}
		potentialObjects.add(litWithSingleQuotes);
		potentialObjects.add(litWithDoubleQuotes);
		potentialObjects.add(litBigPlaceholder);
		Collections.shuffle(potentialObjects, prng);

		potentialPredicates = new ArrayList<IRI>();
		// In particular, the following fuzz tests the ability of the parser to
		// cater for rdf:type predicates with literal endings, in unknown
		// situations. All parsers/writers should preserve these statements,
		// even
		// if they have shortcuts for URIs
		potentialPredicates.add(RDF.TYPE);
		potentialPredicates.add(RDF.NIL);
		potentialPredicates.add(RDF.FIRST);
		potentialPredicates.add(RDF.REST);
		potentialPredicates.add(SKOS.ALT_LABEL);
		potentialPredicates.add(SKOS.PREF_LABEL);
		potentialPredicates.add(SKOS.BROADER_TRANSITIVE);
		potentialPredicates.add(OWL.ONTOLOGY);
		potentialPredicates.add(OWL.ONEOF);
		potentialPredicates.add(DC.TITLE);
		potentialPredicates.add(DCTERMS.ACCESS_RIGHTS);
		potentialPredicates.add(FOAF.KNOWS);
		potentialPredicates.add(EARL.SUBJECT);
		potentialPredicates.add(RDFS.LABEL);
		potentialPredicates.add(SP.DEFAULT_PROPERTY);
		potentialPredicates.add(SP.TEXT_PROPERTY);
		potentialPredicates.add(SP.BIND_CLASS);
		potentialPredicates.add(SP.DOCUMENT_PROPERTY);
		potentialPredicates.add(SPIN.LABEL_TEMPLATE_PROPERTY);
		potentialPredicates.add(SESAME.DIRECTTYPE);
		Collections.shuffle(potentialPredicates, prng);
	}

	/**
	 * Override this method to setup custom settings for WriterConfig needed to pass tests.
	 * <p>
	 * One example of this is that {@link JSONLDMode#EXPAND} does not preserve namespace prefixes, causing the
	 * tests here to be unnecessarily ignored. The fix for that is to override this method and set the mode to
	 * {@link JSONLDMode#COMPACT} that does preserve namespaces.
	 * 
	 * @param config
	 *        The config object to modify.
	 */
	protected void setupWriterConfig(WriterConfig config) {
	}

	/**
	 * Override this method to setup custom settings for ParserConfig needed to pass tests.
	 * 
	 * @param config
	 *        The config object to modify.
	 */
	protected void setupParserConfig(ParserConfig config) {
		config.set(BasicParserSettings.FAIL_ON_UNKNOWN_DATATYPES, true);
		config.set(BasicParserSettings.FAIL_ON_UNKNOWN_LANGUAGES, true);
	}

	@Test
	public void testRoundTripWithXSDString()
		throws RDFHandlerException, IOException, RDFParseException
	{
		roundTrip(true);
	}

	@Test
	public void testRoundTripWithoutXSDString()
		throws RDFHandlerException, IOException, RDFParseException
	{
		roundTrip(false);
	}

	private void roundTrip(boolean serialiseXSDString)
		throws RDFHandlerException, IOException, RDFParseException
	{
		testRoundTripInternal(false);
	}

	@Test
	public void testRoundTripPreserveBNodeIds()
		throws Exception
	{
		testRoundTripInternal(true);
	}

	private void testRoundTripInternal(boolean preserveBNodeIds)
		throws RDFHandlerException, IOException, RDFParseException
	{
		Statement st1 = vf.createStatement(bnode, uri1, plainLit);
		Statement st2 = vf.createStatement(bnodeEmpty, uri1, plainLit);
		Statement st3 = vf.createStatement(bnodeNumeric, uri1, plainLit);
		Statement st4 = vf.createStatement(bnodeDashes, uri1, plainLit);
		Statement st5 = vf.createStatement(bnodeSpecialChars, uri1, plainLit);
		Statement st6 = vf.createStatement(uri2, uri1, bnode);
		Statement st7 = vf.createStatement(uri2, uri1, bnodeEmpty);
		Statement st8 = vf.createStatement(uri2, uri1, bnodeNumeric);
		Statement st9 = vf.createStatement(uri2, uri1, bnodeDashes);
		Statement st10 = vf.createStatement(uri2, uri1, bnodeSpecialChars);
		Statement st11 = vf.createStatement(uri1, uri2, langLit, uri2);
		Statement st12 = vf.createStatement(uri1, uri2, dtLit);
		Statement st13 = vf.createStatement(uri1, uri2, litWithNewlineAtEnd);
		Statement st14 = vf.createStatement(uri1, uri2, litWithNewlineAtStart);
		Statement st15 = vf.createStatement(uri1, uri2, litWithMultipleNewlines);
		Statement st16 = vf.createStatement(uri1, uri2, litWithSingleQuotes);
		Statement st17 = vf.createStatement(uri1, uri2, litWithDoubleQuotes);
		Statement st18 = vf.createStatement(uri1, uri2, uri3);
		Statement st19 = vf.createStatement(uri2, uri3, uri1);
		Statement st20 = vf.createStatement(uri3, uri1, uri2);
		Statement st21 = vf.createStatement(bnodeSingleUseSubject, uri4, uri5);
		Statement st22 = vf.createStatement(uri4, uri5, bnodeSingleUseObject);
		// Blank node use across contexts, which is unique to TriG-1.1 in
		// interpretation
		Statement st23 = vf.createStatement(bnodeUseAcrossContextsSubject, uri4, uri3, uri1);
		Statement st24 = vf.createStatement(bnodeUseAcrossContextsSubject, uri4, uri3, uri2);
		Statement st25 = vf.createStatement(bnodeUseAcrossContextsSubjectAndObject, uri4, uri2, uri1);
		Statement st26 = vf.createStatement(uri4, uri3, bnodeUseAcrossContextsSubjectAndObject, uri3);
		Statement st27 = vf.createStatement(uri3, uri4, bnodeUseAcrossContextsObject, uri1);
		Statement st28 = vf.createStatement(uri3, uri4, bnodeUseAcrossContextsObject, uri2);
		Statement st29 = vf.createStatement(uri5, uri4, uri1, bnodeSpecialChars);

		ByteArrayOutputStream out = new ByteArrayOutputStream();
		RDFWriter rdfWriter = rdfWriterFactory.getWriter(out);
		setupWriterConfig(rdfWriter.getWriterConfig());
		rdfWriter.handleNamespace("ex", exNs);
		rdfWriter.startRDF();
		rdfWriter.handleStatement(st1);
		rdfWriter.handleStatement(st2);
		rdfWriter.handleStatement(st3);
		rdfWriter.handleStatement(st4);
		rdfWriter.handleStatement(st5);
		rdfWriter.handleStatement(st6);
		rdfWriter.handleStatement(st7);
		rdfWriter.handleStatement(st8);
		rdfWriter.handleStatement(st9);
		rdfWriter.handleStatement(st10);
		rdfWriter.handleStatement(st11);
		rdfWriter.handleStatement(st12);
		rdfWriter.handleStatement(st13);
		rdfWriter.handleStatement(st14);
		rdfWriter.handleStatement(st15);
		rdfWriter.handleStatement(st16);
		rdfWriter.handleStatement(st17);
		rdfWriter.handleStatement(st18);
		rdfWriter.handleStatement(st19);
		rdfWriter.handleStatement(st20);
		rdfWriter.handleStatement(st21);
		rdfWriter.handleStatement(st22);
		rdfWriter.handleStatement(st23);
		rdfWriter.handleStatement(st24);
		rdfWriter.handleStatement(st25);
		rdfWriter.handleStatement(st26);
		rdfWriter.handleStatement(st27);
		rdfWriter.handleStatement(st28);
		rdfWriter.handleStatement(st29);
		rdfWriter.endRDF();

		ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
		RDFParser rdfParser = rdfParserFactory.getParser();
		setupParserConfig(rdfParser.getParserConfig());
		if (preserveBNodeIds) {
			rdfParser.getParserConfig().set(BasicParserSettings.PRESERVE_BNODE_IDS, true);
		}
		rdfParser.setValueFactory(vf);
		Model model = new LinkedHashModel();
		rdfParser.setRDFHandler(new StatementCollector(model));

		rdfParser.parse(in, "foo:bar");

		if (rdfParser.getRDFFormat().supportsContexts()) {
			assertEquals("Unexpected number of statements, found " + model.size(), 29, model.size());
		}
		else {
			// Two sets of two statements, st23/st24 and st27/st28 in the input set differ only on context
			// which isn't preserved by this format
			assertEquals("Unexpected number of statements, found " + model.size(), 27, model.size());
		}

		if (rdfParser.getRDFFormat().supportsNamespaces()) {
			assertTrue("Expected at least one namespace, found" + model.getNamespaces().size(),
					model.getNamespaces().size() >= 1);
			assertEquals(exNs, model.getNamespace("ex").get().getName());
		}

		// Test for four unique statements for blank nodes in subject position
		assertEquals(5, model.filter(null, uri1, plainLit).size());
		// Test for four unique statements for blank nodes in object position
		assertEquals(5, model.filter(uri2, uri1, null).size());
		if (rdfParser.getRDFFormat().supportsContexts()) {
			assertTrue("missing statement with language literal and context", model.contains(st11));
		}
		else {
			assertTrue("missing statement with language literal",
					model.contains(vf.createStatement(uri1, uri2, langLit)));
		}
		assertTrue("missing statement with datatype", model.contains(st12));
		if (rdfParser.getRDFFormat().equals(RDFFormat.RDFXML)) {
			System.out.println(
					"FIXME: SES-879: RDFXML Parser does not preserve literals starting or ending in newline character");
		}
		else {
			assertTrue("missing statement with literal ending with newline", model.contains(st13));
			assertTrue("missing statement with literal starting with newline", model.contains(st14));
			assertTrue("missing statement with literal containing multiple newlines", model.contains(st15));
		}
		assertTrue("missing statement with single quotes", model.contains(st16));
		assertTrue("missing statement with double quotes", model.contains(st17));
		assertTrue("missing statement with object URI ending in period", model.contains(st18));
		assertTrue("missing statement with predicate URI ending in period", model.contains(st19));
		assertTrue("missing statement with subject URI ending in period", model.contains(st20));

		if (rdfParser.getRDFFormat().supportsContexts()) {
			assertTrue("missing statement with blank node single use subject", model.contains(st21));
		}
		else {
			assertEquals("missing statement with blank node single use subject", 1,
					model.filter(null, uri4, uri5).size());
		}
		if (rdfParser.getRDFFormat().supportsContexts()) {
			assertTrue("missing statement with blank node single use object: st22", model.contains(st22));
		}
		else {
			assertEquals("missing statement with blank node single use object: st22", 1,
					model.filter(uri4, uri5, null).size());
		}
		if (rdfParser.getRDFFormat().supportsContexts()) {
			assertTrue("missing statement with blank node use across contexts subject: st23",
					model.contains(st23));
			assertTrue("missing statement with blank node use across contexts subject: st24",
					model.contains(st24));
		}
		else {
			assertEquals("missing statement with blank node use across contexts: st23/st24", 1,
					model.filter(null, uri4, uri3).size());
		}
		if (rdfParser.getRDFFormat().supportsContexts()) {
			assertTrue("missing statement with blank node use across contexts subject and object: st25",
					model.contains(st25));
			assertTrue("missing statement with blank node use across contexts subject and object: st26",
					model.contains(st26));
		}
		else {
			assertEquals("missing statement with blank node use across contexts subject and object: st25", 1,
					model.filter(null, uri4, uri2).size());
			assertEquals("missing statement with blank node use across contexts subject and object: st26", 1,
					model.filter(uri4, uri3, null).size());
		}
		if (rdfParser.getRDFFormat().supportsContexts()) {
			assertTrue("missing statement with blank node use across contexts subject: st27",
					model.contains(st27));
			assertTrue("missing statement with blank node use across contexts subject: st28",
					model.contains(st28));
		}
		else {
			assertEquals("missing statement with blank node use across contexts: object: st27/st28", 1,
					model.filter(uri3, uri4, null).size());
		}
		if (rdfParser.getRDFFormat().supportsContexts()) {
			assertTrue("missing statement with blank node context: st29", model.contains(st29));
		}
		else {
			assertEquals("missing statement with blank node context: st29", 1,
					model.filter(uri5, uri4, uri1).size());
		}
	}

	@Test
	public void testPrefixRedefinition()
		throws RDFHandlerException, RDFParseException, IOException
	{
		String ns1 = "a:";
		String ns2 = "b:";
		String ns3 = "c:";

		IRI uri1 = vf.createIRI(ns1, "r1");
		IRI uri2 = vf.createIRI(ns2, "r2");
		IRI uri3 = vf.createIRI(ns3, "r3");
		Statement st = vf.createStatement(uri1, uri2, uri3);

		ByteArrayOutputStream out = new ByteArrayOutputStream();
		RDFWriter rdfWriter = rdfWriterFactory.getWriter(out);
{'fraction_non_alphanumeric': 0.11794015667137536, 'fraction_numerical': 0.020007705149608323, 'mean_word_length': 5.448492878436569, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4824882', 'n_tokens_mistral': 13712, 'n_tokens_neox': 12377, 'n_words': 2709}