<commit_before>package com.jbalint.bs.bookstore;

import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Date;

import com.complexible.common.openrdf.util.ModelBuilder;
import com.complexible.common.openrdf.util.ResourceBuilder;
import com.complexible.common.rdf.model.StardogValueFactory;
import com.complexible.stardog.api.Connection;
import com.complexible.stardog.api.ConnectionConfiguration;
import com.jbalint.bs.ontology.BsLib;
import com.jbalint.bs.ontology.Nepomuk;
import org.apache.commons.codec.digest.DigestUtils;
import org.openrdf.model.IRI;
import org.openrdf.model.Model;

/**
 * Generate metadata structures for Bookstore db.
 */
public class FileMetadataGenerator {
	static final StardogValueFactory VF = StardogValueFactory.instance();

	Connection conn;
	Path root;
	IRI context;

	public FileMetadataGenerator(Connection conn, Path root, IRI context) {
		this.conn = conn;
		this.root = root;
		this.context = context;
	}

	/**
	 * Does an entry for the corresponding path exist in the DB?
	 */
	boolean existsInDb(Path p) {
		return conn.get().context(context).predicate(Nepomuk.NIE.url).object(getIRI(p)).statements().findFirst().isPresent();
	}

	/**
	 * Turn a Path into a model if it doesn't already exist in the DB and it's a regular file
	 */
	Model file(Path p) {
		try {
			Files.getLastModifiedTime(p);
			ResourceBuilder mainFile = new ModelBuilder()
				                           .instance(Nepomuk.NFO.RemoteDataObject)
				                           .addProperty(Nepomuk.NIE.url, getIRI(p))
				                           .addProperty(BsLib.relativePath, root.relativize(p).toString())
				                           .addProperty(Nepomuk.NIE.byteSize, Files.size(p))
				                           .addProperty(Nepomuk.NIE.lastRefreshed, new Date())
				                           .addProperty(Nepomuk.NFO.fileCreated, new Date(Files.getLastModifiedTime(p).toMillis()))
				                           .addProperty(Nepomuk.NFO.fileName, p.getFileName().toString())
				                           .addProperty(Nepomuk.NFO.hasHash, new ModelBuilder().instance(Nepomuk.NFO.FileHash)
				                                                                               .addProperty(Nepomuk.NFO.hashAlgorithm, "MD5")
				                                                                               .addProperty(Nepomuk.NFO.hashValue,
				                                                                                            DigestUtils.md5Hex(Files.newInputStream(p))));
			ResourceBuilder infoElement = new ModelBuilder()
				                              .instance(Nepomuk.NIE.InformationElement)
				                              .addProperty(StardogValueFactory.RDF.TYPE, BsLib.Book)
				                              .addProperty(Nepomuk.NIE.isStoredAs, mainFile);

			return infoElement.model();
		}
		catch (IOException theE) {<commit_msg>Use ask() instead of a truncated statement stream (fixes [BS-50])<commit_after>package com.jbalint.bs.bookstore;

import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Date;

import com.complexible.common.openrdf.util.ModelBuilder;
import com.complexible.common.openrdf.util.ResourceBuilder;
import com.complexible.common.rdf.model.StardogValueFactory;
import com.complexible.stardog.api.Connection;
import com.complexible.stardog.api.ConnectionConfiguration;
import com.jbalint.bs.ontology.BsLib;
import com.jbalint.bs.ontology.Nepomuk;
import org.apache.commons.codec.digest.DigestUtils;
import org.openrdf.model.IRI;
import org.openrdf.model.Model;

/**
 * Generate metadata structures for Bookstore db.
 */
public class FileMetadataGenerator {
	static final StardogValueFactory VF = StardogValueFactory.instance();

	Connection conn;
	Path root;
	IRI context;

	public FileMetadataGenerator(Connection conn, Path root, IRI context) {
		this.conn = conn;
		this.root = root;
		this.context = context;
	}

	/**
	 * Does an entry for the corresponding path exist in the DB?
	 */
	boolean existsInDb(Path p) {
		return conn.get().context(context).predicate(Nepomuk.NIE.url).object(getIRI(p)).ask();
	}

	/**
	 * Turn a Path into a model if it doesn't already exist in the DB and it's a regular file
	 */
	Model file(Path p) {
		try {
			Files.getLastModifiedTime(p);
			ResourceBuilder mainFile = new ModelBuilder()
				                           .instance(Nepomuk.NFO.RemoteDataObject)
				                           .addProperty(Nepomuk.NIE.url, getIRI(p))
				                           .addProperty(BsLib.relativePath, root.relativize(p).toString())
				                           .addProperty(Nepomuk.NIE.byteSize, Files.size(p))
				                           .addProperty(Nepomuk.NIE.lastRefreshed, new Date())
				                           .addProperty(Nepomuk.NFO.fileCreated, new Date(Files.getLastModifiedTime(p).toMillis()))
				                           .addProperty(Nepomuk.NFO.fileName, p.getFileName().toString())
				                           .addProperty(Nepomuk.NFO.hasHash, new ModelBuilder().instance(Nepomuk.NFO.FileHash)
				                                                                               .addProperty(Nepomuk.NFO.hashAlgorithm, "MD5")
				                                                                               .addProperty(Nepomuk.NFO.hashValue,
				                                                                                            DigestUtils.md5Hex(Files.newInputStream(p))));
			ResourceBuilder infoElement = new ModelBuilder()
				                              .instance(Nepomuk.NIE.InformationElement)
				                              .addProperty(StardogValueFactory.RDF.TYPE, BsLib.Book)
				                              .addProperty(Nepomuk.NIE.isStoredAs, mainFile);

			return infoElement.model();
		}
		catch (IOException theE) {
{'fraction_non_alphanumeric': 0.1002493765586035, 'fraction_numerical': 0.000997506234413965, 'mean_word_length': 2.0740929994890136, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1221499', 'n_tokens_mistral': 1856, 'n_tokens_neox': 1686, 'n_words': 307}