<commit_before>package org.petapico.nanopub.indexer;

import java.io.IOException;
import java.io.InputStream;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.zip.GZIPInputStream;

import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClientBuilder;
import org.nanopub.MultiNanopubRdfHandler;
import org.nanopub.Nanopub;
import org.nanopub.NanopubUtils;
import org.nanopub.MultiNanopubRdfHandler.NanopubHandler;
import org.nanopub.extra.server.GetNanopub;
import org.nanopub.extra.server.NanopubServerUtils;
import org.nanopub.extra.server.ServerInfo;
import org.nanopub.extra.server.ServerIterator;
import org.openrdf.model.Resource;
import org.openrdf.model.Statement;
import org.openrdf.model.URI;
import org.openrdf.model.Value;
import org.openrdf.rio.RDFFormat;
import org.openrdf.rio.RDFHandlerException;

public class Indexer {
	public static final int SECTION_HEAD = 1;
	public static final int SECTION_ASSERTION = 2;
	public static final int SECTION_PROVENANCE = 3;
	public static final int SECTION_PUBINFO = 4;

	NanopubDatabase db = null;
	
	public static List<Nanopub> nanopubs; //used by the callback function of the MultiNanopubRdfHandler class -> can we do this better?
	
	public static void main(String[] args) {
		
		/*
		args = new String[3];
		args[0] = "root";
		args[1] = "admin";
		args[2] = "true";
		*/
		
		if (args.length < 2){
			System.out.printf("Invalid arguments expected: dbusername, dbpassword\n");
			System.exit(1);
		}
		
		while (true){
			try {
				Indexer indexer = new Indexer(args[0], args[1]);
				indexer.run();
			}
			catch (Exception E){
				System.out.println("Run error");
				System.out.println(E.getMessage());
			}
			// sleep ?
		}
	}

	public Indexer(String dbusername, String dbpassword) throws ClassNotFoundException, SQLException {
		db = new NanopubDatabase(dbusername, dbpassword);
	}

	public void run() throws IOException, RDFHandlerException, Exception {
		ServerIterator serverIterator = new ServerIterator();

		while (serverIterator.hasNext()) {<commit_msg>some debug options<commit_after>package org.petapico.nanopub.indexer;

import java.io.IOException;
import java.io.InputStream;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.zip.GZIPInputStream;

import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClientBuilder;
import org.nanopub.MultiNanopubRdfHandler;
import org.nanopub.Nanopub;
import org.nanopub.NanopubUtils;
import org.nanopub.MultiNanopubRdfHandler.NanopubHandler;
import org.nanopub.extra.server.GetNanopub;
import org.nanopub.extra.server.NanopubServerUtils;
import org.nanopub.extra.server.ServerInfo;
import org.nanopub.extra.server.ServerIterator;
import org.openrdf.model.Resource;
import org.openrdf.model.Statement;
import org.openrdf.model.URI;
import org.openrdf.model.Value;
import org.openrdf.rio.RDFFormat;
import org.openrdf.rio.RDFHandlerException;

public class Indexer {
	public static final int SECTION_HEAD = 1;
	public static final int SECTION_ASSERTION = 2;
	public static final int SECTION_PROVENANCE = 3;
	public static final int SECTION_PUBINFO = 4;

	NanopubDatabase db = null;
	
	public static List<Nanopub> nanopubs; //used by the callback function of the MultiNanopubRdfHandler class -> can we do this better?
	
	public static void main(String[] args) {
		
		/*
		args = new String[3];
		args[0] = "root";
		args[1] = "admin";
		args[2] = "true";
		*/
		
		if (args.length < 2){
			System.out.printf("Invalid arguments expected: dbusername, dbpassword\n");
			System.exit(1);
		}
		
		while (true){
			try {
				Indexer indexer = new Indexer(args[0], args[1]);
				indexer.run();
			}
			catch (Exception E){
				System.out.println("Run error");
				System.out.println(E.getMessage());
			}
			// sleep ?
		}
	}

	public Indexer(String dbusername, String dbpassword) throws ClassNotFoundException, SQLException {
		db = new NanopubDatabase(dbusername, dbpassword);
		System.out.printf("db connected '%s' '%s'\n", dbusername, dbpassword);
	}

	public void run() throws IOException, RDFHandlerException, Exception {
		ServerIterator serverIterator = new ServerIterator();

		while (serverIterator.hasNext()) {
{'fraction_non_alphanumeric': 0.11252653927813164, 'fraction_numerical': 0.005095541401273885, 'mean_word_length': 4.801724137931035, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3431774', 'n_tokens_mistral': 1678, 'n_tokens_neox': 1532, 'n_words': 348}