<commit_before>package org.ticklefish.sisot;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintWriter;

import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.TimeZone;

import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import javax.servlet.ServletConfig;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;

import org.json.JSONObject;
import org.json.JSONArray;

/**
 * Sisot indexer.
 *
 * @author escowles
 * @since 2014-07-01
**/
public class SolrIndexer extends AbstractServlet implements Runnable
{
	private SimpleDateFormat utcFormat = new SimpleDateFormat(
		"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
	);
	private SimpleDateFormat jsonFormat = new SimpleDateFormat(
		"EEE MMM dd HH:mm:ss z yyyy"
	);
	private SimpleDateFormat localFormat = new SimpleDateFormat("MM/dd HH:mm");
	private File dir;
	public void init( ServletConfig cfg )
	{
		super.init(cfg);
		init(props);
		utcFormat.setTimeZone(TimeZone.getTimeZone("UTC"));

        // schedule execution
        ScheduledExecutorService exec = new ScheduledThreadPoolExecutor(1);
        exec.scheduleWithFixedDelay(this, 0, 30, TimeUnit.MINUTES);
	}
	protected void init( Properties props )
	{
		super.init(props);
		dir = new File( props.getProperty("data.dir") );
	}

	public void run()
	{
        // make sure we're configured
        if ( solr == null )
        {
            System.out.println("Not configured, will retry in 30 minutes");
            return;
        }

		// get last tweet id
        String last = "";
        try
        {
            // lookup last-indexed tweet
            SolrQuery query = new SolrQuery();
            query.addSortField( "id", SolrQuery.ORDER.desc );
            query.setFields( "id" );
            query.setRows( 1 );
            SolrDocumentList results = solr.query(query).getResults();
            if ( results.size() > 0 )
            {
                last = (String)results.get(0).getFirstValue("id");
            }
        }
        catch ( Exception ex )
        {
            ex.printStackTrace();
        }

		index(last, new PrintWriter(System.out));
	}

	public void doGet( HttpServletRequest req, HttpServletResponse res )
	{
		if ( solr == null )
		{
			try
			{
				res.sendRedirect("config");
			}
			catch ( Exception ex )
			{
				ex.printStackTrace();
			}
			return;
		}

		PrintWriter out = null;
		try
		{
			res.setContentType("text/plain");
			out = res.getWriter();
		}
		catch ( Exception ex )
		{
			ex.printStackTrace();
			return;
		}

		int records = 0;
		out.println("reindexing");
		out.flush();

		index(null, out);
		out.flush();
	}<commit_msg>Adding incremental/full reindexing option to html indexer<commit_after>package org.ticklefish.sisot;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintWriter;

import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.TimeZone;

import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import javax.servlet.ServletConfig;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;

import org.json.JSONObject;
import org.json.JSONArray;

/**
 * Sisot indexer.
 *
 * @author escowles
 * @since 2014-07-01
**/
public class SolrIndexer extends AbstractServlet implements Runnable
{
	private SimpleDateFormat utcFormat = new SimpleDateFormat(
		"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
	);
	private SimpleDateFormat jsonFormat = new SimpleDateFormat(
		"EEE MMM dd HH:mm:ss z yyyy"
	);
	private SimpleDateFormat localFormat = new SimpleDateFormat("MM/dd HH:mm");
	private File dir;
	public void init( ServletConfig cfg )
	{
		super.init(cfg);
		init(props);
		utcFormat.setTimeZone(TimeZone.getTimeZone("UTC"));

        // schedule execution
        ScheduledExecutorService exec = new ScheduledThreadPoolExecutor(1);
        exec.scheduleWithFixedDelay(this, 0, 20, TimeUnit.MINUTES);
	}
	protected void init( Properties props )
	{
		super.init(props);
		dir = new File( props.getProperty("data.dir") );
	}

	public void run()
	{
        // make sure we're configured
        if ( solr == null )
        {
            System.out.println("Not configured, will retry in 30 minutes");
            return;
        }

		index(last(), new PrintWriter(System.out));
	}

	public void doGet( HttpServletRequest req, HttpServletResponse res )
	{
		// if we're not configured, redirect to config form
		if ( solr == null )
		{
			try
			{
				res.sendRedirect("config");
			}
			catch ( Exception ex )
			{
				ex.printStackTrace();
			}
			return;
		}

		// setup output
		PrintWriter out = null;
		try
		{
			res.setContentType("text/html");
			out = res.getWriter();
		}
		catch ( Exception ex )
		{
			ex.printStackTrace();
			return;
		}

		// output html header
		out.println("<html>");
		out.println("<head>");
		out.println("<link rel=\"stylesheet\" href=\"sisot.css\"></link>");
		out.println("</head>");
		out.println("<body>");
		out.println("<h1>indexer</h1>");
		out.println("<form method=\"GET\">");
		out.println("<input type=\"submit\" name=\"reindex\" value=\"reindex all records\"/>");
		out.println("<input type=\"submit\" value=\"index new records\"/>");
		out.println("</form>");
		out.println("<pre>");
		out.println("reindexing");
		out.flush();

		// check reindex parameter to trigger full reindex
		String last = null;
		String reindex = req.getParameter("reindex");
		if ( reindex == null || reindex.trim().equals("") )
		{
			last = last();
		}
		index(last, out);

		// finish output
		out.println("</pre>");
		out.println("</body></html>");
		out.flush();
	}

	private String last()
	{
		// get last tweet id
        String last = "";
        try
        {
            // lookup last-indexed tweet
            SolrQuery query = new SolrQuery();
            query.addSortField( "id", SolrQuery.ORDER.desc );
            query.setFields( "id" );
            query.setRows( 1 );
            SolrDocumentList results = solr.query(query).getResults();
            if ( results.size() > 0 )
            {
                last = (String)results.get(0).getFirstValue("id");
            }
        }
        catch ( Exception ex )
        {
            ex.printStackTrace();
        }
		return last;
	}
{'fraction_non_alphanumeric': 0.13083470620538165, 'fraction_numerical': 0.005079626578802856, 'mean_word_length': 3.0630228667038484, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4144819', 'n_tokens_mistral': 2472, 'n_tokens_neox': 2275, 'n_words': 487}