<commit_before>package de.danielbasedow.prospecter.core;

import com.google.inject.Guice;
import com.google.inject.Injector;
import de.danielbasedow.prospecter.core.analysis.Analyzer;
import de.danielbasedow.prospecter.core.analysis.filters.NormalizeWhiteSpaceFilter;
import de.danielbasedow.prospecter.core.analysis.filters.RemoveNonAlphaNumFilter;
import de.danielbasedow.prospecter.core.analysis.filters.ToLowerCaseFilter;
import de.danielbasedow.prospecter.core.document.Document;
import de.danielbasedow.prospecter.core.document.DocumentBuilder;
import de.danielbasedow.prospecter.core.document.TextField;
import de.danielbasedow.prospecter.core.index.FullTextIndex;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;

public class Application {
    public Application() {

    }

    public static Schema buildSchema() {
        Schema schema = new SchemaImpl();
        schema.addFieldIndex("_all", new FullTextIndex("_all"));
        return schema;
    }

    public static Document buildDoc(DocumentBuilder builder, String query) {
        HashMap<String, String> rawFields = new HashMap<String, String>();
        rawFields.put("_all", query);
        return builder.build(rawFields);
    }


    public static void main(String[] args) {
        Injector injector = Guice.createInjector(new ProspecterModule());
        QueryManager queryManager = injector.getInstance(QueryManager.class);
        QueryBuilder queryBuilder = injector.getInstance(QueryBuilder.class);
        Schema schema = buildSchema();
        DocumentBuilder docBuilder = injector.getInstance(DocumentBuilder.class);
        HashMap<Long, String> rawQueries = new HashMap<Long, String>();

        Analyzer analyzer = queryBuilder.getAnalyzer();
        analyzer.addFilter(new NormalizeWhiteSpaceFilter());
        analyzer.addFilter(new ToLowerCaseFilter());

        try {
            BufferedReader br = new BufferedReader(new FileReader(new File(args[0])));
            String line;
            long i = 0;
            System.out.println("start indexing " + (new Date()).getTime());
            while ((line = br.readLine()) != null) {
                String[] columns = line.trim().split("\\t");
                i++;
                if (columns.length == 3) {
                    rawQueries.put(i, columns[2].trim());
                    Query q = queryBuilder.buildFromString(i, columns[2].trim());
                    queryManager.addQuery(q);
                    schema.addPostingsToField("_all", q.getPostings());
                }
            }
            br.close();
            System.out.println("indexing done " + (new Date()).getTime());
            System.out.println(i);
            BufferedReader testDoc = new BufferedReader(new FileReader(new File(args[1])));
            String queryStr = "";
            while ((line = testDoc.readLine()) != null) {
                queryStr = queryStr + " " + line;
            }
            //Document doc = buildDoc(docBuilder, "yahoo search is the part of the log");
            System.out.println("start matching " + (new Date()).getTime());
            Document doc = buildDoc(docBuilder, queryStr);
            Matcher matcher = injector.getInstance(Matcher.class);
            schema.matchDocument(doc, matcher);

            //matcher.printResultStats();
            ArrayList<Query> queries = matcher.getMatchedQueries();
            System.out.println("matching done " + (new Date()).getTime());


            System.out.println("Queries returned: " + Integer.toString(queries.size()));
            for (Query query : queries) {
                System.out.print(query.getQueryId());
                System.out.print(",");
                //System.out.println(": " + rawQueries.get(query.getQueryId()));
            }
            printVMStats();
        } catch (IOException e) {
            e.printStackTrace();
        } catch (UndefinedIndexFieldException e) {
            e.printStackTrace();
        }
        System.out.println(injector.getInstance(TokenMapper.class).getNewTermId());
    }

    public static void printVMStats() {
        Runtime runtime = Runtime.getRuntime();
        System.out.println("used: " + (runtime.totalMemory() - runtime.freeMemory()));
    }<commit_msg>stop tracking raw queries in test application<commit_after>package de.danielbasedow.prospecter.core;

import com.google.inject.Guice;
import com.google.inject.Injector;
import de.danielbasedow.prospecter.core.analysis.Analyzer;
import de.danielbasedow.prospecter.core.analysis.filters.NormalizeWhiteSpaceFilter;
import de.danielbasedow.prospecter.core.analysis.filters.RemoveNonAlphaNumFilter;
import de.danielbasedow.prospecter.core.analysis.filters.ToLowerCaseFilter;
import de.danielbasedow.prospecter.core.document.Document;
import de.danielbasedow.prospecter.core.document.DocumentBuilder;
import de.danielbasedow.prospecter.core.document.TextField;
import de.danielbasedow.prospecter.core.index.FullTextIndex;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;

public class Application {
    public Application() {

    }

    public static Schema buildSchema() {
        Schema schema = new SchemaImpl();
        schema.addFieldIndex("_all", new FullTextIndex("_all"));
        return schema;
    }

    public static Document buildDoc(DocumentBuilder builder, String query) {
        HashMap<String, String> rawFields = new HashMap<String, String>();
        rawFields.put("_all", query);
        return builder.build(rawFields);
    }


    public static void main(String[] args) {
        Injector injector = Guice.createInjector(new ProspecterModule());
        QueryManager queryManager = injector.getInstance(QueryManager.class);
        QueryBuilder queryBuilder = injector.getInstance(QueryBuilder.class);
        Schema schema = buildSchema();
        DocumentBuilder docBuilder = injector.getInstance(DocumentBuilder.class);

        Analyzer analyzer = queryBuilder.getAnalyzer();
        analyzer.addFilter(new NormalizeWhiteSpaceFilter());
        analyzer.addFilter(new ToLowerCaseFilter());

        try {
            BufferedReader br = new BufferedReader(new FileReader(new File(args[0])));
            String line;
            long i = 0;
            System.out.println("start indexing " + (new Date()).getTime());
            while ((line = br.readLine()) != null) {
                String[] columns = line.trim().split("\\t");
                i++;
                if (columns.length == 3) {
                    Query q = queryBuilder.buildFromString(i, columns[2].trim());
                    queryManager.addQuery(q);
                    schema.addPostingsToField("_all", q.getPostings());
                }
            }
            br.close();
            System.out.println("indexing done " + (new Date()).getTime());
            System.out.println(i);
            BufferedReader testDoc = new BufferedReader(new FileReader(new File(args[1])));
            String queryStr = "";
            while ((line = testDoc.readLine()) != null) {
                queryStr = queryStr + " " + line;
            }
            //Document doc = buildDoc(docBuilder, "yahoo search is the part of the log");
            System.out.println("start matching " + (new Date()).getTime());
            Document doc = buildDoc(docBuilder, queryStr);
            Matcher matcher = injector.getInstance(Matcher.class);
            schema.matchDocument(doc, matcher);

            //matcher.printResultStats();
            ArrayList<Query> queries = matcher.getMatchedQueries();
            System.out.println("matching done " + (new Date()).getTime());


            System.out.println("Queries returned: " + Integer.toString(queries.size()));
            for (Query query : queries) {
                System.out.print(query.getQueryId());
                System.out.print(",");
                //System.out.println(": " + rawQueries.get(query.getQueryId()));
            }
            printVMStats();
        } catch (IOException e) {
            e.printStackTrace();
        } catch (UndefinedIndexFieldException e) {
            e.printStackTrace();
        }
        System.out.println(injector.getInstance(TokenMapper.class).getNewTermId());
    }

    public static void printVMStats() {
        Runtime runtime = Runtime.getRuntime();
        System.out.println("used: " + (runtime.totalMemory() - runtime.freeMemory()));
        runtime.gc();
        System.out.println("used: " + (runtime.totalMemory() - runtime.freeMemory()));
    }
{'fraction_non_alphanumeric': 0.12267275842375785, 'fraction_numerical': 0.0012564249000571101, 'mean_word_length': 2.6437786100707448, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3435326', 'n_tokens_mistral': 2363, 'n_tokens_neox': 2311, 'n_words': 452}