<commit_before>import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.*;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class Retriever {

    private IndexSearcher searcher;
    private Analyzer standardAnalyzer;

    public Retriever(String indxDir) throws IOException, ParseException {
        IndexReader reader = DirectoryReader.open(FSDirectory.open(new File(indxDir)));
        searcher = new IndexSearcher(reader);
        standardAnalyzer = new StandardAnalyzer(Version.LUCENE_46);
    }

    // depreciated
    public void search(String searchFor, String searchInField, int maxHits) throws ParseException, IOException {
        QueryParser parser = new QueryParser(Version.LUCENE_46, searchInField, standardAnalyzer);
        Query qry = parser.parse(searchFor);
        System.out.println("Searching for: " + qry.toString());
        TopDocs results = searcher.search(qry, maxHits, new Sort(new SortField("title", SortField.Type.STRING)));
        ScoreDoc[] hits = results.scoreDocs;

        int numTotalHits = results.totalHits;
        System.out.println(numTotalHits + " total matching documents");

        for (ScoreDoc hit : hits) {
            Document doc = searcher.doc(hit.doc);
            System.out.println(doc.get("filename"));
//            System.out.printf("%.10f %s\n", hits[i].score, doc.get("filename"));
        }
    }

    public List<XmlDocument> topRelevantResults(String searchFor, int maxHits) throws ParseException, IOException {
        QueryParser parser1 = new QueryParser(Version.LUCENE_46, "title",standardAnalyzer);
        Query qry1 = parser1.parse(searchFor);
        qry1.setBoost((float) 2.0);
        QueryParser parser2 = new QueryParser(Version.LUCENE_46, "contents",standardAnalyzer);
        Query qry2 = parser2.parse(searchFor);

        BooleanQuery finalQuery = new BooleanQuery();
        finalQuery.add(qry1, BooleanClause.Occur.SHOULD);
        finalQuery.add(qry2, BooleanClause.Occur.SHOULD);

        TopDocs results = searcher.search(finalQuery, maxHits);
        ScoreDoc[] hits = results.scoreDocs;

        int numTotalHits = results.totalHits;
        System.out.println(numTotalHits + " total matching documents");

        ArrayList<XmlDocument> xmlDocuments = new ArrayList<XmlDocument>();
        for (ScoreDoc hit : hits) {
            Document doc = searcher.doc(hit.doc);
            XmlDocument xml = new XmlDocument();
            xml.setFilename(doc.get("filename"));
            xml.setTitle(doc.get("title"));
            xml.setContent(doc.get("contents"));
            xmlDocuments.add(xml);
        }

        return xmlDocuments;
    }
}
<commit_msg>Inverted index search on topic modelled cluster: implemented method, not used<commit_after>import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.*;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class Retriever {

    private IndexSearcher searcher;
    private Analyzer standardAnalyzer;
    private Analyzer whiteSpaceAnalyzer;

    public Retriever(String indxDir) throws IOException, ParseException {
        IndexReader reader = DirectoryReader.open(FSDirectory.open(new File(indxDir)));
        searcher = new IndexSearcher(reader);
        standardAnalyzer = new StandardAnalyzer(Version.LUCENE_46);
        whiteSpaceAnalyzer = new WhitespaceAnalyzer(Version.LUCENE_46);
    }

    // depreciated
    public void search(String searchFor, String searchInField, int maxHits) throws ParseException, IOException {
        QueryParser parser = new QueryParser(Version.LUCENE_46, searchInField, standardAnalyzer);
        Query qry = parser.parse(searchFor);
        System.out.println("Searching for: " + qry.toString());
        TopDocs results = searcher.search(qry, maxHits, new Sort(new SortField("title", SortField.Type.STRING)));
        ScoreDoc[] hits = results.scoreDocs;

        int numTotalHits = results.totalHits;
        System.out.println(numTotalHits + " total matching documents");

        for (ScoreDoc hit : hits) {
            Document doc = searcher.doc(hit.doc);
            System.out.println(doc.get("filename"));
//            System.out.printf("%.10f %s\n", hits[i].score, doc.get("filename"));
        }
    }

    public List<XmlDocument> topRelevantResults(String searchFor, int maxHits) throws ParseException, IOException {
        QueryParser parser1 = new QueryParser(Version.LUCENE_46, "title",standardAnalyzer);
        Query qry1 = parser1.parse(searchFor);
        qry1.setBoost((float) 2.0);
        QueryParser parser2 = new QueryParser(Version.LUCENE_46, "contents",standardAnalyzer);
        Query qry2 = parser2.parse(searchFor);

        BooleanQuery finalQuery = new BooleanQuery();
        finalQuery.add(qry1, BooleanClause.Occur.SHOULD);
        finalQuery.add(qry2, BooleanClause.Occur.SHOULD);

        TopDocs results = searcher.search(finalQuery, maxHits);
        ScoreDoc[] hits = results.scoreDocs;

        int numTotalHits = results.totalHits;
        System.out.println(numTotalHits + " total matching documents");

        ArrayList<XmlDocument> xmlDocuments = new ArrayList<XmlDocument>();
        for (ScoreDoc hit : hits) {
            Document doc = searcher.doc(hit.doc);
            XmlDocument xml = new XmlDocument();
            xml.setFilename(doc.get("filename"));
            xml.setTitle(doc.get("title"));
            xml.setContent(doc.get("contents"));
            xmlDocuments.add(xml);
        }

        return xmlDocuments;
    }

    public List<XmlDocument> searchXinY(String x, String[] y, int maxHits) throws ParseException, IOException {
        QueryParser parser1 = new QueryParser(Version.LUCENE_46, "title",standardAnalyzer);
        Query qry1 = parser1.parse(x);
        qry1.setBoost((float)2.0);
        QueryParser parser2 = new QueryParser(Version.LUCENE_46, "contents",standardAnalyzer);
        Query qry2 = parser2.parse(x);
        QueryParser parser3 = new QueryParser(Version.LUCENE_46, "filename", whiteSpaceAnalyzer);

        BooleanQuery innerQuery1 = new BooleanQuery();
        innerQuery1.add(qry1, BooleanClause.Occur.SHOULD);
        innerQuery1.add(qry2, BooleanClause.Occur.SHOULD);

        BooleanQuery innerQuery2 = new BooleanQuery();
        for(String s : y) {
            innerQuery2.add(parser3.parse(s), BooleanClause.Occur.SHOULD);
        }

        BooleanQuery finalQuery = new BooleanQuery();
        finalQuery.add(new BooleanClause(innerQuery1, BooleanClause.Occur.MUST));
        finalQuery.add(new BooleanClause(innerQuery2, BooleanClause.Occur.MUST));

        TopDocs results = searcher.search(innerQuery2, maxHits);
        ScoreDoc[] hits = results.scoreDocs;

        int numTotalHits = results.totalHits;
        System.out.println(numTotalHits + " total matching documents");

        ArrayList<XmlDocument> xmlDocuments = new ArrayList<XmlDocument>();
        for (ScoreDoc hit : hits) {
            Document doc = searcher.doc(hit.doc);
            XmlDocument xml = new XmlDocument();
            xml.setFilename(doc.get("filename"));
            xml.setTitle(doc.get("title"));
            xml.setContent(doc.get("contents"));
            xmlDocuments.add(xml);
        }

        return xmlDocuments;
    }
}

{'fraction_non_alphanumeric': 0.10870359099208765, 'fraction_numerical': 0.008642726719415702, 'mean_word_length': 3.3447911158117396, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 24}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6915800', 'n_tokens_mistral': 2455, 'n_tokens_neox': 2359, 'n_words': 468}