<commit_before>package de.ids_mannheim.korap;

import java.util.*;
import java.io.*;
import java.net.URLDecoder;

import static org.junit.Assert.*;

import de.ids_mannheim.korap.KrillQuery;
import de.ids_mannheim.korap.query.QueryBuilder;
import de.ids_mannheim.korap.index.*;
import de.ids_mannheim.korap.query.wrap.SpanQueryWrapper;
import de.ids_mannheim.korap.util.QueryException;
import de.ids_mannheim.korap.util.CorpusDataException;

import static de.ids_mannheim.korap.util.KrillByte.*;

import org.apache.lucene.index.*;
import org.apache.lucene.document.*;
import org.apache.lucene.search.spans.Spans;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.util.Bits;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Helper class for testing the KrillIndex framework (Simple).
 * 
 * @author diewald
 */
public class TestSimple {

    private static Logger log  = LoggerFactory.getLogger(TestSimple.class);
    
    // Add document
    public static void addDoc (IndexWriter w, Map<String, String> m)
            throws IOException {
        Document doc = new Document();

        FieldType textFieldWithTermVectors = new FieldType(
                TextField.TYPE_STORED);
        textFieldWithTermVectors.setStoreTermVectors(true);
        /*
          No offsets are stored.
          textFieldWithTermVectors.setStoreTermVectorOffsets(true);
        */
        textFieldWithTermVectors.setStoreTermVectorPositions(true);
        textFieldWithTermVectors.setStoreTermVectorPayloads(true);

        Field textFieldAnalyzed = new Field("text", m.get("textStr"),
                textFieldWithTermVectors);

        MultiTermTokenStream ts = getTermVector(m.get("text"));

        textFieldAnalyzed.setTokenStream(ts);

        doc.add(textFieldAnalyzed);

        // Add document to writer
        w.addDocument(doc);
    };


    // Get Term Vector
    public static MultiTermTokenStream getTermVector (String stream) {
        MultiTermTokenStream ts = new MultiTermTokenStream();

        int pos = 0;
        for (String seg : stream.split(" ")) {
            //	    System.err.println("** Prepare " + seg);
            String[] tokens = seg.split("\\|");

            int i = 0;

            while (tokens[i].length() == 0)
                i++;

            try {
                MultiTermToken mtt = new MultiTermToken(tokens[i]);
                //	    System.err.println("** Add term " + tokens[i]);
                i++;
                for (; i < tokens.length; i++) {
                    if (tokens[i].length() == 0)
                        continue;
                    mtt.add(tokens[i]);
                };
                ts.addMultiTermToken(mtt);<commit_msg>Adding helper methods for fuzzing<commit_after>package de.ids_mannheim.korap;

import java.util.*;
import java.io.*;
import java.net.URLDecoder;

import static org.junit.Assert.*;

import de.ids_mannheim.korap.KrillQuery;
import de.ids_mannheim.korap.query.QueryBuilder;
import de.ids_mannheim.korap.index.*;
import de.ids_mannheim.korap.query.wrap.SpanQueryWrapper;
import de.ids_mannheim.korap.util.QueryException;
import de.ids_mannheim.korap.util.CorpusDataException;

import static de.ids_mannheim.korap.util.KrillByte.*;

import org.apache.lucene.index.*;
import org.apache.lucene.document.*;
import org.apache.lucene.search.spans.Spans;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.util.Bits;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Helper class for testing the KrillIndex framework (Simple).
 * 
 * @author diewald
 */
public class TestSimple {

    private static Logger log  = LoggerFactory.getLogger(TestSimple.class);
    
    // Add document
    public static void addDoc (IndexWriter w, Map<String, String> m)
            throws IOException {
        Document doc = new Document();

        FieldType textFieldWithTermVectors = new FieldType(
                TextField.TYPE_STORED);
        textFieldWithTermVectors.setStoreTermVectors(true);
        /*
          No offsets are stored.
          textFieldWithTermVectors.setStoreTermVectorOffsets(true);
        */
        textFieldWithTermVectors.setStoreTermVectorPositions(true);
        textFieldWithTermVectors.setStoreTermVectorPayloads(true);

        Field textFieldAnalyzed = new Field("text", m.get("textStr"),
                textFieldWithTermVectors);

        MultiTermTokenStream ts = getTermVector(m.get("text"));

        textFieldAnalyzed.setTokenStream(ts);

        doc.add(textFieldAnalyzed);

        // Add document to writer
        w.addDocument(doc);
    };

    // Add document
    public static FieldDocument simpleFieldDoc (String s) {
        String[] characters = s.split("");

        FieldDocument fd = new FieldDocument();
        String surface = "";
        String annotation = "";

        for (int i = 0; i < characters.length; i++) {
            String fixChar = characters[i];
            surface += fixChar;
            annotation +=
                "[("+i+"-"+(i+1)+")s:"+fixChar+
                "|_"+i+"$<i>"+i+"<i>"+(i+1)+"]";
        };

        fd.addTV("base",surface, annotation);
        return fd;
    };

    public static FieldDocument simpleFuzzyFieldDoc (List<String> chars, int minLength, int maxLength) {
        String surface = "";

        for (int i = 0; i < (int)(Math.random() * (maxLength - minLength)) + minLength; i++) {
            String randomChar = chars.get((int)(Math.random() * 6));
            surface += randomChar;
        };
        return simpleFieldDoc(surface);

    };

    // Get Term Vector
    public static MultiTermTokenStream getTermVector (String stream) {
        MultiTermTokenStream ts = new MultiTermTokenStream();

        int pos = 0;
        for (String seg : stream.split(" ")) {
            //	    System.err.println("** Prepare " + seg);
            String[] tokens = seg.split("\\|");

            int i = 0;

            while (tokens[i].length() == 0)
                i++;

            try {
                MultiTermToken mtt = new MultiTermToken(tokens[i]);
                //	    System.err.println("** Add term " + tokens[i]);
                i++;
                for (; i < tokens.length; i++) {
                    if (tokens[i].length() == 0)
                        continue;
                    mtt.add(tokens[i]);
                };
                ts.addMultiTermToken(mtt);
{'fraction_non_alphanumeric': 0.11925891328039857, 'fraction_numerical': 0.0026467382842908296, 'mean_word_length': 2.358076319916362, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7552949', 'n_tokens_mistral': 1932, 'n_tokens_neox': 1850, 'n_words': 385}