<commit_before><?xml version="1.0" encoding="UTF-8" ?>

<schema name="capitolwords" version="1.2">
  <types>

    <!-- Default numeric field types. For faster range queries, consider the
        tint/tfloat/tlong/tdouble types. 
    -->
    <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
    <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
    <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
    <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>

    <!-- The format for this date field must be 1995-12-31T23:59:59Z, -->
   <fieldType name="date" class="solr.TrieDateField" omitNorms="true" precisionStep="0" positionIncrementGap="0"/>

<!-- The StrField type is not analyzed, but indexed/stored verbatim.  
       - StrField and TextField support an optional compressThreshold which
       limits compression (if enabled in the derived fields) to values which
       exceed a certain size (in characters).
    -->
    <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>

   <!-- General text field that splits on whitespace with some special
    delimiter handling for punctuation. has a custom stopword list and normalizes
    case -->

    <fieldType name="CRtext" class="solr.TextField" positionIncrementGap="1" >
        <analyzer>
            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" 
                    generateNumberParts="1" catenateWords="0" catenateNumbers="0" 
                    catenateAll="0" splitOnCaseChange="0"/>
            <filter class="solr.LowerCaseFilterFactory"/>
            <filter class="solr.StopFilterFactory"
                ignoreCase="true"
                words="stopwords.txt"
                enablePositionIncrements="true"
                />
        </analyzer>
    </fieldType>

    <!-- same as the above field except with a positionIncrementGap of 100 for
        multi-valued fields where we don't want distinct input fields to be
        concatenated together and inadvertently match a search query -->

    <fieldType name="CRtext_multivalued" class="solr.TextField" positionIncrementGap="100" >
        <analyzer>
            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" 
                    generateNumberParts="1" catenateWords="0" catenateNumbers="0" 
                    catenateAll="0" splitOnCaseChange="0"/>
            <filter class="solr.LowerCaseFilterFactory"/>
            <filter class="solr.StopFilterFactory"
                ignoreCase="true"
                words="stopwords.txt"
                enablePositionIncrements="true"
                />
        </analyzer>
    </fieldType>


   <!-- the ShingleFilterFactory produces n-grams. Note that stopwords are 
        removed. Ideally, we would have a stop list for n-grams. -->
    <fieldType name="bigrams" class="solr.TextField" positionIncrementGap="1" >
        <analyzer>
            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" 
                    generateNumberParts="1" catenateWords="0" catenateNumbers="0" 
                    catenateAll="0" splitOnCaseChange="0"/>
            <filter class="solr.LowerCaseFilterFactory"/>
            <filter class="solr.StopFilterFactory"
                ignoreCase="true"
                words="stopwords.txt"
                enablePositionIncrements="true"
                />
             <filter class="solr.ShingleFilterFactory" 
                minShingleSize="2"
                maxShingleSize="2" 
                outputUnigrams="false" 
            />
       </analyzer>
    </fieldType>
    <fieldType name="trigrams" class="solr.TextField" positionIncrementGap="1" >
        <analyzer>
            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" 
                    generateNumberParts="1" catenateWords="0" catenateNumbers="0" 
                    catenateAll="0" splitOnCaseChange="0"/>
            <filter class="solr.LowerCaseFilterFactory"/>
            <filter class="solr.StopFilterFactory"
                ignoreCase="true"
                words="stopwords.txt"
                enablePositionIncrements="true"
                />
             <filter class="solr.ShingleFilterFactory" 
                minShingleSize="3" 
                maxShingleSize="3" 
                outputUnigrams="false" 
            />
       </analyzer>
    </fieldType>

    <fieldType name="quadgrams" class="solr.TextField" positionIncrementGap="1" >
        <analyzer>
            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" 
                    generateNumberParts="1" catenateWords="0" catenateNumbers="0" 
                    catenateAll="0" splitOnCaseChange="0"/>
            <filter class="solr.LowerCaseFilterFactory"/>
            <filter class="solr.StopFilterFactory"
                ignoreCase="true"
                words="stopwords.txt"
                enablePositionIncrements="true"
                />
             <filter class="solr.ShingleFilterFactory" 
                minShingleSize="4" 
                maxShingleSize="4" 
                outputUnigrams="false" 
            />
       </analyzer>
    </fieldType>

    <fieldType name="pentagrams" class="solr.TextField" positionIncrementGap="1" >
        <analyzer>
            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" 
                    generateNumberParts="1" catenateWords="0" catenateNumbers="0" 
                    catenateAll="0" splitOnCaseChange="0"/>
            <filter class="solr.LowerCaseFilterFactory"/>
            <filter class="solr.StopFilterFactory"
                ignoreCase="true"
                words="stopwords.txt"
                enablePositionIncrements="true"
                />
             <filter class="solr.ShingleFilterFactory" 
                minShingleSize="5" 
                maxShingleSize="5" 
                outputUnigrams="false" 
            />
       </analyzer>
    </fieldType>


     <!-- other options we may consider: indexing in reverse as well to support more
        efficient wildcard queries, adding more flexible matching for search including
        synonyms, word delimiters, and stemming-->

    <!-- since fields of this type are by default not stored or indexed,
         any data added to them will be ignored outright.  --> 
    <fieldtype name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" /> 

 </types>

 <fields>

   <field name="id" type="string" indexed="true" stored="true" required="true" /> 
   <field name="crdoc" type="string" indexed="true" stored="true" required="true" /> 
   <field name="volume" type="int" indexed="true" stored="true" required="true" /> 
   <field name="number" type="int" indexed="true" stored="true" required="true" /> 
   <field name="date" type="date" indexed="true" stored="true" required="true" /> 
   <field name="chamber" type="string" indexed="true" stored="false" required="true" /> 
   <field name="pages" type="string" indexed="true" stored="true" required="true" /> <commit_msg>Change solr schema to not use stopwords for n-grams where n > 1<commit_after><?xml version="1.0" encoding="UTF-8" ?>

<schema name="capitolwords" version="1.2">
  <types>

    <!-- Default numeric field types. For faster range queries, consider the
        tint/tfloat/tlong/tdouble types. 
    -->
    <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
    <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
    <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
    <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>

    <!-- The format for this date field must be 1995-12-31T23:59:59Z, -->
   <fieldType name="date" class="solr.TrieDateField" omitNorms="true" precisionStep="0" positionIncrementGap="0"/>

<!-- The StrField type is not analyzed, but indexed/stored verbatim.  
       - StrField and TextField support an optional compressThreshold which
       limits compression (if enabled in the derived fields) to values which
       exceed a certain size (in characters).
    -->
    <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>

   <!-- General text field that splits on whitespace with some special
    delimiter handling for punctuation. has a custom stopword list and normalizes
    case -->

    <fieldType name="CRtext" class="solr.TextField" positionIncrementGap="1" >
        <analyzer>
            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" 
                    generateNumberParts="1" catenateWords="0" catenateNumbers="0" 
                    catenateAll="0" splitOnCaseChange="0"/>
            <filter class="solr.LowerCaseFilterFactory"/>
            <filter class="solr.StopFilterFactory"
                ignoreCase="true"
                words="stopwords.txt"
                enablePositionIncrements="true"
                />
        </analyzer>
    </fieldType>

    <!-- same as the above field except with a positionIncrementGap of 100 for
        multi-valued fields where we don't want distinct input fields to be
        concatenated together and inadvertently match a search query -->

    <fieldType name="CRtext_multivalued" class="solr.TextField" positionIncrementGap="100" >
        <analyzer>
            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" 
                    generateNumberParts="1" catenateWords="0" catenateNumbers="0" 
                    catenateAll="0" splitOnCaseChange="0"/>
            <filter class="solr.LowerCaseFilterFactory"/>
            <filter class="solr.StopFilterFactory"
                ignoreCase="true"
                words="stopwords.txt"
                enablePositionIncrements="true"
                />
        </analyzer>
    </fieldType>


   <!-- the ShingleFilterFactory produces n-grams. Note that stopwords are 
        removed. Ideally, we would have a stop list for n-grams. -->
    <fieldType name="bigrams" class="solr.TextField" positionIncrementGap="1" >
        <analyzer>
            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" 
                    generateNumberParts="1" catenateWords="0" catenateNumbers="0" 
                    catenateAll="0" splitOnCaseChange="0"/>
            <filter class="solr.LowerCaseFilterFactory"/>
            <filter class="solr.ShingleFilterFactory" 
                minShingleSize="2"
                maxShingleSize="2" 
                outputUnigrams="false" 
            />
       </analyzer>
    </fieldType>
    <fieldType name="trigrams" class="solr.TextField" positionIncrementGap="1" >
        <analyzer>
            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" 
                    generateNumberParts="1" catenateWords="0" catenateNumbers="0" 
                    catenateAll="0" splitOnCaseChange="0"/>
            <filter class="solr.LowerCaseFilterFactory"/>
            <filter class="solr.ShingleFilterFactory" 
                minShingleSize="3" 
                maxShingleSize="3" 
                outputUnigrams="false" 
            />
       </analyzer>
    </fieldType>

    <fieldType name="quadgrams" class="solr.TextField" positionIncrementGap="1" >
        <analyzer>
            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" 
                    generateNumberParts="1" catenateWords="0" catenateNumbers="0" 
                    catenateAll="0" splitOnCaseChange="0"/>
            <filter class="solr.LowerCaseFilterFactory"/>
            <filter class="solr.ShingleFilterFactory" 
                minShingleSize="4" 
                maxShingleSize="4" 
                outputUnigrams="false" 
            />
       </analyzer>
    </fieldType>

    <fieldType name="pentagrams" class="solr.TextField" positionIncrementGap="1" >
        <analyzer>
            <tokenizer class="solr.WhitespaceTokenizerFactory"/>
            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" 
                    generateNumberParts="1" catenateWords="0" catenateNumbers="0" 
                    catenateAll="0" splitOnCaseChange="0"/>
            <filter class="solr.LowerCaseFilterFactory"/>
            <filter class="solr.ShingleFilterFactory" 
                minShingleSize="5" 
                maxShingleSize="5" 
                outputUnigrams="false" 
            />
       </analyzer>
    </fieldType>


     <!-- other options we may consider: indexing in reverse as well to support more
        efficient wildcard queries, adding more flexible matching for search including
        synonyms, word delimiters, and stemming-->

    <!-- since fields of this type are by default not stored or indexed,
         any data added to them will be ignored outright.  --> 
    <fieldtype name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" /> 

 </types>

 <fields>

   <field name="id" type="string" indexed="true" stored="true" required="true" /> 
   <field name="crdoc" type="string" indexed="true" stored="true" required="true" /> 
   <field name="volume" type="int" indexed="true" stored="true" required="true" /> 
   <field name="number" type="int" indexed="true" stored="true" required="true" /> 
   <field name="date" type="date" indexed="true" stored="true" required="true" /> 
   <field name="chamber" type="string" indexed="true" stored="false" required="true" /> 
   <field name="pages" type="string" indexed="true" stored="true" required="true" /> 
{'fraction_non_alphanumeric': 0.11768707482993197, 'fraction_numerical': 0.011496598639455782, 'mean_word_length': 2.606722276741904, 'pattern_counts': {'":': 0, '<': 157, '<?xml version=': 2, '>': 158, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3659078', 'n_tokens_mistral': 4194, 'n_tokens_neox': 3949, 'n_words': 853}