<commit_before><?xml version="1.0" encoding="UTF-8" ?>

<schema name="panda" version="1.4">

    <types>

        <!-- Basic types -->
        <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true" />
        <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true" />
        <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0" />
        <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0" />
        <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0" />
        <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" positionIncrementGap="0" />
        <fieldType name="date" class="solr.TrieDateField" omitNorms="true" precisionStep="0" positionIncrementGap="0" />

        <!-- Fast types -->
        <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" omitNorms="true" positionIncrementGap="0" />
        <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" omitNorms="true" positionIncrementGap="0" />
        <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" omitNorms="true" positionIncrementGap="0" />
        <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" omitNorms="true" positionIncrementGap="0" />
        <fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" precisionStep="6" positionIncrementGap="0" />

        <!-- Full-text types -->
        <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100">
          <analyzer type="index">
            <tokenizer class="solr.StandardTokenizerFactory" />
                <filter class="solr.LowerCaseFilterFactory" />
            </analyzer>
            <analyzer type="query">
                <tokenizer class="solr.StandardTokenizerFactory" />
                <filter class="solr.LowerCaseFilterFactory" />
            </analyzer>
        </fieldType>

        <!-- Geographic types -->
        <!--<fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate" />-->

        <!-- Miscellaneous types -->
        <fieldtype name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" />

    </types>

    <fields>

        <!-- Static fields -->
        <field name="id" type="string" indexed="true" stored="true" required="true" />
        <field name="dataset_id" type="string" indexed="true" stored="true" required="true" />
        <field name="row" type="int" indexed="true" stored="true" required="true" />
        <field name="full_text" type="text_general" indexed="true" stored="false" />
        <field name="csv_data" type="string" indexed="false" stored="true" required="true" />

        <!-- Dynamic typed fields that directly correspond to Python types -->
        <dynamicField name="*_unicode"  type="string"  indexed="true"  stored="true" />
        <dynamicField name="*_int"  type="long"   indexed="true"  stored="true" />
        <dynamicField name="*_bool"  type="boolean" indexed="true"  stored="true" />
        <dynamicField name="*_float"  type="float"  indexed="true"  stored="true" />
        <dynamicField name="*_none" type="ignored" />

        <!-- Dynamic, typed fileds that correspond to Python types after some processing -->
        <dynamicField name="*_datetime" type="date"    indexed="true"  stored="true" />
        <dynamicField name="*_date" type="string"    indexed="true"  stored="true" />

        <!-- Dynamic, typed fields that don't correspond to Python types -->
        <dynamicField name="*_time" type="string"    indexed="true"  stored="true" />
        <!--<dynamicField name="*_loc"  type="location" indexed="true" stored="true" />-->

    </fields>

    <uniqueKey>id</uniqueKey>
    <defaultSearchField>full_text</defaultSearchField>
    <solrQueryParser defaultOperator="OR"/>
<commit_msg>Fix column name in solr config.<commit_after><?xml version="1.0" encoding="UTF-8" ?>

<schema name="panda" version="1.4">

    <types>

        <!-- Basic types -->
        <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true" />
        <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true" />
        <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0" />
        <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0" />
        <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0" />
        <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" positionIncrementGap="0" />
        <fieldType name="date" class="solr.TrieDateField" omitNorms="true" precisionStep="0" positionIncrementGap="0" />

        <!-- Fast types -->
        <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" omitNorms="true" positionIncrementGap="0" />
        <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" omitNorms="true" positionIncrementGap="0" />
        <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" omitNorms="true" positionIncrementGap="0" />
        <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" omitNorms="true" positionIncrementGap="0" />
        <fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" precisionStep="6" positionIncrementGap="0" />

        <!-- Full-text types -->
        <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100">
          <analyzer type="index">
            <tokenizer class="solr.StandardTokenizerFactory" />
                <filter class="solr.LowerCaseFilterFactory" />
            </analyzer>
            <analyzer type="query">
                <tokenizer class="solr.StandardTokenizerFactory" />
                <filter class="solr.LowerCaseFilterFactory" />
            </analyzer>
        </fieldType>

        <!-- Geographic types -->
        <!--<fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate" />-->

        <!-- Miscellaneous types -->
        <fieldtype name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" />

    </types>

    <fields>

        <!-- Static fields -->
        <field name="id" type="string" indexed="true" stored="true" required="true" />
        <field name="dataset_id" type="string" indexed="true" stored="true" required="true" />
        <field name="row" type="int" indexed="true" stored="true" required="true" />
        <field name="full_text" type="text_general" indexed="true" stored="false" />
        <field name="data" type="string" indexed="false" stored="true" required="true" />

        <!-- Dynamic typed fields that directly correspond to Python types -->
        <dynamicField name="*_unicode"  type="string"  indexed="true"  stored="true" />
        <dynamicField name="*_int"  type="long"   indexed="true"  stored="true" />
        <dynamicField name="*_bool"  type="boolean" indexed="true"  stored="true" />
        <dynamicField name="*_float"  type="float"  indexed="true"  stored="true" />
        <dynamicField name="*_none" type="ignored" />

        <!-- Dynamic, typed fileds that correspond to Python types after some processing -->
        <dynamicField name="*_datetime" type="date"    indexed="true"  stored="true" />
        <dynamicField name="*_date" type="string"    indexed="true"  stored="true" />

        <!-- Dynamic, typed fields that don't correspond to Python types -->
        <dynamicField name="*_time" type="string"    indexed="true"  stored="true" />
        <!--<dynamicField name="*_loc"  type="location" indexed="true" stored="true" />-->

    </fields>

    <uniqueKey>id</uniqueKey>
    <defaultSearchField>full_text</defaultSearchField>
    <solrQueryParser defaultOperator="OR"/>

{'fraction_non_alphanumeric': 0.16514553527380366, 'fraction_numerical': 0.006906758756783424, 'mean_word_length': 3.641671436748712, 'pattern_counts': {'":': 0, '<': 123, '<?xml version=': 2, '>': 123, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1321399', 'n_tokens_mistral': 2445, 'n_tokens_neox': 2394, 'n_words': 457}