<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/. */

package lux.index;

import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;

import javax.xml.stream.XMLStreamException;

import lux.xml.JDOMBuilder;
import lux.xml.XmlReader;

import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.Version;
import org.jdom.Document;
import org.jdom.filter.ContentFilter;
import org.jdom.output.XMLOutputter;

public class XmlIndexer {
    
    private XmlReader xmlReader;
    private JDOMBuilder jdomBuilder;
    private XMLOutputter jdomSerializer;
    private XmlPathMapper pathMapper;
    private List<XmlField> fields = new ArrayList<XmlField>();
    private MultiFieldAnalyzer fieldAnalyzers;
    
    private int options = 0;
    
    private static final Version luceneVersion = Version.LUCENE_34;
    
    public XmlIndexer () {
        this (DEFAULT_OPTIONS);
    }
    
    public final static int BUILD_JDOM=1;
    public final static int SERIALIZE_XML=2;
    public final static int NAMESPACE_AWARE=4;
    public final static int STORE_XML=8;
    public final static int STORE_PTREE=16;
    public final static int INDEX_QNAMES=32;
    public final static int INDEX_PATHS=64;
    public final static int INDEX_FULLTEXT=128;
    public final static int DEFAULT_OPTIONS = BUILD_JDOM | STORE_XML | INDEX_QNAMES | INDEX_PATHS;
    
    public XmlIndexer (int options) {
        this.options = options;
        fieldAnalyzers = new MultiFieldAnalyzer();
        xmlReader = new XmlReader();        
        // accumulate XML paths and QNames for indexing
        pathMapper = new XmlPathMapper();
        xmlReader.addHandler (pathMapper);<commit_msg>INDEXES constant for selecting whether any indexes are configured<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/. */

package lux.index;

import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;

import javax.xml.stream.XMLStreamException;

import lux.xml.JDOMBuilder;
import lux.xml.XmlReader;

import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.Version;
import org.jdom.Document;
import org.jdom.filter.ContentFilter;
import org.jdom.output.XMLOutputter;

public class XmlIndexer {
    
    private XmlReader xmlReader;
    private JDOMBuilder jdomBuilder;
    private XMLOutputter jdomSerializer;
    private XmlPathMapper pathMapper;
    private List<XmlField> fields = new ArrayList<XmlField>();
    private MultiFieldAnalyzer fieldAnalyzers;
    
    private int options = 0;
    
    private static final Version luceneVersion = Version.LUCENE_34;
    
    public XmlIndexer () {
        this (DEFAULT_OPTIONS);
    }
    
    public final static int BUILD_JDOM=1;
    public final static int SERIALIZE_XML=2;
    public final static int NAMESPACE_AWARE=4;
    public final static int STORE_XML=8;
    public final static int STORE_PTREE=16;
    public final static int INDEX_QNAMES=32;
    public final static int INDEX_PATHS=64;
    public final static int INDEX_FULLTEXT=128;
    public final static int INDEXES = INDEX_QNAMES | INDEX_PATHS | INDEX_FULLTEXT;
    public final static int DEFAULT_OPTIONS = BUILD_JDOM | STORE_XML | INDEX_QNAMES | INDEX_PATHS;

    
    public XmlIndexer (int options) {
        this.options = options;
        fieldAnalyzers = new MultiFieldAnalyzer();
        xmlReader = new XmlReader();        
        // accumulate XML paths and QNames for indexing
        pathMapper = new XmlPathMapper();
        xmlReader.addHandler (pathMapper);
{'fraction_non_alphanumeric': 0.07751605995717345, 'fraction_numerical': 0.008565310492505354, 'mean_word_length': 3.820433436532508, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4540948', 'n_tokens_mistral': 1455, 'n_tokens_neox': 1380, 'n_words': 397}