<commit_before>package store.server.service;

import com.google.common.base.Optional;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import static com.google.common.io.BaseEncoding.base16;
import java.io.IOException;
import java.io.InputStream;
import static java.lang.Math.min;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import static java.util.Collections.emptyList;
import java.util.HashSet;
import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.DateTools.Resolution;
import static org.apache.lucene.document.DateTools.timeToString;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.SingleInstanceLockFactory;
import org.apache.lucene.util.Version;
import org.apache.tika.Tika;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import store.common.ContentInfo;
import store.common.ContentInfoTree;
import store.common.IndexEntry;
import store.common.hash.Hash;
import store.common.value.Value;
import store.server.exception.BadRequestException;
import store.server.exception.InvalidRepositoryPathException;
import store.server.exception.RepositoryClosedException;
import store.server.exception.WriteException;

/**
 * A Lucene index on a repository.
 */
class Index {

    private static final String CONTENT = "content";
    private static final String LENGTH = "length";
    private static final String REVISION = "revision";
    private static final String BODY = "body";
    private static final Logger LOG = LoggerFactory.getLogger(Index.class);
    private final String name;
    private final Directory directory;
    private final Analyzer analyzer;

    private Index(String name, Path path) throws IOException {
        this.name = name;
        directory = FSDirectory.open(path.toFile(), new SingleInstanceLockFactory());
        analyzer = new StandardAnalyzer(Version.LUCENE_47);
    }

    private IndexWriter newIndexWriter() throws IOException {
        IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_47, analyzer);
        return new IndexWriter(directory, config);
    }

    public void close() {
        try {
            directory.close();

        } catch (IOException e) {
            LOG.error("[" + name + "] Failed to close index", e);
        }
    }

    /**
     * Create a new index.
     *
     * @param name index name.
     * @param path index home.
     * @return Created index.
     */
    public static Index create(String name, Path path) {
        try {
            Files.createDirectory(path);
            return new Index(name, path);

        } catch (IOException e) {
            throw new WriteException(e);
        }
    }

    /**
     * Open an existing index.
     *
     * @param name index name.
     * @param path index home.
     * @return Opened index.
     */
    public static Index open(String name, Path path) {
        if (!Files.isDirectory(path)) {
            throw new InvalidRepositoryPathException();
        }
        try {
            return new Index(name, path);

        } catch (IOException ex) {
            throw new WriteException(ex);
        }
    }

    /**
     * Index supplied content.
     *
     * @param contentInfo Info associated with content to index.
     * @param inputStream Input stream of the content to index.
     */
    public void index(ContentInfoTree contentInfoTree, InputStream inputStream) {
        LOG.info("[{}] Indexing {}, at revision {}", name, contentInfoTree.getContent(), contentInfoTree.getHead());
        Optional<IndexEntry> existing = getEntry(contentInfoTree.getContent());
        if (existing.isPresent() && existing.get().getRevisions().equals(contentInfoTree.getHead())) {
            // already indexed !
            return;
        }
        try (IndexWriter writer = newIndexWriter()) {
            // First delete any existing document.
            writer.deleteDocuments(new Term(CONTENT, contentInfoTree.getContent().asHexadecimalString()));

            // Then (re)create the document.
            Document document = new Document();
            document.add(new TextField(CONTENT, contentInfoTree.getContent().asHexadecimalString(), Store.YES));
            for (Hash rev : contentInfoTree.getHead()) {
                document.add(new TextField(REVISION, rev.asHexadecimalString(), Store.YES));
            }
            document.add(new LongField(LENGTH, contentInfoTree.getLength(), Store.NO));
            for (Entry<String, Collection<Value>> entry : headMetadata(contentInfoTree).asMap().entrySet()) {
                String key = entry.getKey();
                for (Value value : entry.getValue()) {
                    add(document, key, value);
                }
            }
            document.add(new TextField(BODY, new Tika().parse(inputStream)));
            writer.addDocument(document, analyzer);

        } catch (AlreadyClosedException e) {
            throw new RepositoryClosedException(e);

        } catch (IOException e) {
            throw new WriteException(e);
        }
    }

    private Multimap<String, Value> headMetadata(ContentInfoTree contentInfoTree) {
        Multimap<String, Value> metadata = HashMultimap.create();
        for (Hash rev : contentInfoTree.getHead()) {
            ContentInfo contentInfo = contentInfoTree.get(rev);
            for (Entry<String, Value> entry : contentInfo.getMetadata().entrySet()) {
                metadata.put(entry.getKey(), entry.getValue());
            }
        }
        return metadata;
    }

    private void add(Document document, String key, Value value) {
        switch (value.type()) {<commit_msg>Correction anomalie Sonar : complexité cyclomatique de Index.index().<commit_after>package store.server.service;

import com.google.common.base.Optional;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import static com.google.common.io.BaseEncoding.base16;
import java.io.IOException;
import java.io.InputStream;
import static java.lang.Math.min;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import static java.util.Collections.emptyList;
import java.util.HashSet;
import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.DateTools.Resolution;
import static org.apache.lucene.document.DateTools.timeToString;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.SingleInstanceLockFactory;
import org.apache.lucene.util.Version;
import org.apache.tika.Tika;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import store.common.ContentInfo;
import store.common.ContentInfoTree;
import store.common.IndexEntry;
import store.common.hash.Hash;
import store.common.value.Value;
import store.server.exception.BadRequestException;
import store.server.exception.InvalidRepositoryPathException;
import store.server.exception.RepositoryClosedException;
import store.server.exception.WriteException;

/**
 * A Lucene index on a repository.
 */
class Index {

    private static final String CONTENT = "content";
    private static final String LENGTH = "length";
    private static final String REVISION = "revision";
    private static final String BODY = "body";
    private static final Logger LOG = LoggerFactory.getLogger(Index.class);
    private final String name;
    private final Directory directory;
    private final Analyzer analyzer;

    private Index(String name, Path path) throws IOException {
        this.name = name;
        directory = FSDirectory.open(path.toFile(), new SingleInstanceLockFactory());
        analyzer = new StandardAnalyzer(Version.LUCENE_47);
    }

    private IndexWriter newIndexWriter() throws IOException {
        IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_47, analyzer);
        return new IndexWriter(directory, config);
    }

    public void close() {
        try {
            directory.close();

        } catch (IOException e) {
            LOG.error("[" + name + "] Failed to close index", e);
        }
    }

    /**
     * Create a new index.
     *
     * @param name index name.
     * @param path index home.
     * @return Created index.
     */
    public static Index create(String name, Path path) {
        try {
            Files.createDirectory(path);
            return new Index(name, path);

        } catch (IOException e) {
            throw new WriteException(e);
        }
    }

    /**
     * Open an existing index.
     *
     * @param name index name.
     * @param path index home.
     * @return Opened index.
     */
    public static Index open(String name, Path path) {
        if (!Files.isDirectory(path)) {
            throw new InvalidRepositoryPathException();
        }
        try {
            return new Index(name, path);

        } catch (IOException ex) {
            throw new WriteException(ex);
        }
    }

    /**
     * Index supplied content.
     *
     * @param contentInfo Info associated with content to index.
     * @param inputStream Input stream of the content to index.
     */
    public void index(ContentInfoTree contentInfoTree, InputStream inputStream) {
        LOG.info("[{}] Indexing {}, at revision {}", name, contentInfoTree.getContent(), contentInfoTree.getHead());
        Optional<IndexEntry> existing = getEntry(contentInfoTree.getContent());
        if (existing.isPresent() && existing.get().getRevisions().equals(contentInfoTree.getHead())) {
            // already indexed !
            return;
        }
        try (IndexWriter writer = newIndexWriter()) {
            index(writer, contentInfoTree, inputStream);

        } catch (AlreadyClosedException e) {
            throw new RepositoryClosedException(e);

        } catch (IOException e) {
            throw new WriteException(e);
        }
    }

    private void index(IndexWriter writer, ContentInfoTree contentInfoTree, InputStream inputStream) throws IOException {
        // First delete any existing document.
        writer.deleteDocuments(new Term(CONTENT, contentInfoTree.getContent().asHexadecimalString()));

        // Then (re)create the document.
        Document document = new Document();
        document.add(new TextField(CONTENT, contentInfoTree.getContent().asHexadecimalString(), Store.YES));
        for (Hash rev : contentInfoTree.getHead()) {
            document.add(new TextField(REVISION, rev.asHexadecimalString(), Store.YES));
        }
        document.add(new LongField(LENGTH, contentInfoTree.getLength(), Store.NO));
        for (Entry<String, Collection<Value>> entry : headMetadata(contentInfoTree).asMap().entrySet()) {
            String key = entry.getKey();
            for (Value value : entry.getValue()) {
                add(document, key, value);
            }
        }
        document.add(new TextField(BODY, new Tika().parse(inputStream)));
        writer.addDocument(document, analyzer);
    }

    private Multimap<String, Value> headMetadata(ContentInfoTree contentInfoTree) {
        Multimap<String, Value> metadata = HashMultimap.create();
        for (Hash rev : contentInfoTree.getHead()) {
            ContentInfo contentInfo = contentInfoTree.get(rev);
            for (Entry<String, Value> entry : contentInfo.getMetadata().entrySet()) {
                metadata.put(entry.getKey(), entry.getValue());
            }
        }
        return metadata;
    }

    private void add(Document document, String key, Value value) {
        switch (value.type()) {
{'fraction_non_alphanumeric': 0.1064052678838671, 'fraction_numerical': 0.001197246333433104, 'mean_word_length': 3.0996932515337425, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7485143', 'n_tokens_mistral': 3725, 'n_tokens_neox': 3595, 'n_words': 861}