<commit_before>package fr.adrienbrault.idea.symfony2plugin.stubs.indexes;

import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import com.intellij.ide.highlighter.XmlFileType;
import com.intellij.openapi.fileTypes.FileType;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.PsiFile;
import com.intellij.util.indexing.*;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.EnumeratorStringDescriptor;
import com.intellij.util.io.KeyDescriptor;
import com.jetbrains.php.lang.PhpFileType;
import fr.adrienbrault.idea.symfony2plugin.Symfony2ProjectComponent;
import fr.adrienbrault.idea.symfony2plugin.doctrine.DoctrineUtil;
import fr.adrienbrault.idea.symfony2plugin.doctrine.dict.DoctrineModel;
import fr.adrienbrault.idea.symfony2plugin.doctrine.dict.DoctrineModelInterface;
import gnu.trove.THashMap;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.yaml.YAMLFileType;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Collection;
import java.util.Map;

public class DoctrineMetadataFileStubIndex extends FileBasedIndexExtension<String, DoctrineModelInterface> {

    public static final ID<String, DoctrineModelInterface> KEY = ID.create("fr.adrienbrault.idea.symfony2plugin.doctrine_metadata_json");
    private final KeyDescriptor<String> myKeyDescriptor = new EnumeratorStringDescriptor();
    private static JsonDataExternalizer JSON_EXTERNALIZER = new JsonDataExternalizer();

    private static int MAX_FILE_BYTE_SIZE = 1048576;

    private static class MyStringStringFileContentDataIndexer implements DataIndexer<String, DoctrineModelInterface, FileContent> {
        @NotNull
        @Override
        public Map<String, DoctrineModelInterface> map(@NotNull FileContent fileContent) {

            Map<String, DoctrineModelInterface> map = new THashMap<String, DoctrineModelInterface>();

            PsiFile psiFile = fileContent.getPsiFile();
            if(!Symfony2ProjectComponent.isEnabledForIndex(psiFile.getProject()) || !isValidForIndex(fileContent, psiFile)) {
                return map;
            }

            Collection<Pair<String, String>> classRepositoryPair = DoctrineUtil.getClassRepositoryPair(psiFile);
            if(classRepositoryPair == null || classRepositoryPair.size() == 0) {
                return map;
            }

            for (Pair<String, String> pair : classRepositoryPair) {
                String first = pair.getFirst();
                if(first == null || first.length() == 0) {
                    continue;
                }
                map.put(first, new DoctrineModel(first).setRepositoryClass(pair.getSecond()));
            }

            return map;
        }
    }

    @NotNull
    @Override
    public ID<String, DoctrineModelInterface> getName() {
        return KEY;
    }

    @NotNull
    @Override
    public DataIndexer<String, DoctrineModelInterface, FileContent> getIndexer() {
        return new MyStringStringFileContentDataIndexer();
    }

    @NotNull
    @Override
    public KeyDescriptor<String> getKeyDescriptor() {
        return this.myKeyDescriptor;
    }

    @NotNull
    @Override
    public DataExternalizer<DoctrineModelInterface> getValueExternalizer() {
        return JSON_EXTERNALIZER;
    }

    @NotNull
    @Override
    public FileBasedIndex.InputFilter getInputFilter() {
        return new FileBasedIndex.InputFilter() {
            @Override
            public boolean acceptInput(@NotNull VirtualFile file) {
                FileType fileType = file.getFileType();
                return fileType == XmlFileType.INSTANCE || fileType == YAMLFileType.YML || fileType == PhpFileType.INSTANCE;
            }
        };
    }

    @Override
    public boolean dependsOnFileContent() {
        return true;
    }

    @Override<commit_msg>try DefaultFileTypeSpecificInputFilter for file type filtering on index<commit_after>package fr.adrienbrault.idea.symfony2plugin.stubs.indexes;

import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import com.intellij.ide.highlighter.XmlFileType;
import com.intellij.openapi.fileTypes.FileType;
import com.intellij.openapi.util.Pair;
import com.intellij.openapi.vfs.VirtualFile;
import com.intellij.psi.PsiFile;
import com.intellij.util.indexing.*;
import com.intellij.util.io.DataExternalizer;
import com.intellij.util.io.EnumeratorStringDescriptor;
import com.intellij.util.io.KeyDescriptor;
import com.jetbrains.php.lang.PhpFileType;
import fr.adrienbrault.idea.symfony2plugin.Symfony2ProjectComponent;
import fr.adrienbrault.idea.symfony2plugin.doctrine.DoctrineUtil;
import fr.adrienbrault.idea.symfony2plugin.doctrine.dict.DoctrineModel;
import fr.adrienbrault.idea.symfony2plugin.doctrine.dict.DoctrineModelInterface;
import gnu.trove.THashMap;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.yaml.YAMLFileType;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Collection;
import java.util.Map;

public class DoctrineMetadataFileStubIndex extends FileBasedIndexExtension<String, DoctrineModelInterface> {

    public static final ID<String, DoctrineModelInterface> KEY = ID.create("fr.adrienbrault.idea.symfony2plugin.doctrine_metadata_json");
    private final KeyDescriptor<String> myKeyDescriptor = new EnumeratorStringDescriptor();
    private static JsonDataExternalizer JSON_EXTERNALIZER = new JsonDataExternalizer();

    private static int MAX_FILE_BYTE_SIZE = 1048576;

    private static class MyStringStringFileContentDataIndexer implements DataIndexer<String, DoctrineModelInterface, FileContent> {
        @NotNull
        @Override
        public Map<String, DoctrineModelInterface> map(@NotNull FileContent fileContent) {

            Map<String, DoctrineModelInterface> map = new THashMap<String, DoctrineModelInterface>();

            PsiFile psiFile = fileContent.getPsiFile();
            if(!Symfony2ProjectComponent.isEnabledForIndex(psiFile.getProject()) || !isValidForIndex(fileContent, psiFile)) {
                return map;
            }

            Collection<Pair<String, String>> classRepositoryPair = DoctrineUtil.getClassRepositoryPair(psiFile);
            if(classRepositoryPair == null || classRepositoryPair.size() == 0) {
                return map;
            }

            for (Pair<String, String> pair : classRepositoryPair) {
                String first = pair.getFirst();
                if(first == null || first.length() == 0) {
                    continue;
                }
                map.put(first, new DoctrineModel(first).setRepositoryClass(pair.getSecond()));
            }

            return map;
        }
    }

    @NotNull
    @Override
    public ID<String, DoctrineModelInterface> getName() {
        return KEY;
    }

    @NotNull
    @Override
    public DataIndexer<String, DoctrineModelInterface, FileContent> getIndexer() {
        return new MyStringStringFileContentDataIndexer();
    }

    @NotNull
    @Override
    public KeyDescriptor<String> getKeyDescriptor() {
        return this.myKeyDescriptor;
    }

    @NotNull
    @Override
    public DataExternalizer<DoctrineModelInterface> getValueExternalizer() {
        return JSON_EXTERNALIZER;
    }

    @NotNull
    @Override
    public FileBasedIndex.InputFilter getInputFilter() {
        return new DefaultFileTypeSpecificInputFilter(XmlFileType.INSTANCE, PhpFileType.INSTANCE, YAMLFileType.YML);
    }

    @Override
    public boolean dependsOnFileContent() {
        return true;
    }

    @Override
{'fraction_non_alphanumeric': 0.08836235109307501, 'fraction_numerical': 0.0044508443513548894, 'mean_word_length': 3.5207100591715976, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2409452', 'n_tokens_mistral': 2192, 'n_tokens_neox': 2115, 'n_words': 394}