<commit_before>package com.dubture.symfony.annotation.parser;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.CharStream;
import org.antlr.runtime.CommonTokenStream;
import org.antlr.runtime.RecognitionException;

import com.dubture.symfony.annotation.model.Annotation;
import com.dubture.symfony.annotation.parser.antlr.AnnotationLexer;
import com.dubture.symfony.annotation.parser.antlr.AnnotationParser;
import com.dubture.symfony.annotation.parser.antlr.error.IAnnotationErrorReporter;
import com.dubture.symfony.annotation.parser.tree.AnnotationCommonTree;
import com.dubture.symfony.annotation.parser.tree.AnnotationCommonTreeAdaptor;
import com.dubture.symfony.annotation.parser.tree.visitor.AnnotationNodeVisitor;

public class AnnotationCommentParser {

    static public List<Annotation> parseFromString(String comment) {
        AnnotationCommentParser parser = new AnnotationCommentParser(comment);

        return parser.parse();
    }

    static public List<Annotation> parseFromString(String comment, List<String> excludedAnnotationClassNames) {
        AnnotationCommentParser parser = new AnnotationCommentParser(comment, 0, excludedAnnotationClassNames);

        return parser.parse();
    }

    static public List<Annotation> parseFromString(String comment,
                                                   int commentCharOffset,
                                                   List<String> excludedAnnotationClassNames) {
        AnnotationCommentParser parser = new AnnotationCommentParser(comment,
                                                                     commentCharOffset,
                                                                     excludedAnnotationClassNames);

        return parser.parse();
    }

    protected StringBuffer buffer;
    protected List<String> includedClassNames;
    protected List<String> excludedClassNames;

    protected int lineOffset;
    protected int columnOffset;
    protected int currentCharOffset;
    protected int commentCharOffset;

    public AnnotationCommentParser(String comment) {
        this(comment, 0, new ArrayList<String>());
    }

    public AnnotationCommentParser(String comment, int commentOffset) {
        this(comment, commentOffset, new ArrayList<String>());
    }

    public AnnotationCommentParser(String comment, List<String> excludedClassNames) {
        this(comment, 0, excludedClassNames);
    }

    public AnnotationCommentParser(String comment, int commentCharOffset, List<String> excludedClassNames) {
        this.buffer = new StringBuffer(comment);
        this.excludedClassNames = excludedClassNames;

        this.lineOffset = 0;
        this.columnOffset = 0;
        this.currentCharOffset = 0;
        this.commentCharOffset = commentCharOffset;
    }

    public void setIncludedClassNames(String[] classNames) {
        includedClassNames = Arrays.asList(classNames);
    }

    public void setExcludedClassNames(String[] classNames) {
        excludedClassNames = Arrays.asList(classNames);
    }

    public List<Annotation> parse() {
        List<Annotation> annotations = new LinkedList<Annotation>();

        while (hasMoreChunk()) {
            String chunk = getNextChunk();
            Annotation annotation = parseChunk(chunk);

            if (annotation == null || annotation.getSourcePosition().endOffset == -1) {
                // If we couldn't get an annotation or it is not ended, increase offset an try at the next @
                currentCharOffset++;
                continue;
            }

            annotations.add(annotation);
            currentCharOffset = annotation.getSourcePosition().endOffset + 1 - commentCharOffset;
        }

        return postProcess(annotations);
    }

    /**
     * This method will post process the annotations. The post process is used mainly
     * to remove unwanted annotation like {@literal @param} annotation.
     *
     * @return A post-processed list of annotations
     */
    protected List<Annotation> postProcess(List<Annotation> annotations) {
        boolean hasIncludeRestriction = includedClassNames != null && includedClassNames.size() > 0;
        boolean hasExcludeRestriction = excludedClassNames != null && excludedClassNames.size() > 0;

        if (!hasIncludeRestriction && !hasExcludeRestriction) {
            return annotations;
        }

        Iterator<Annotation> iterator = annotations.iterator();
        while (iterator.hasNext()) {
            Annotation annotation = iterator.next();
            String fullyQualifiedName = annotation.getFullyQualifiedName();

            if (hasExcludeRestriction && excludedClassNames.contains(fullyQualifiedName)) {
                iterator.remove();
            }

            if (hasIncludeRestriction && !includedClassNames.contains(fullyQualifiedName)) {
                iterator.remove();
            }
        }

        return annotations;
    }

    protected Annotation parseChunk(String chunk) {
        CharStream content = new ANTLRStringStream(chunk);

        // FIXME: The error reporter should be passed to the parser<commit_msg>improved api of AnnotationCommentParser and fixed a bug  * When a class was removed by an exclusion, a bug could occurs if it was    also removed by not being in the inclusion list  * Added methods to add included/excluded class names<commit_after>package com.dubture.symfony.annotation.parser;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.CharStream;
import org.antlr.runtime.CommonTokenStream;
import org.antlr.runtime.RecognitionException;

import com.dubture.symfony.annotation.model.Annotation;
import com.dubture.symfony.annotation.parser.antlr.AnnotationLexer;
import com.dubture.symfony.annotation.parser.antlr.AnnotationParser;
import com.dubture.symfony.annotation.parser.antlr.error.IAnnotationErrorReporter;
import com.dubture.symfony.annotation.parser.tree.AnnotationCommonTree;
import com.dubture.symfony.annotation.parser.tree.AnnotationCommonTreeAdaptor;
import com.dubture.symfony.annotation.parser.tree.visitor.AnnotationNodeVisitor;

public class AnnotationCommentParser {

    static public List<Annotation> parseFromString(String comment) {
        AnnotationCommentParser parser = new AnnotationCommentParser(comment);

        return parser.parse();
    }

    static public List<Annotation> parseFromString(String comment, List<String> excludedAnnotationClassNames) {
        AnnotationCommentParser parser = new AnnotationCommentParser(comment, 0, excludedAnnotationClassNames);

        return parser.parse();
    }

    static public List<Annotation> parseFromString(String comment,
                                                   int commentCharOffset,
                                                   List<String> excludedAnnotationClassNames) {
        AnnotationCommentParser parser = new AnnotationCommentParser(comment,
                                                                     commentCharOffset,
                                                                     excludedAnnotationClassNames);

        return parser.parse();
    }

    protected StringBuffer buffer;
    protected List<String> includedClassNames;
    protected List<String> excludedClassNames;

    protected int lineOffset;
    protected int columnOffset;
    protected int currentCharOffset;
    protected int commentCharOffset;

    public AnnotationCommentParser(String comment) {
        this(comment, 0, new ArrayList<String>());
    }

    public AnnotationCommentParser(String comment, int commentOffset) {
        this(comment, commentOffset, new ArrayList<String>());
    }

    public AnnotationCommentParser(String comment, List<String> excludedClassNames) {
        this(comment, 0, excludedClassNames);
    }

    public AnnotationCommentParser(String comment, int commentCharOffset, List<String> excludedClassNames) {
        this.buffer = new StringBuffer(comment);
        this.excludedClassNames = excludedClassNames;
        this.includedClassNames = new LinkedList<String>();

        this.lineOffset = 0;
        this.columnOffset = 0;
        this.currentCharOffset = 0;
        this.commentCharOffset = commentCharOffset;
    }

    /**
     * Add a class name that should be included when parsing a
     * comment.
     *
     * @param className The class name to include
     */
    public void addIncludedClassName(String className) {
        includedClassNames.add(className);
    }

    /**
     * Add multiple class names that should be included when parsing a
     * comment.
     *
     * @param classNames The class names to include
     */
    public void addIncludedClassNames(String[] classNames) {
        includedClassNames.addAll(Arrays.asList(classNames));
    }

    /**
     * Add a class name that should be excluded when parsing a
     * comment. Excluded class name always has higher priority
     * than included class name.
     *
     * @param className The class name to exclude
     */
    public void addExcludedClassName(String className) {
        excludedClassNames.add(className);
    }

    /**
     * Add multiple class names that should be excluded when parsing a
     * comment. Excluded class names always have higher priority
     * than included class name.
     *
     * @param classNames The class names to exclude
     */
    public void addExcludedClassNames(String[] classNames) {
        excludedClassNames.addAll(Arrays.asList(classNames));
    }

    /**
     * Set the class names that should be included when parsing
     * a comment.
     *
     * @param classNames The new included class names
     */
    public void setIncludedClassNames(String[] classNames) {
        includedClassNames = Arrays.asList(classNames);
    }

    /**
     * Set the class names that should be excluded when parsing
     * a comment. Excluded class names always have higher priority
     * than included class names.
     *
     * @param classNames The new excluded class names
     */
    public void setExcludedClassNames(String[] classNames) {
        excludedClassNames = Arrays.asList(classNames);
    }

    public List<Annotation> parse() {
        List<Annotation> annotations = new LinkedList<Annotation>();

        while (hasMoreChunk()) {
            String chunk = getNextChunk();
            Annotation annotation = parseChunk(chunk);

            if (annotation == null || annotation.getSourcePosition().endOffset == -1) {
                // If we couldn't get an annotation or it is not ended, increase offset an try at the next @
                currentCharOffset++;
                continue;
            }

            annotations.add(annotation);
            currentCharOffset = annotation.getSourcePosition().endOffset + 1 - commentCharOffset;
        }

        return postProcess(annotations);
    }

    /**
     * This method will post process the annotations. The post process is used mainly
     * to remove unwanted annotation like {@literal @param} annotation.
     *
     * @return A post-processed list of annotations
     */
    protected List<Annotation> postProcess(List<Annotation> annotations) {
        boolean hasIncludeRestriction = includedClassNames != null && includedClassNames.size() > 0;
        boolean hasExcludeRestriction = excludedClassNames != null && excludedClassNames.size() > 0;

        if (!hasIncludeRestriction && !hasExcludeRestriction) {
            return annotations;
        }

        Iterator<Annotation> iterator = annotations.iterator();
        while (iterator.hasNext()) {
            Annotation annotation = iterator.next();
            String fullyQualifiedName = annotation.getFullyQualifiedName();
            boolean excluded = false;

            if (hasExcludeRestriction && excludedClassNames.contains(fullyQualifiedName)) {
                iterator.remove();
                excluded = true;
            }

            if (!excluded && hasIncludeRestriction && !includedClassNames.contains(fullyQualifiedName)) {
                iterator.remove();
                excluded = true;
            }
        }

        return annotations;
    }

    protected Annotation parseChunk(String chunk) {
        CharStream content = new ANTLRStringStream(chunk);

        // FIXME: The error reporter should be passed to the parser
{'fraction_non_alphanumeric': 0.0787302585055394, 'fraction_numerical': 0.001571462245619549, 'mean_word_length': 2.7479387514723204, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3321915', 'n_tokens_mistral': 3170, 'n_tokens_neox': 3005, 'n_words': 805}