<commit_before>/*
 * Copyright (C) 2009-2011 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.parboiled;

import com.google.common.base.Joiner;
import org.parboiled.matchers.ActionMatcher;
import org.parboiled.matchers.AnyMatcher;
import org.parboiled.matchers.AnyOfMatcher;
import org.parboiled.matchers.CharIgnoreCaseMatcher;
import org.parboiled.matchers.CharMatcher;
import org.parboiled.matchers.CharRangeMatcher;
import org.parboiled.matchers.CustomMatcher;
import org.parboiled.matchers.EmptyMatcher;
import org.parboiled.matchers.FirstOfMatcher;
import org.parboiled.matchers.FirstOfStringsMatcher;
import org.parboiled.matchers.Matcher;
import org.parboiled.matchers.MemoMismatchesMatcher;
import org.parboiled.matchers.NothingMatcher;
import org.parboiled.matchers.OneOrMoreMatcher;
import org.parboiled.matchers.OptionalMatcher;
import org.parboiled.matchers.ProxyMatcher;
import org.parboiled.matchers.SequenceMatcher;
import org.parboiled.matchers.StringMatcher;
import org.parboiled.matchers.TestMatcher;
import org.parboiled.matchers.TestNotMatcher;
import org.parboiled.matchers.VarFramingMatcher;
import org.parboiled.matchers.ZeroOrMoreMatcher;
import org.parboiled.matchers.unicode.UnicodeCharMatcher;
import org.parboiled.matchers.unicode.UnicodeRangeMatcher;
import org.parboiled.matchervisitors.MatcherVisitor;

import java.nio.CharBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import static org.parboiled.common.Preconditions.checkArgNotNull;

public class ParserStatistics
    implements MatcherVisitor<ParserStatistics>
{
    private static final Joiner NEWLINE = Joiner.on('\n');
    private static final Joiner COMMA = Joiner.on(", ");
    /*
     * Modify these maps if you add matcher classes!
     */
    private static final Set<Class<? extends Matcher>> REGULAR_MATCHER_CLASSES;
    private static final Set<Class<? extends Matcher>> SPECIAL_MATCHER_CLASSES;

    static {
        REGULAR_MATCHER_CLASSES = new LinkedHashSet<Class<? extends Matcher>>();

        REGULAR_MATCHER_CLASSES.add(AnyMatcher.class);
        REGULAR_MATCHER_CLASSES.add(CharIgnoreCaseMatcher.class);
        REGULAR_MATCHER_CLASSES.add(CharMatcher.class);
        REGULAR_MATCHER_CLASSES.add(UnicodeCharMatcher.class);
        REGULAR_MATCHER_CLASSES.add(CustomMatcher.class);
        REGULAR_MATCHER_CLASSES.add(CharRangeMatcher.class);
        REGULAR_MATCHER_CLASSES.add(UnicodeRangeMatcher.class);
        REGULAR_MATCHER_CLASSES.add(AnyOfMatcher.class);
        REGULAR_MATCHER_CLASSES.add(EmptyMatcher.class);
        REGULAR_MATCHER_CLASSES.add(FirstOfMatcher.class);
        REGULAR_MATCHER_CLASSES.add(FirstOfStringsMatcher.class);
        REGULAR_MATCHER_CLASSES.add(NothingMatcher.class);
        REGULAR_MATCHER_CLASSES.add(OneOrMoreMatcher.class);
        REGULAR_MATCHER_CLASSES.add(OptionalMatcher.class);
        REGULAR_MATCHER_CLASSES.add(SequenceMatcher.class);
        REGULAR_MATCHER_CLASSES.add(StringMatcher.class);
        REGULAR_MATCHER_CLASSES.add(TestMatcher.class);
        REGULAR_MATCHER_CLASSES.add(TestNotMatcher.class);
        REGULAR_MATCHER_CLASSES.add(ZeroOrMoreMatcher.class);

        SPECIAL_MATCHER_CLASSES = new LinkedHashSet<Class<? extends Matcher>>();

        SPECIAL_MATCHER_CLASSES.add(ProxyMatcher.class);
        SPECIAL_MATCHER_CLASSES.add(VarFramingMatcher.class);
        SPECIAL_MATCHER_CLASSES.add(MemoMismatchesMatcher.class);
    }

    private final Matcher root;
    private int totalRules;

    private final Map<Class<?>, MatcherStats<?>> regularMatcherStats
        = new LinkedHashMap<Class<?>, MatcherStats<?>>();

    private final Map<Class<?>, MatcherStats<?>> specialMatcherStats
        = new LinkedHashMap<Class<?>, MatcherStats<?>>();

    private final Set<Action<?>> actions = new HashSet<Action<?>>();
    private final Set<Class<?>> actionClasses = new HashSet<Class<?>>();

    public static ParserStatistics generateFor(final Rule rule)
    {
        checkArgNotNull(rule, "rule");
        final Matcher matcher = (Matcher) rule;
        return matcher.accept(new ParserStatistics(matcher));
    }

    private ParserStatistics(final Matcher root)
    {
        this.root = root;
        for (final Class<? extends Matcher> c: REGULAR_MATCHER_CLASSES)
            regularMatcherStats.put(c, MatcherStats.forClass(c, false));
        for (final Class<? extends Matcher> c: SPECIAL_MATCHER_CLASSES)
            specialMatcherStats.put(c, MatcherStats.forClass(c, true));
        countSpecials(root);
    }

    public int getTotalRules()
    {
        return totalRules;
    }

    public Map<Class<?>, MatcherStats<?>> getRegularMatcherStats()
    {
        return Collections.unmodifiableMap(regularMatcherStats);
    }

    public Map<Class<?>, MatcherStats<?>> getSpecialMatcherStats()
    {
        return Collections.unmodifiableMap(specialMatcherStats);
    }

    public Set<Action<?>> getActions()
    {
        return Collections.unmodifiableSet(actions);
    }

    public Set<Class<?>> getActionClasses()
    {
        return Collections.unmodifiableSet(actionClasses);
    }

    // MatcherVisitor interface

    @Override
    public ParserStatistics visit(final ActionMatcher matcher)
    {
        if (actions.add(matcher.action)) {
            totalRules++;
            actionClasses.add(matcher.action.getClass());
        }
        return this;
    }

    @Override
    public ParserStatistics visit(final AnyMatcher matcher)
    {
        return doVisit(matcher);
    }

    @Override
    public ParserStatistics visit(final CharIgnoreCaseMatcher matcher)
    {
        return doVisit(matcher);
    }

    @Override
    public ParserStatistics visit(final CharMatcher matcher)
    {
        return doVisit(matcher);
    }

    @Override
    public ParserStatistics visit(final UnicodeCharMatcher matcher)
    {
        return doVisit(matcher, UnicodeCharMatcher.class);
    }

    @Override
    public ParserStatistics visit(final CustomMatcher matcher)
    {
        return doVisit(matcher);
    }

    @Override
    public ParserStatistics visit(final CharRangeMatcher matcher)
    {
        return doVisit(matcher);
    }

    @Override
    public ParserStatistics visit(final UnicodeRangeMatcher matcher)
    {
        return doVisit(matcher, UnicodeRangeMatcher.class);
    }

    @Override
    public ParserStatistics visit(final AnyOfMatcher matcher)
    {
        return doVisit(matcher);
    }

    @Override
    public ParserStatistics visit(final EmptyMatcher matcher)
    {
        return doVisit(matcher);
    }

    @Override
    public ParserStatistics visit(final FirstOfMatcher matcher)
    {
        return doVisit(matcher);
    }

    @Override
    public ParserStatistics visit(final NothingMatcher matcher)
    {
        return doVisit(matcher);
    }

    @Override
    public ParserStatistics visit(final OneOrMoreMatcher matcher)
    {
        return doVisit(matcher);
    }

    @Override
    public ParserStatistics visit(final OptionalMatcher matcher)
    {
        return doVisit(matcher);
    }

    @Override
    public ParserStatistics visit(final SequenceMatcher matcher)
    {
        return doVisit(matcher);
    }

    @Override
    public ParserStatistics visit(final TestMatcher matcher)
    {
        return doVisit(matcher);
    }

    @Override
    public ParserStatistics visit(final TestNotMatcher matcher)
    {
        return doVisit(matcher);
    }

    @Override
    public ParserStatistics visit(final ZeroOrMoreMatcher matcher)
    {
        return doVisit(matcher);
    }

    private <M extends Matcher> ParserStatistics doVisit(final M matcher)
    {
        return doVisit(matcher, matcher.getClass());
    }

    private <M extends Matcher> ParserStatistics doVisit(final M matcher,
        final Class<? extends M> c)
    {
        final MatcherStats<?> stats = REGULAR_MATCHER_CLASSES.contains(c)
            ? regularMatcherStats.get(c)
            : specialMatcherStats.get(c);
        checkArgNotNull(stats, c.getCanonicalName() + " not recorded??");
        if (stats.recordInstance(matcher)) {
            totalRules++;
            for (final Matcher child: matcher.getChildren()) {
                countSpecials(child);<commit_msg>ParserStatistics: always use the classname...<commit_after>/*
 * Copyright (C) 2009-2011 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.parboiled;

import com.google.common.base.Joiner;
import org.parboiled.matchers.ActionMatcher;
import org.parboiled.matchers.AnyMatcher;
import org.parboiled.matchers.AnyOfMatcher;
import org.parboiled.matchers.CharIgnoreCaseMatcher;
import org.parboiled.matchers.CharMatcher;
import org.parboiled.matchers.CharRangeMatcher;
import org.parboiled.matchers.CustomMatcher;
import org.parboiled.matchers.EmptyMatcher;
import org.parboiled.matchers.FirstOfMatcher;
import org.parboiled.matchers.FirstOfStringsMatcher;
import org.parboiled.matchers.Matcher;
import org.parboiled.matchers.MemoMismatchesMatcher;
import org.parboiled.matchers.NothingMatcher;
import org.parboiled.matchers.OneOrMoreMatcher;
import org.parboiled.matchers.OptionalMatcher;
import org.parboiled.matchers.ProxyMatcher;
import org.parboiled.matchers.SequenceMatcher;
import org.parboiled.matchers.StringMatcher;
import org.parboiled.matchers.TestMatcher;
import org.parboiled.matchers.TestNotMatcher;
import org.parboiled.matchers.VarFramingMatcher;
import org.parboiled.matchers.ZeroOrMoreMatcher;
import org.parboiled.matchers.unicode.UnicodeCharMatcher;
import org.parboiled.matchers.unicode.UnicodeRangeMatcher;
import org.parboiled.matchervisitors.MatcherVisitor;

import java.nio.CharBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import static org.parboiled.common.Preconditions.checkArgNotNull;

public class ParserStatistics
    implements MatcherVisitor<ParserStatistics>
{
    private static final Joiner NEWLINE = Joiner.on('\n');
    private static final Joiner COMMA = Joiner.on(", ");
    /*
     * Modify these maps if you add matcher classes!
     */
    private static final Set<Class<? extends Matcher>> REGULAR_MATCHER_CLASSES;
    private static final Set<Class<? extends Matcher>> SPECIAL_MATCHER_CLASSES;

    static {
        REGULAR_MATCHER_CLASSES = new LinkedHashSet<Class<? extends Matcher>>();

        REGULAR_MATCHER_CLASSES.add(AnyMatcher.class);
        REGULAR_MATCHER_CLASSES.add(CharIgnoreCaseMatcher.class);
        REGULAR_MATCHER_CLASSES.add(CharMatcher.class);
        REGULAR_MATCHER_CLASSES.add(UnicodeCharMatcher.class);
        REGULAR_MATCHER_CLASSES.add(CustomMatcher.class);
        REGULAR_MATCHER_CLASSES.add(CharRangeMatcher.class);
        REGULAR_MATCHER_CLASSES.add(UnicodeRangeMatcher.class);
        REGULAR_MATCHER_CLASSES.add(AnyOfMatcher.class);
        REGULAR_MATCHER_CLASSES.add(EmptyMatcher.class);
        REGULAR_MATCHER_CLASSES.add(FirstOfMatcher.class);
        REGULAR_MATCHER_CLASSES.add(FirstOfStringsMatcher.class);
        REGULAR_MATCHER_CLASSES.add(NothingMatcher.class);
        REGULAR_MATCHER_CLASSES.add(OneOrMoreMatcher.class);
        REGULAR_MATCHER_CLASSES.add(OptionalMatcher.class);
        REGULAR_MATCHER_CLASSES.add(SequenceMatcher.class);
        REGULAR_MATCHER_CLASSES.add(StringMatcher.class);
        REGULAR_MATCHER_CLASSES.add(TestMatcher.class);
        REGULAR_MATCHER_CLASSES.add(TestNotMatcher.class);
        REGULAR_MATCHER_CLASSES.add(ZeroOrMoreMatcher.class);

        SPECIAL_MATCHER_CLASSES = new LinkedHashSet<Class<? extends Matcher>>();

        SPECIAL_MATCHER_CLASSES.add(ProxyMatcher.class);
        SPECIAL_MATCHER_CLASSES.add(VarFramingMatcher.class);
        SPECIAL_MATCHER_CLASSES.add(MemoMismatchesMatcher.class);
    }

    private final Matcher root;
    private int totalRules;

    private final Map<Class<?>, MatcherStats<?>> regularMatcherStats
        = new LinkedHashMap<Class<?>, MatcherStats<?>>();

    private final Map<Class<?>, MatcherStats<?>> specialMatcherStats
        = new LinkedHashMap<Class<?>, MatcherStats<?>>();

    private final Set<Action<?>> actions = new HashSet<Action<?>>();
    private final Set<Class<?>> actionClasses = new HashSet<Class<?>>();

    public static ParserStatistics generateFor(final Rule rule)
    {
        checkArgNotNull(rule, "rule");
        final Matcher matcher = (Matcher) rule;
        return matcher.accept(new ParserStatistics(matcher));
    }

    private ParserStatistics(final Matcher root)
    {
        this.root = root;
        for (final Class<? extends Matcher> c: REGULAR_MATCHER_CLASSES)
            regularMatcherStats.put(c, MatcherStats.forClass(c, false));
        for (final Class<? extends Matcher> c: SPECIAL_MATCHER_CLASSES)
            specialMatcherStats.put(c, MatcherStats.forClass(c, true));
        countSpecials(root);
    }

    public int getTotalRules()
    {
        return totalRules;
    }

    public Map<Class<?>, MatcherStats<?>> getRegularMatcherStats()
    {
        return Collections.unmodifiableMap(regularMatcherStats);
    }

    public Map<Class<?>, MatcherStats<?>> getSpecialMatcherStats()
    {
        return Collections.unmodifiableMap(specialMatcherStats);
    }

    public Set<Action<?>> getActions()
    {
        return Collections.unmodifiableSet(actions);
    }

    public Set<Class<?>> getActionClasses()
    {
        return Collections.unmodifiableSet(actionClasses);
    }

    // MatcherVisitor interface

    @Override
    public ParserStatistics visit(final ActionMatcher matcher)
    {
        if (actions.add(matcher.action)) {
            totalRules++;
            actionClasses.add(matcher.action.getClass());
        }
        return this;
    }

    @Override
    public ParserStatistics visit(final AnyMatcher matcher)
    {
        return doVisit(matcher, AnyMatcher.class);
    }

    @Override
    public ParserStatistics visit(final CharIgnoreCaseMatcher matcher)
    {
        return doVisit(matcher, CharIgnoreCaseMatcher.class);
    }

    @Override
    public ParserStatistics visit(final CharMatcher matcher)
    {
        return doVisit(matcher, CharMatcher.class);
    }

    @Override
    public ParserStatistics visit(final UnicodeCharMatcher matcher)
    {
        return doVisit(matcher, UnicodeCharMatcher.class);
    }

    @Override
    public ParserStatistics visit(final CustomMatcher matcher)
    {
        return doVisit(matcher, CustomMatcher.class);
    }

    @Override
    public ParserStatistics visit(final CharRangeMatcher matcher)
    {
        return doVisit(matcher, CharRangeMatcher.class);
    }

    @Override
    public ParserStatistics visit(final UnicodeRangeMatcher matcher)
    {
        return doVisit(matcher, UnicodeRangeMatcher.class);
    }

    @Override
    public ParserStatistics visit(final AnyOfMatcher matcher)
    {
        return doVisit(matcher, AnyOfMatcher.class);
    }

    @Override
    public ParserStatistics visit(final EmptyMatcher matcher)
    {
        return doVisit(matcher, EmptyMatcher.class);
    }

    @Override
    public ParserStatistics visit(final FirstOfMatcher matcher)
    {
        return doVisit(matcher, FirstOfMatcher.class);
    }

    @Override
    public ParserStatistics visit(final NothingMatcher matcher)
    {
        return doVisit(matcher, NothingMatcher.class);
    }

    @Override
    public ParserStatistics visit(final OneOrMoreMatcher matcher)
    {
        return doVisit(matcher, OneOrMoreMatcher.class);
    }

    @Override
    public ParserStatistics visit(final OptionalMatcher matcher)
    {
        return doVisit(matcher, OptionalMatcher.class);
    }

    @Override
    public ParserStatistics visit(final SequenceMatcher matcher)
    {
        return doVisit(matcher, SequenceMatcher.class);
    }

    @Override
    public ParserStatistics visit(final TestMatcher matcher)
    {
        return doVisit(matcher, TestMatcher.class);
    }

    @Override
    public ParserStatistics visit(final TestNotMatcher matcher)
    {
        return doVisit(matcher, TestNotMatcher.class);
    }

    @Override
    public ParserStatistics visit(final ZeroOrMoreMatcher matcher)
    {
        return doVisit(matcher, ZeroOrMoreMatcher.class);
    }

    private <M extends Matcher> ParserStatistics doVisit(final M matcher)
    {
        return doVisit(matcher, matcher.getClass());
    }

    private <M extends Matcher> ParserStatistics doVisit(final M matcher,
        final Class<? extends M> c)
    {
        final MatcherStats<?> stats = REGULAR_MATCHER_CLASSES.contains(c)
            ? regularMatcherStats.get(c)
            : specialMatcherStats.get(c);
        checkArgNotNull(stats, c.getCanonicalName() + " not recorded??");
        if (stats.recordInstance(matcher)) {
            totalRules++;
            for (final Matcher child: matcher.getChildren()) {
                countSpecials(child);
{'fraction_non_alphanumeric': 0.08914749958829665, 'fraction_numerical': 0.0013174507328319702, 'mean_word_length': 3.6168271667511402, 'pattern_counts': {'":': 0, '<': 95, '<?xml version=': 0, '>': 95, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6339146', 'n_tokens_mistral': 5490, 'n_tokens_neox': 5191, 'n_words': 1082}