<commit_before>#!/usr/bin/env perl
#
# Cheat sheet tester created by <NAME> (https://github.com/dnmfarrell)
# from http://perltricks.com/article/190/2015/8/28/Check-your-DuckDuckGo-cheatsheets-with-Perl
#
use strict;
use warnings;
use open ':std', ':encoding(utf8)';
use Test::More;
use Term::ANSIColor;
use JSON;
use IO::All;
use List::Util qw(first none);
use YAML::XS qw(LoadFile);

my $json_dir = "share/goodie/cheat_sheets/json";
my $json;

my $triggers_yaml = LoadFile('share/goodie/cheat_sheets/triggers.yaml');

my %triggers;

my $template_map = $triggers_yaml->{template_map};

sub flat_triggers {
    my $data = shift;
    if (my $triggers = $data->{triggers}) {
        return map { @$_ } values $triggers;
    }
    return ();
}

sub id_to_file_name {
    my $id = shift;
    return unless $id =~ s/_cheat_sheet//;
    $id =~ s/_/-/g;
    return $id . '.json';
}

sub check_aliases_for_triggers {
    my ($aliases, $trigger_types) = @_;
    my @aliases = @$aliases;
    while (my ($trigger_type, $triggers) = each %{$trigger_types}) {
        my @triggers = @$triggers;
        foreach my $alias (@aliases) {
            my $trigger;
            if (
                ($trigger_type =~ /^start/
                    && ($trigger = first { $alias =~ /^$_/ } @triggers))
            ||  ($trigger_type =~ /end$/
                    && ($trigger = first { $alias =~ /$_$/ } @triggers))
            ) {
                return ($alias, $trigger);
            }
        }
    }
    return;
}

# Iterate over all Cheat Sheet JSON files...
foreach my $path (glob("$json_dir/*.json")){
    next if $ARGV[0] && $path ne  "$json_dir/$ARGV[0].json";

    my ($file_name) = $path =~ /$json_dir\/(.+)/;
    my ($name) = $path =~ /.+\/(.+)\.json$/;
    my $defaultName = $name =~ s/-/ /gr;
    my @tests;

    ### File tests ###
    my $temp_pass = (my $content < io($path))? 1 : 0;
    push(@tests, {msg => 'file content can be read', critical => 1, pass => $temp_pass});

    $temp_pass = ($json = decode_json($content))? 1 : 0;
    push(@tests, {msg => 'content is valid JSON', critical => 1, pass => $temp_pass});

    ### Headers tests ###
    $temp_pass = (exists $json->{id} && $json->{id})? 1 : 0;
    push(@tests, {msg => 'has id', critical => 1, pass => $temp_pass});;

    $temp_pass = (exists $json->{name} && $json->{name})? 1 : 0;
    push(@tests, {msg => 'has name', critical => 1, pass => $temp_pass});

    $temp_pass = (!exists $json->{description} && !$json->{description})? 0 : 1;
    push(@tests, {msg => "has description (optional but suggested)", critical => 0, pass => $temp_pass});

    ### Template Type tests ###
    $temp_pass = (exists $json->{template_type} && $json->{template_type})? 1 : 0;
    push(@tests, {msg => 'must specify a template type', critical => 1, pass => $temp_pass});
    my $template_type = $json->{template_type};
    $temp_pass = (exists $template_map->{$template_type});
    push(@tests, {msg => "Invalid template_type '$template_type'", critical => 1, pass => $temp_pass});

    my %categories;

    if (my $cheat_id = $json->{id}) {
        ### ID tests ###
        $temp_pass = id_to_file_name($cheat_id) eq $file_name;
        push(@tests, {msg => "Invalid file name ($file_name) for ID ($cheat_id)", critical => 1, pass => $temp_pass});

        ### Trigger tests ###
        if (my $custom = $triggers_yaml->{custom_triggers}->{$cheat_id}) {
            %categories = map { $_ => 1 } @{$custom->{additional_categories}};
            # Duplicate triggers
            foreach my $trigger (flat_triggers($custom)) {
                $temp_pass = $triggers{$trigger}++ ? 0 : 1;
                push(@tests, {msg => "trigger '$trigger' already in use", critical => 1, pass => $temp_pass});
            }
            # Re-adding category
            foreach my $category (keys %categories) {
                $temp_pass = none { $_ eq $category } @{$template_map->{$template_type}};
                push(@tests, {msg => "Category '$category' already assigned", critical => 1, pass => $temp_pass});
            }
        }

        ### Alias tests ###
        if (my $aliases = $json->{aliases}) {
            my @aliases = @{$aliases};
            if ("@aliases" =~ /[[:upper:]]/) {
                push(@tests, {msg => "uppercase detected in alias - aliases should be lowercase", critical => 1});
            }
            if (first { lc $_ eq $defaultName } @aliases) {
                push(@tests, {msg => "aliases should not contain the cheat sheet name ($defaultName)", critical => 1});
            }
            # Make sure aliases don't contain any category triggers.
            while (my ($category, $trigger_types) = each %{$triggers_yaml->{categories}}) {
                my $critical = $categories{$category};
                if (my ($alias, $trigger) = check_aliases_for_triggers(\@aliases, $trigger_types)) {
                    push(@tests, {msg => "alias ($alias) contains a trigger ($trigger) defined in the '$category' category", critical => $critical});
                }
            }
            # Make sure aliases don't contain any custom triggers for the cheat sheet.
            if (my $custom = $triggers_yaml->{custom_triggers}{$cheat_id}) {
                if (my ($alias, $trigger) = check_aliases_for_triggers(\@aliases, $custom)) {
                    push(@tests, {msg => "alias ($alias) contains a custom trigger ($trigger)", critical => 1});
                }
            }
        }
    }

    %categories = (%categories, map { $_ => 1 } @{$template_map->{$template_type}});

    ### Metadata tests ###
    my $has_meta = exists $json->{metadata};

    $temp_pass = $has_meta? 1 : 0;
    push(@tests, {msg => 'has metadata (optional but suggested)', critical => 0, pass => $temp_pass, skip => 1});

    $temp_pass = exists $json->{metadata}{sourceName}? 1 : 0;
    push(@tests, {msg => "has metadata sourceName $name", critical => 1, pass => $temp_pass, skip => 1});

    $temp_pass = exists $json->{metadata}{sourceUrl}? 1 : 0;
    push(@tests, {msg => "has metadata sourceUrl $name", critical => 0, pass => $temp_pass, skip => 1});

    $temp_pass = (my $url = $json->{metadata}{sourceUrl})? 1 : 0;
    push(@tests, {msg => "sourceUrl is not undef $name", critical => 1, pass => $temp_pass, skip => 1});;

    ### Sections tests ###
    $temp_pass = (my $order = $json->{section_order})? 1 : 0;
    push(@tests, {msg => 'has section_order', critical => 1, pass => $temp_pass});

    $temp_pass = (ref $order eq 'ARRAY')? 1 : 0;
    push(@tests, {msg => 'section_order is an array of section names', critical => 1, pass => $temp_pass});

    # we're going to handle section case mismatches on frontend
    $_ = lc for @$order;

    $temp_pass = (my $sections = $json->{sections})? 1 : 0;
    push(@tests, {msg => 'has sections', critical => 1, pass => $temp_pass});

    $temp_pass = (ref $sections eq 'HASH')? 1 : 0;
    push(@tests, {msg => 'sections is a hash of section key/pairs', critical => 1, pass => $temp_pass});

    my %sections = map { lc $_ => $sections->{$_} } (keys %{$sections});

    for my $section_name (@$order) {
        push(@tests, {msg => "Expected '$section_name' but not found", critical => 1, pass => 0}) unless defined $sections{$section_name};
    }

    for my $section_name (keys %sections) {
        push(@tests, {msg => "Section '$section_name' defined, but not listed in section_order", critical => 1, pass => 0}) unless grep(/\Q$section_name\E/, @$order);

        $temp_pass = (ref $sections{$section_name} eq 'ARRAY')? 1 : 0;
        push(@tests, {msg => "'$section_name' is an array from $name",  critical => 1, pass => $temp_pass});

        my $entry_count = 0;
        for my $entry (@{$sections{$section_name}}) {
            # Only show it when it fails, otherwise it clutters the output
            push(@tests, {msg => "'$section_name' entry: $entry_count has a key from $name", critical => 1, pass => 0}) unless exists $entry->{key};

            #push(@tests, {msg => "'$section_name' entry: $entry_count has a val from $name", critical => 1, pass => 0}) unless exists $entry->{val};
            $entry_count++;
        }
    }


    %sections = %{$json->{sections}};

    for my $section_name (keys %sections) {
        for my $entry (@{$sections{$section_name}}){
            # spacing in keys ([a]/[b])'
            if ($entry->{val}) {
                if (($entry->{val} =~ /\(\[.*\]\/\[.+\]\)/g)) {
                    push(@tests, {msg => "keys ([a]/[b]) should have white spaces: $entry->{val} from  $name", critical => 0, pass => 0});
                }<commit_msg>Make section tests case-sensitive<commit_after>#!/usr/bin/env perl
#
# Cheat sheet tester created by <NAME> (https://github.com/dnmfarrell)
# from http://perltricks.com/article/190/2015/8/28/Check-your-DuckDuckGo-cheatsheets-with-Perl
#
use strict;
use warnings;
use open ':std', ':encoding(utf8)';
use Test::More;
use Term::ANSIColor;
use JSON;
use IO::All;
use List::Util qw(first none);
use YAML::XS qw(LoadFile);

my $json_dir = "share/goodie/cheat_sheets/json";
my $json;

my $triggers_yaml = LoadFile('share/goodie/cheat_sheets/triggers.yaml');

my %triggers;

my $template_map = $triggers_yaml->{template_map};

sub flat_triggers {
    my $data = shift;
    if (my $triggers = $data->{triggers}) {
        return map { @$_ } values $triggers;
    }
    return ();
}

sub id_to_file_name {
    my $id = shift;
    return unless $id =~ s/_cheat_sheet//;
    $id =~ s/_/-/g;
    return $id . '.json';
}

sub check_aliases_for_triggers {
    my ($aliases, $trigger_types) = @_;
    my @aliases = @$aliases;
    while (my ($trigger_type, $triggers) = each %{$trigger_types}) {
        my @triggers = @$triggers;
        foreach my $alias (@aliases) {
            my $trigger;
            if (
                ($trigger_type =~ /^start/
                    && ($trigger = first { $alias =~ /^$_/ } @triggers))
            ||  ($trigger_type =~ /end$/
                    && ($trigger = first { $alias =~ /$_$/ } @triggers))
            ) {
                return ($alias, $trigger);
            }
        }
    }
    return;
}

# Iterate over all Cheat Sheet JSON files...
foreach my $path (glob("$json_dir/*.json")){
    next if $ARGV[0] && $path ne  "$json_dir/$ARGV[0].json";

    my ($file_name) = $path =~ /$json_dir\/(.+)/;
    my ($name) = $path =~ /.+\/(.+)\.json$/;
    my $defaultName = $name =~ s/-/ /gr;
    my @tests;

    ### File tests ###
    my $temp_pass = (my $content < io($path))? 1 : 0;
    push(@tests, {msg => 'file content can be read', critical => 1, pass => $temp_pass});

    $temp_pass = ($json = decode_json($content))? 1 : 0;
    push(@tests, {msg => 'content is valid JSON', critical => 1, pass => $temp_pass});

    ### Headers tests ###
    $temp_pass = (exists $json->{id} && $json->{id})? 1 : 0;
    push(@tests, {msg => 'has id', critical => 1, pass => $temp_pass});;

    $temp_pass = (exists $json->{name} && $json->{name})? 1 : 0;
    push(@tests, {msg => 'has name', critical => 1, pass => $temp_pass});

    $temp_pass = (!exists $json->{description} && !$json->{description})? 0 : 1;
    push(@tests, {msg => "has description (optional but suggested)", critical => 0, pass => $temp_pass});

    ### Template Type tests ###
    $temp_pass = (exists $json->{template_type} && $json->{template_type})? 1 : 0;
    push(@tests, {msg => 'must specify a template type', critical => 1, pass => $temp_pass});
    my $template_type = $json->{template_type};
    $temp_pass = (exists $template_map->{$template_type});
    push(@tests, {msg => "Invalid template_type '$template_type'", critical => 1, pass => $temp_pass});

    my %categories;

    if (my $cheat_id = $json->{id}) {
        ### ID tests ###
        $temp_pass = id_to_file_name($cheat_id) eq $file_name;
        push(@tests, {msg => "Invalid file name ($file_name) for ID ($cheat_id)", critical => 1, pass => $temp_pass});

        ### Trigger tests ###
        if (my $custom = $triggers_yaml->{custom_triggers}->{$cheat_id}) {
            %categories = map { $_ => 1 } @{$custom->{additional_categories}};
            # Duplicate triggers
            foreach my $trigger (flat_triggers($custom)) {
                $temp_pass = $triggers{$trigger}++ ? 0 : 1;
                push(@tests, {msg => "trigger '$trigger' already in use", critical => 1, pass => $temp_pass});
            }
            # Re-adding category
            foreach my $category (keys %categories) {
                $temp_pass = none { $_ eq $category } @{$template_map->{$template_type}};
                push(@tests, {msg => "Category '$category' already assigned", critical => 1, pass => $temp_pass});
            }
        }

        ### Alias tests ###
        if (my $aliases = $json->{aliases}) {
            my @aliases = @{$aliases};
            if ("@aliases" =~ /[[:upper:]]/) {
                push(@tests, {msg => "uppercase detected in alias - aliases should be lowercase", critical => 1});
            }
            if (first { lc $_ eq $defaultName } @aliases) {
                push(@tests, {msg => "aliases should not contain the cheat sheet name ($defaultName)", critical => 1});
            }
            # Make sure aliases don't contain any category triggers.
            while (my ($category, $trigger_types) = each %{$triggers_yaml->{categories}}) {
                my $critical = $categories{$category};
                if (my ($alias, $trigger) = check_aliases_for_triggers(\@aliases, $trigger_types)) {
                    push(@tests, {msg => "alias ($alias) contains a trigger ($trigger) defined in the '$category' category", critical => $critical});
                }
            }
            # Make sure aliases don't contain any custom triggers for the cheat sheet.
            if (my $custom = $triggers_yaml->{custom_triggers}{$cheat_id}) {
                if (my ($alias, $trigger) = check_aliases_for_triggers(\@aliases, $custom)) {
                    push(@tests, {msg => "alias ($alias) contains a custom trigger ($trigger)", critical => 1});
                }
            }
        }
    }

    %categories = (%categories, map { $_ => 1 } @{$template_map->{$template_type}});

    ### Metadata tests ###
    my $has_meta = exists $json->{metadata};

    $temp_pass = $has_meta? 1 : 0;
    push(@tests, {msg => 'has metadata (optional but suggested)', critical => 0, pass => $temp_pass, skip => 1});

    $temp_pass = exists $json->{metadata}{sourceName}? 1 : 0;
    push(@tests, {msg => "has metadata sourceName $name", critical => 1, pass => $temp_pass, skip => 1});

    $temp_pass = exists $json->{metadata}{sourceUrl}? 1 : 0;
    push(@tests, {msg => "has metadata sourceUrl $name", critical => 0, pass => $temp_pass, skip => 1});

    $temp_pass = (my $url = $json->{metadata}{sourceUrl})? 1 : 0;
    push(@tests, {msg => "sourceUrl is not undef $name", critical => 1, pass => $temp_pass, skip => 1});;

    ### Sections tests ###
    $temp_pass = (my $order = $json->{section_order})? 1 : 0;
    push(@tests, {msg => 'has section_order', critical => 1, pass => $temp_pass});

    $temp_pass = (ref $order eq 'ARRAY')? 1 : 0;
    push(@tests, {msg => 'section_order is an array of section names', critical => 1, pass => $temp_pass});

    $temp_pass = (my $sections = $json->{sections})? 1 : 0;
    push(@tests, {msg => 'has sections', critical => 1, pass => $temp_pass});

    $temp_pass = (ref $sections eq 'HASH')? 1 : 0;
    push(@tests, {msg => 'sections is a hash of section key/pairs', critical => 1, pass => $temp_pass});

    my %sections = %$sections;

    for my $section_name (@$order) {
        $temp_pass = defined $sections{$section_name};
        push(@tests, {msg => "Section '$section_name' defined in section_order but not used", critical => 1, pass => $temp_pass});
    }

    for my $section_name (keys %sections) {
        $temp_pass = grep(/\Q$section_name\E/, @$order);
        push(@tests, {msg => "Section '$section_name' used, but not listed in section_order", critical => 1, pass => $temp_pass});

        $temp_pass = (ref $sections{$section_name} eq 'ARRAY')? 1 : 0;
        push(@tests, {msg => "'$section_name' is an array from $name",  critical => 1, pass => $temp_pass});

        my $entry_count = 0;
        for my $entry (@{$sections{$section_name}}) {
            # Only show it when it fails, otherwise it clutters the output
            push(@tests, {msg => "'$section_name' entry: $entry_count has a key from $name", critical => 1, pass => 0}) unless exists $entry->{key};

            #push(@tests, {msg => "'$section_name' entry: $entry_count has a val from $name", critical => 1, pass => 0}) unless exists $entry->{val};
            $entry_count++;
        }
    }

    for my $section_name (keys %sections) {
        for my $entry (@{$sections{$section_name}}){
            # spacing in keys ([a]/[b])'
            if ($entry->{val}) {
                if (($entry->{val} =~ /\(\[.*\]\/\[.+\]\)/g)) {
                    push(@tests, {msg => "keys ([a]/[b]) should have white spaces: $entry->{val} from  $name", critical => 0, pass => 0});
                }
{'fraction_non_alphanumeric': 0.18504672897196262, 'fraction_numerical': 0.00969626168224299, 'mean_word_length': 2.645092612305727, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 244, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '795891', 'n_tokens_mistral': 5701, 'n_tokens_neox': 5365, 'n_words': 1534}