<commit_before>#!/usr/bin/perl -w

use lib 'lib';
use Regex;
use strict;

my $operation;
my $expr;
my $tree_opt = 1;
my $list_opt = 1;
my $debug = 0;
foreach (@ARGV) {
    if (/--no(-?)optimize/) {
        $tree_opt = 0;
        $list_opt = 0;
    } elsif (/--optimize=(.*)/) {
        my $opts = $1;
        $tree_opt = ($opts =~ /t/i);
        $list_opt = ($opts =~ /l/i);
    } elsif (/--debug/ || $_ eq '-d') {
        $debug = 1;
    } elsif (! defined $expr) {
        $expr = $_;
    } elsif (! defined $operation) {
        $operation = $_;
    } else {
        die "too many args!";
    }
}

die "need expression!" if ! defined $expr;
$operation ||= "compile";

my %options;
$options{'no-tree-optimize'} = 1 if ! $tree_opt;
$options{'no-list-optimize'} = 1 if ! $list_opt;
$options{'DEBUG'} = 1 if $debug;

my $tree = Regex::expr_to_tree($expr, %options);

if ($operation eq 'unparse' || $operation eq 'render') {
    print $tree->render(), "\n";
    exit;
} elsif ($operation eq 'dump') {
    print $tree->dump_tree(), "\n";
    exit;
}

my @code = Regex::tree_to_list($tree, %options);

my @asm = Regex::list_to_pasm(\@code, %options);

print join("\n", @asm), "\n";<commit_msg>Update to new types<commit_after>#!/usr/bin/perl -w

use lib 'lib';
use Regex;
use strict;

my $operation;
my $expr;
my $tree_opt = 1;
my $list_opt = 1;
my $debug = 0;
foreach (@ARGV) {
    if (/--no(-?)optimize/) {
        $tree_opt = 0;
        $list_opt = 0;
    } elsif (/--optimize=(.*)/) {
        my $opts = $1;
        $tree_opt = ($opts =~ /t/i);
        $list_opt = ($opts =~ /l/i);
    } elsif (/--debug/ || $_ eq '-d') {
        $debug = 1;
    } elsif (! defined $expr) {
        $expr = $_;
    } elsif (! defined $operation) {
        $operation = $_;
    } else {
        die "too many args!";
    }
}

die "need expression!" if ! defined $expr;
$operation ||= "compile";

my %options;
$options{'no-tree-optimize'} = 1 if ! $tree_opt;
$options{'no-list-optimize'} = 1 if ! $list_opt;
$options{'DEBUG'} = 1 if $debug;

my $tree = Regex::expr_to_tree($expr, %options);

if ($operation eq 'unparse' || $operation eq 'render') {
    print $tree->render(), "\n";
    exit;
} elsif ($operation eq 'dump') {
    print $tree->dump_tree(), "\n";
    exit;
}

my $code = Regex::tree_to_list($tree, %options);

my @asm = Regex::list_to_pasm($code, %options);

print join("\n", @asm), "\n";
{'fraction_non_alphanumeric': 0.21695342005874949, 'fraction_numerical': 0.00839278220730172, 'mean_word_length': 2.6733436055469952, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6505456', 'n_tokens_mistral': 943, 'n_tokens_neox': 924, 'n_words': 236}