<commit_before>#!/usr/bin/perl -w
use strict;
use Getopt::Long;
use Cwd;
my $CWD = getcwd;

my $SCRIPT_DIR; BEGIN { use Cwd qw/ abs_path /; use File::Basename; $SCRIPT_DIR = dirname(abs_path($0)); push @INC, $SCRIPT_DIR; }

# featurize_grammar may add multiple features from a single feature extractor
# the key in this map is the extractor name, the value is a list of the extracted features
my $feat_map = {
  "LogRuleCount" => [ "LogRuleCount", "SingletonRule" ] ,
#  "XFeatures" => [ "XFE","XEF" ] ,
  "XFeatures" => [ "XFE","XEF","LabelledEF","LabelledFE"] ,
  "LabelledRuleConditionals" => [ "LabelledFE","LabelledEF" ] ,
  "LexProb" => [ "LexE2F", "LexF2E" ] ,
  "BackoffRule" => [ "BackoffRule" ] ,
  "RulePenalty" => [ "RulePenalty" ] ,
  "LHSProb" => [ "LHSProb" ] ,
  "LabellingShape" => [ "LabellingShape" ] ,
  "GenerativeProb" => [ "GenerativeProb" ] ,
};

my %init_weights = qw(
  EGivenF -0.735245
  FGivenE -0.219391
  Glue -0.306709
  GlueTop 0.0473331
  LanguageModel 2.40403
  LexE2F -0.266989
  LexF2E -0.550373
  LogECount -0.129853
  LogFCount -0.194037
  LogRuleCount 0.256706
  BackoffRule 0.5
  XFE -0.256706
  XEF -0.256706
  LabelledFE -0.256706
  LabelledEF -0.256706
  PassThrough -0.9304905
  SingletonE -3.04161
  SingletonF 0.0714027
  SingletonRule -0.889377
  WordPenalty -1.99495
  RulePenalty -0.1
  LabellingShape -0.1
  LHSProb -0.1
  GenerativeProb -0.1
);


# these features are included by default
my @DEFAULT_FEATS = qw( Glue GlueTop LanguageModel WordPenalty );



my $CDEC = "$SCRIPT_DIR/../../decoder/cdec";
my $PARALLELIZE = "$SCRIPT_DIR/../../vest/parallelize.pl";
my $EXTOOLS = "$SCRIPT_DIR/../../extools";
die "Can't find extools: $EXTOOLS" unless -e $EXTOOLS && -d $EXTOOLS;
my $VEST = "$SCRIPT_DIR/../../vest";
die "Can't find vest: $VEST" unless -e $VEST && -d $VEST;
my $DISTVEST = "$VEST/dist-vest.pl";
my $FILTER = "$EXTOOLS/filter_grammar";
my $FEATURIZE = "$EXTOOLS/featurize_grammar";
assert_exec($CDEC, $PARALLELIZE, $FILTER, $FEATURIZE, $DISTVEST);

my $numtopics = 25;
<commit_msg>forgotten feature<commit_after>#!/usr/bin/perl -w
use strict;
use Getopt::Long;
use Cwd;
my $CWD = getcwd;

my $SCRIPT_DIR; BEGIN { use Cwd qw/ abs_path /; use File::Basename; $SCRIPT_DIR = dirname(abs_path($0)); push @INC, $SCRIPT_DIR; }

# featurize_grammar may add multiple features from a single feature extractor
# the key in this map is the extractor name, the value is a list of the extracted features
my $feat_map = {
  "LogRuleCount" => [ "LogRuleCount", "SingletonRule" ] ,
#  "XFeatures" => [ "XFE","XEF" ] ,
  "XFeatures" => [ "XFE","XEF","LabelledEF","LabelledFE"] ,
  "LabelledRuleConditionals" => [ "LabelledFE","LabelledEF" ] ,
  "LexProb" => [ "LexE2F", "LexF2E" ] ,
  "BackoffRule" => [ "BackoffRule" ] ,
  "RulePenalty" => [ "RulePenalty" ] ,
  "LHSProb" => [ "LHSProb" ] ,
  "LabellingShape" => [ "LabellingShape" ] ,
  "GenerativeProb" => [ "GenerativeProb" ] ,
};

my %init_weights = qw(
  EGivenF -0.735245
  FGivenE -0.219391
  Glue -0.306709
  GlueTop 0.0473331
  LanguageModel 2.40403
  LexE2F -0.266989
  LexF2E -0.550373
  LogECount -0.129853
  LogFCount -0.194037
  LogRuleCount 0.256706
  BackoffRule 0.5
  XFE -0.256706
  XEF -0.256706
  LabelledFE -0.256706
  LabelledEF -0.256706
  PassThrough -0.9304905
  SingletonE -3.04161
  SingletonF 0.0714027
  SingletonRule -0.889377
  WordPenalty -1.99495
  RulePenalty -0.1
  LabellingShape -0.1
  LHSProb -0.1
  GenerativeProb -0.1
);


# these features are included by default
my @DEFAULT_FEATS = qw( PassThrough Glue GlueTop LanguageModel WordPenalty );



my $CDEC = "$SCRIPT_DIR/../../decoder/cdec";
my $PARALLELIZE = "$SCRIPT_DIR/../../vest/parallelize.pl";
my $EXTOOLS = "$SCRIPT_DIR/../../extools";
die "Can't find extools: $EXTOOLS" unless -e $EXTOOLS && -d $EXTOOLS;
my $VEST = "$SCRIPT_DIR/../../vest";
die "Can't find vest: $VEST" unless -e $VEST && -d $VEST;
my $DISTVEST = "$VEST/dist-vest.pl";
my $FILTER = "$EXTOOLS/filter_grammar";
my $FEATURIZE = "$EXTOOLS/featurize_grammar";
assert_exec($CDEC, $PARALLELIZE, $FILTER, $FEATURIZE, $DISTVEST);

my $numtopics = 25;

{'fraction_non_alphanumeric': 0.16089712335446124, 'fraction_numerical': 0.07313505607020965, 'mean_word_length': 3.867141162514828, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1192246', 'n_tokens_mistral': 1927, 'n_tokens_neox': 1667, 'n_words': 382}