<commit_before>#!/usr/bin/perl

# $Id: reuse-weights.perl 41 2007-03-14 22:54:18Z hieu $

use strict;

die("ERROR: syntax: reuse-weights.perl weights.ini < moses.ini > weighted.ini")
    unless scalar @ARGV == 1;
my ($weight_file) = @ARGV;

my %WEIGHT;
my $current_weight = "";
open(WEIGHT,$weight_file)
    || die("ERROR: could not open weight file: $weight_file");
while(<WEIGHT>) {
    if (/^\[weight\-(\S+)\]/) {
	$current_weight = $1;
    }
    elsif ($current_weight && /^([\-\d\.e]+)\s*$/) {
	push @{$WEIGHT{$current_weight}},$1;
    }
    elsif (/^\[/) {
	$current_weight = "";
    }
}
close(WEIGHT);

my %IGNORE;
while(<STDIN>) {
    if (/^\[weight\-(\S+)\]/) {
	$current_weight = $1;
	if (!defined($WEIGHT{$current_weight})) {    
	    print STDERR "(reuse-weights) WARNING: no weights for weight-$1, deleting\n";
	    $current_weight = "xxx";
	}
	else {
	    print $_;
	    foreach (@{$WEIGHT{$current_weight}}) {
		print $_."\n";
	    }
	}
    }
    elsif ($current_weight && /^([\-\d\.]+)\s*$/) {
	$IGNORE{$current_weight}++;
    }
    elsif (/^\[/) {
	$current_weight = "";
	print $_;
    }
    else {
	print $_;
    }
}

foreach my $weight (keys %WEIGHT) {
    if (! defined($IGNORE{$weight})) {
	print STDERR "(reuse-weights) WARNING: new weights weight-$weight\n";
	print "\n[weight-$weight]\n";
	foreach (@{$WEIGHT{$weight}}) {<commit_msg>There is potential bug for parsing numbers in scientific notation.<commit_after>#!/usr/bin/perl

# $Id: reuse-weights.perl 41 2007-03-14 22:54:18Z hieu $

use strict;

die("ERROR: syntax: reuse-weights.perl weights.ini < moses.ini > weighted.ini")
    unless scalar @ARGV == 1;
my ($weight_file) = @ARGV;

my %WEIGHT;
my $current_weight = "";
open(WEIGHT,$weight_file)
    || die("ERROR: could not open weight file: $weight_file");
while(<WEIGHT>) {
    if (/^\[weight\-(\S+)\]/) {
	$current_weight = $1;
    }
  elsif ($current_weight && /^([\-\d\.]+)([Ee][+-]?[\d]+)?$/) {
	push @{$WEIGHT{$current_weight}},$1;
    }
    elsif (/^\[/) {
	$current_weight = "";
    }
}
close(WEIGHT);

my %IGNORE;
while(<STDIN>) {
    if (/^\[weight\-(\S+)\]/) {
	$current_weight = $1;
	if (!defined($WEIGHT{$current_weight})) {    
	    print STDERR "(reuse-weights) WARNING: no weights for weight-$1, deleting\n";
	    $current_weight = "xxx";
	}
	else {
	    print $_;
	    foreach (@{$WEIGHT{$current_weight}}) {
		print $_."\n";
	    }
	}
    }
    elsif ($current_weight && /^([\-\d\.]+)([Ee][+-]?[\d]+)?$/) {
	$IGNORE{$current_weight}++;
    }
    elsif (/^\[/) {
	$current_weight = "";
	print $_;
    }
    else {
	print $_;
    }
}

foreach my $weight (keys %WEIGHT) {
    if (! defined($IGNORE{$weight})) {
	print STDERR "(reuse-weights) WARNING: new weights weight-$weight\n";
	print "\n[weight-$weight]\n";
	foreach (@{$WEIGHT{$weight}}) {
{'fraction_non_alphanumeric': 0.23173352435530087, 'fraction_numerical': 0.01504297994269341, 'mean_word_length': 3.1748878923766815, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '916641', 'n_tokens_mistral': 1148, 'n_tokens_neox': 1063, 'n_words': 209}