<commit_before>#!/usr/bin/perl -w
# driver for tagging time expressions
# this driver is aimed at TDT data
# Author:         <NAME>  - <EMAIL>
# Last modified:  November, 2001
# Copyright 2001  The MITRE Corporation


# Suppress warnings because I want nothing to do with debugging GUTime
local $SIG{__WARN__} = sub{};

## Expecting POS tags of the form <lex pos="NN">dog</lex>
## Change this if you use a different name
SetLexTagName("lex");


###   Nothing to change below this line

### variables added by jfrank ####
#these vars also need to be changed in: TempEx.pm
my $docTag = "DOC";
my $tever = 3;		#Timex Version
my $valTagName = "VAL";

#if set to 1, prints out non-consuming TIMEXes at the end of the
#printout, these timexes are for the begin and end points
my $printNCTs = 0;	

###  End of jfrank variables  ###


# Read one document at a time
$/ = "<\/$docTag>";

use  TempEx;
use  TempEx qw($TEmonthabbr %Month2Num);

$HL        = 3;
$FirstDate = 0;
$TagHeader = 1;
$WARN      = 1;

#Process command line flags
while((defined $ARGV[0]) && ($ARGV[0] =~ /^-/))
{
    $flag = shift(@ARGV);
    if($flag eq "-h") { &Usage(); exit(1); }
    elsif($flag =~ /\A-D(\d)\Z/o)  { $TE_DEBUG  = $1; }
    elsif($flag =~ /\A-HL(\d)\Z/o) { $HL        = $1; }
    elsif($flag =~ /\A-FD\Z/o)     { $FirstDate = 1; }
    elsif($flag =~ /\A-FDNW\Z/o)   { $FirstDate = 1; $WARN = 0;}
    elsif($flag =~ /\A-TH\Z/o)     { $TagHeader = 1; }
    else {
        print "Unknown flag $flag\n";
        &Usage();
        exit(1);
    }
}
<commit_msg>Fixed issue with GUTIme and Perl compatibility (issue #105).<commit_after>#!/usr/bin/perl -w
# driver for tagging time expressions
# this driver is aimed at TDT data
# Author:         <NAME>  - <EMAIL>
# Last modified:  November, 2001
# Copyright 2001  The MITRE Corporation


# Suppress warnings because I want nothing to do with debugging GUTime
local $SIG{__WARN__} = sub{};

## Expecting POS tags of the form <lex pos="NN">dog</lex>
## Change this if you use a different name
SetLexTagName("lex");


###   Nothing to change below this line

### variables added by jfrank ####
#these vars also need to be changed in: TempEx.pm
my $docTag = "DOC";
my $tever = 3;		#Timex Version
my $valTagName = "VAL";

#if set to 1, prints out non-consuming TIMEXes at the end of the
#printout, these timexes are for the begin and end points
my $printNCTs = 0;	

###  End of jfrank variables  ###


# Read one document at a time
$/ = "<\/$docTag>";

use File::Basename;
use lib dirname (__FILE__);

use  TempEx;
use  TempEx qw($TEmonthabbr %Month2Num);

$HL        = 3;
$FirstDate = 0;
$TagHeader = 1;
$WARN      = 1;

#Process command line flags
while((defined $ARGV[0]) && ($ARGV[0] =~ /^-/))
{
    $flag = shift(@ARGV);
    if($flag eq "-h") { &Usage(); exit(1); }
    elsif($flag =~ /\A-D(\d)\Z/o)  { $TE_DEBUG  = $1; }
    elsif($flag =~ /\A-HL(\d)\Z/o) { $HL        = $1; }
    elsif($flag =~ /\A-FD\Z/o)     { $FirstDate = 1; }
    elsif($flag =~ /\A-FDNW\Z/o)   { $FirstDate = 1; $WARN = 0;}
    elsif($flag =~ /\A-TH\Z/o)     { $TagHeader = 1; }
    else {
        print "Unknown flag $flag\n";
        &Usage();
        exit(1);
    }
}

{'fraction_non_alphanumeric': 0.16834804539722573, 'fraction_numerical': 0.01733921815889029, 'mean_word_length': 2.8554070473876063, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '637597', 'n_tokens_mistral': 1244, 'n_tokens_neox': 1165, 'n_words': 376}