<commit_before>#!/usr/bin/perl -w

use strict;
use Getopt::Long "GetOptions";
use FindBin qw($Bin);
use File::Basename;
use File::Temp qw/tempfile/;

my $BITPAR = "/home/pkoehn/statmt/project/bitpar/GermanParser";
my $TMPDIR = "tmp";

my $DEBUG = 0;
my $BASIC = 0;

my $RAW = "";

GetOptions(
    "basic" => \$BASIC,
    "bitpar=s" => \$BITPAR,
    "raw=s" => \$RAW
    ) or die("ERROR: unknown options");

`mkdir -p $TMPDIR`;
my ($scriptname, $directories) = fileparse($0);
my ($TMP, $tmpfile) = tempfile("$scriptname-XXXXXXXXXX", DIR=>$TMPDIR, UNLINK=>1);
open(INPUT,"iconv -c -f utf8 -t iso-8859-1 |");
while(<INPUT>)
{
    foreach (split)
    {
        s/\(/\*LRB\*/g;
        s/\)/\*RRB\*/g;
        print $TMP $_."\n";
    }
    print $TMP "\n";
}
close($TMP);

my $pipeline = "cat $tmpfile | $BITPAR/BitPar/src/bitpar -ts '()' -s TOP -v $BITPAR/GermanParser/Tiger/grammar $BITPAR/GermanParser/Tiger/lexicon -u $BITPAR/GermanParser/Tiger/open-class-tags -w $BITPAR/GermanParser/Tiger/wordclass.txt |";
if ($RAW)
{
    $pipeline .= "tee \"$RAW\" |";
}
$pipeline .= "iconv -c -t utf8 -f iso-8859-1 |";
open(PARSER,$pipeline);
while(my $line = <PARSER>) {
    if ($line =~ /^No parse for/) {
        print "\n";
        next;
    }
    print STDERR $line if $DEBUG;
    chop($line);
    my @LABEL = ();
    my @OUT = ();
    for(my $i=0;$i<length($line);$i++) {
        # print STDERR substr($line,$i)."\n";
	if (substr($line,$i,4) eq "(*T*") {
	   my ($trace,$rest) = split(/\)/,substr($line,$i+1)); 
	   $i+=length($trace)+2;
	   $i++ if substr($line,$i+1,1) eq " ";<commit_msg>add empty line if input is empty line<commit_after>#!/usr/bin/perl -w

use strict;
use Getopt::Long "GetOptions";
use FindBin qw($Bin);
use File::Basename;
use File::Temp qw/tempfile/;

my $BITPAR = "/Users/hieuhoang/workspace/bitpar";
my $TMPDIR = "tmp";

my $DEBUG = 0;
my $BASIC = 0;

my $RAW = "";

GetOptions(
    "basic" => \$BASIC,
    "bitpar=s" => \$BITPAR,
    "raw=s" => \$RAW
    ) or die("ERROR: unknown options");

`mkdir -p $TMPDIR`;
my ($scriptname, $directories) = fileparse($0);
my ($TMP, $tmpfile) = tempfile("$scriptname-XXXXXXXXXX", DIR=>$TMPDIR, UNLINK=>1);
open(INPUT,"iconv -c -f UTF-8 -t iso-8859-1 |");
while(<INPUT>)
{
		my $hasWords = 0;
	  foreach (split)
    {
        s/\(/\*LRB\*/g;
        s/\)/\*RRB\*/g;
        print $TMP $_."\n";

				$hasWords = 1;        
    }
    
    if ($hasWords == 0) {
    	print $TMP " \n";
    }
        
    print $TMP "\n";
}
close($TMP);

my $pipeline = "cat $tmpfile | $BITPAR/BitPar/src/bitpar -ts '()' -s TOP -v $BITPAR/GermanParser/Tiger/grammar $BITPAR/GermanParser/Tiger/lexicon -u $BITPAR/GermanParser/Tiger/open-class-tags -w $BITPAR/GermanParser/Tiger/wordclass.txt |";
if ($RAW)
{
    $pipeline .= "tee \"$RAW\" |";
}
$pipeline .= "iconv -c -t UTF-8 -f iso-8859-1 |";
open(PARSER,$pipeline);
while(my $line = <PARSER>) {
    if ($line =~ /^No parse for/) {
        print "\n";
        next;
    }
    print STDERR $line if $DEBUG;
    chop($line);
    my @LABEL = ();
    my @OUT = ();
    for(my $i=0;$i<length($line);$i++) {
        # print STDERR substr($line,$i)."\n";
	if (substr($line,$i,4) eq "(*T*") {
	   my ($trace,$rest) = split(/\)/,substr($line,$i+1)); 
	   $i+=length($trace)+2;
	   $i++ if substr($line,$i+1,1) eq " ";
{'fraction_non_alphanumeric': 0.230722525804493, 'fraction_numerical': 0.01426836672738312, 'mean_word_length': 2.908659549228944, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4618418', 'n_tokens_mistral': 1412, 'n_tokens_neox': 1359, 'n_words': 292}