<commit_before>#!/usr/bin/perl
#
# Script to create Makefile-style dependencies.
#
# Usage: perl [-s path-separator] [-o obj-ext] mkdep.pl dir... > deps
#

use File::Spec;
use File::Basename;
use Fcntl;

$barrier = "#-- Everything below is generated by mkdep.pl - do not edit --#\n";

#
# Scan files for dependencies
#
sub scandeps($) {
    my($file) = @_;
    my($line, $nf);
    my(@xdeps) = ();
    my(@mdeps) = ();

    sysopen(FILE, $file, O_RDONLY)
	or return;		# If not openable, assume generated

    while ( defined($line = <FILE>) ) {
	chomp $line;
	$line =~ s:/\*.*\*/::g;
	$line =~ s://.*$::;
	if ( $line =~ /^\s*\#\s*include\s+\"(.*)\"\s*$/ ) {
	    $nf = $1;
	    push(@mdeps, $nf);
	    push(@xdeps, $nf) unless ( defined($deps{$nf}) );
	}
    }
    close(FILE);
    $deps{$file} = [@mdeps];

    foreach $file ( @xdeps ) {
	scandeps($file);
    }
}

# %deps contains direct dependencies.  This subroutine resolves
# indirect dependencies that result.
sub alldeps($) {
    my($file) = @_;
    my(%adeps);
    my($dep,$idep);

    foreach $dep ( @{$deps{$file}} ) {
	$adeps{$dep} = 1;
	foreach $idep ( alldeps($dep) ) {
	    $adeps{$idep} = 1;
	}
    }
    return keys(%adeps);
}

# This converts a filename from host syntax to target syntax
# This almost certainly works only on relative filenames...
sub convert_file($$) {
    my($file,$sep) = @_;
    my(@fspec) = (basename($file));
    while ( ($file = dirname($file)) ne File::Spec->curdir() &&
	    $file ne File::Spec->rootdir() ) {
	unshift(@fspec, basename($file));
    }

    return join($sep, @fspec);
}

#
# Insert dependencies into a Makefile
#
sub insert_deps($) {
    my($file) = @_;
    $nexttemp++;		# Unique serial number for each temp file
    my($tmp) = File::Spec->catfile(dirname($file), 'tmp.'.$nexttemp);
    
    sysopen(IN, $file, O_RDONLY)
	or die "$0: Cannot open input: $file\n";
    sysopen(OUT, $tmp, O_WRONLY|O_CREAT|O_TRUNC, 0666)
	or die "$0: Cannot open output: $tmp\n";

    my($line,$parm,$val);
    my($obj) = '.o';		# Defaults
    my($sep) = '/';

    while ( defined($line = <IN>) ) {
	if ( $line =~ /^\s*\#\s*@([a-z0-9-]+):\s*\"([^\"]*)\"/ ) {
	    $parm = $1;  $val = $2;
	    if ( $parm eq 'object-ending' ) {
		$obj = $val;
	    } elsif ( $parm eq 'path-separator' ) {
		$sep = $val;
	    }
	} elsif ( $line eq $barrier ) {
	    last;		# Stop reading input at barrier line
	}
	print OUT $line;
    }
    close(IN);

    my $dfile, $ofile;
    my @deps, $dep;

    print OUT $barrier;

    foreach $dfile ( sort(keys(%deps)) ) {
	if ( $dfile =~ /\.[Cc]$/ ) {
	    $ofile = $dfile; $ofile =~ s/\.[Cc]$//;
	    print OUT convert_file($ofile,$sep), $obj, ':';
	    foreach $dep ($dfile, alldeps($dfile)) {
		print OUT ' ', convert_file($dep,$sep);
	    }
	    print OUT "\n";
	}
    }
    close(OUT);
    
    (unlink($file) && rename($tmp, $file)) 
	or die "$0: Failed to change $tmp -> $file\n";
}

#
# Main program
#
    
%deps = ();
@files = ();
@mkfiles = ();
$mkmode = 0;

while ( defined($arg = shift(@ARGV)) ) {
    if ( $arg eq '-m' ) {
	$arg = shift(@ARGV);
	push(@mkfiles, $arg);
    } elsif ( $arg eq '-M' ) {
	$mkmode = 1;		# Futher filenames are output Makefile names
    } elsif ( $arg eq '--' && $mkmode ) {
	$mkmode = 0;
    } elsif ( $arg =~ /^-/ ) {
	die "Unknown option: $arg\n";
    } else {
	if ( $mkmode ) {<commit_msg>Break long lines by default.<commit_after>#!/usr/bin/perl
#
# Script to create Makefile-style dependencies.
#
# Usage: perl [-s path-separator] [-o obj-ext] mkdep.pl dir... > deps
#

use File::Spec;
use File::Basename;
use Fcntl;

$barrier = "#-- Everything below is generated by mkdep.pl - do not edit --#\n";

#
# Scan files for dependencies
#
sub scandeps($) {
    my($file) = @_;
    my($line, $nf);
    my(@xdeps) = ();
    my(@mdeps) = ();

    sysopen(FILE, $file, O_RDONLY)
	or return;		# If not openable, assume generated

    while ( defined($line = <FILE>) ) {
	chomp $line;
	$line =~ s:/\*.*\*/::g;
	$line =~ s://.*$::;
	if ( $line =~ /^\s*\#\s*include\s+\"(.*)\"\s*$/ ) {
	    $nf = $1;
	    push(@mdeps, $nf);
	    push(@xdeps, $nf) unless ( defined($deps{$nf}) );
	}
    }
    close(FILE);
    $deps{$file} = [@mdeps];

    foreach $file ( @xdeps ) {
	scandeps($file);
    }
}

# %deps contains direct dependencies.  This subroutine resolves
# indirect dependencies that result.
sub alldeps($) {
    my($file) = @_;
    my(%adeps);
    my($dep,$idep);

    foreach $dep ( @{$deps{$file}} ) {
	$adeps{$dep} = 1;
	foreach $idep ( alldeps($dep) ) {
	    $adeps{$idep} = 1;
	}
    }
    return keys(%adeps);
}

# This converts a filename from host syntax to target syntax
# This almost certainly works only on relative filenames...
sub convert_file($$) {
    my($file,$sep) = @_;
    my(@fspec) = (basename($file));
    while ( ($file = dirname($file)) ne File::Spec->curdir() &&
	    $file ne File::Spec->rootdir() ) {
	unshift(@fspec, basename($file));
    }

    return join($sep, @fspec);
}

#
# Insert dependencies into a Makefile
#
sub insert_deps($) {
    my($file) = @_;
    $nexttemp++;		# Unique serial number for each temp file
    my($tmp) = File::Spec->catfile(dirname($file), 'tmp.'.$nexttemp);
    
    sysopen(IN, $file, O_RDONLY)
	or die "$0: Cannot open input: $file\n";
    sysopen(OUT, $tmp, O_WRONLY|O_CREAT|O_TRUNC, 0666)
	or die "$0: Cannot open output: $tmp\n";

    my($line,$parm,$val);
    my($obj) = '.o';		# Defaults
    my($sep) = '/';
    my($maxline) = 78;		# SMTP uses this value...

    while ( defined($line = <IN>) ) {
	if ( $line =~ /^\s*\#\s*@([a-z0-9-]+):\s*\"([^\"]*)\"/ ) {
	    $parm = $1;  $val = $2;
	    if ( $parm eq 'object-ending' ) {
		$obj = $val;
	    } elsif ( $parm eq 'path-separator' ) {
		$sep = $val;
	    } elsif ( $parm eq 'line-width' ) {
		$maxline = $val+0;
	    }
	} elsif ( $line eq $barrier ) {
	    last;		# Stop reading input at barrier line
	}
	print OUT $line;
    }
    close(IN);

    my $dfile, $ofile, $str, $sl, $len;
    my @deps, $dep;

    print OUT $barrier;

    foreach $dfile ( sort(keys(%deps)) ) {
	if ( $dfile =~ /\.[Cc]$/ ) {
	    $ofile = $dfile; $ofile =~ s/\.[Cc]$//;
	    $str = convert_file($ofile,$sep).$obj.':';
	    $len = length($str);
	    print OUT $str;
	    foreach $dep ($dfile, alldeps($dfile)) {
		$str = convert_file($dep,$sep);
		$sl = length($str)+1;
		if ( $len+$sl > $maxline-2 ) {
		    print OUT " \\\n ", $str;
		    $len = $sl;
		} else {
		    print OUT ' ', $str;
		    $len += $sl;
		}
	    }
	    print OUT "\n";
	}
    }
    close(OUT);
    
    (unlink($file) && rename($tmp, $file)) 
	or die "$0: Failed to change $tmp -> $file\n";
}

#
# Main program
#
    
%deps = ();
@files = ();
@mkfiles = ();
$mkmode = 0;

while ( defined($arg = shift(@ARGV)) ) {
    if ( $arg eq '-m' ) {
	$arg = shift(@ARGV);
	push(@mkfiles, $arg);
    } elsif ( $arg eq '-M' ) {
	$mkmode = 1;		# Futher filenames are output Makefile names
    } elsif ( $arg eq '--' && $mkmode ) {
	$mkmode = 0;
    } elsif ( $arg =~ /^-/ ) {
	die "Unknown option: $arg\n";
    } else {
	if ( $mkmode ) {
{'fraction_non_alphanumeric': 0.2200909349246945, 'fraction_numerical': 0.005541346973572037, 'mean_word_length': 2.6604264170566823, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6007608', 'n_tokens_mistral': 2949, 'n_tokens_neox': 2818, 'n_words': 674}