<commit_before>#!/usr/bin/perl -W
# $Id$
# %PSC_START_COPYRIGHT%
# -----------------------------------------------------------------------------
# Copyright (c) 2009-2010, Pittsburgh Supercomputing Center (PSC).
#
# Permission to use, copy, and modify this software and its documentation
# without fee for personal use or non-commercial use within your organization
# is hereby granted, provided that the above copyright notice is preserved in
# all copies and that the copyright and this permission notice appear in
# supporting documentation.  Permission to redistribute this software to other
# organizations or individuals is not permitted without the written permission
# of the Pittsburgh Supercomputing Center.  PSC makes no representations about
# the suitability of this software for any purpose.  It is provided "as is"
# without express or implied warranty.
# -----------------------------------------------------------------------------
# %PSC_END_COPYRIGHT%

use Getopt::Std;
use strict;
use warnings;

sub usage {
	die "usage: $0 file ...\n";
}

sub errx {
	die "$0: ", @_, "\n";
}

sub err {
	die "$0: ", @_, ": $!\n";
}

sub uniq {
	local $_;
	my @uniq;
	my $last = undef;

	foreach (@_) {
		unless (defined $last and $last eq $_) {
			push @uniq, $_;
			$last = $_;
		}
	}
	return @uniq;
}

sub excl {
	my ($master, $mask) = @_;
	my @excl;

	my $j = 0;
	for (my $i = 0; $i < @$master; $i++) {
		for (; $j < @$mask && $master->[$i] gt $mask->[$j]; $j++) {
		}
		push @excl, $master->[$i] unless
		    $j < @$mask && $master->[$i] eq $mask->[$j];
	}
	return (@excl);
}

sub maxlen {
	my ($ra) = @_;
	my $maxlen = 0;
	foreach my $t (@$ra) {
		my $tlen = length $t;
		$maxlen = $tlen if $tlen > $maxlen;
	}
	return ($maxlen);
};

my %opts;
getopts("", \%opts) or usage();
usage() unless @ARGV;

my %syms = (
	defs		=> [],
	funcs		=> [],
	structs		=> [],
	typedefs	=> [],
	undefs		=> [],
	unions		=> [],
	vars		=> [],
);
my %msyms = (
	defs		=> [],
	funcs		=> [],
	structs		=> [],
	typedefs	=> [],
	undefs		=> [],
	unions		=> [],
	vars		=> [],
);
my (@srcs, @hdrs);

foreach my $file (@ARGV) {
	if ($file =~ /\.[cly]$/) {
		push @srcs, $file;
	} elsif ($file =~ /\.h$/) {
		push @hdrs, $file;
	} else {
		errx "$file: unknown file type";
	}
}

foreach my $hdr (@hdrs) {
	local $/;
	open HDR, "<", $hdr or err $hdr;
	my $line = <HDR>;
	close HDR;

	1 while $line =~ s/{[^{}]*?}//gs;
	$line =~ s/\\\n//gs;
	$line =~ s!/\*.*?\*/!!gs;
	$line =~ s!//.*!!gm;
	$line =~ s/^\s*#\s*(?:if|ifdef|ifndef|include|endif|else)\b.*//gm;
	$line =~ s/\n(?=\w+\s*\()/ /gs;
	$line =~ s/\n(?=\s+\*?\w+)/ /gs;

	while ($line =~ s/^\s*#\s*define\s+(\w+).*//m) {
		my $def = $1;
		push @{ $syms{defs} }, $def unless $def =~ /^_/;
	}

	while ($line =~ s/^\s*#\s*undef\s+(\w+)\s*$//m) {
		my $undef = $1;
		push @{ $syms{undefs} }, $undef unless $undef =~ /^_/;
	}

	while ($line =~ s/^\s*struct\s+(\w+)\s*;//m) {
		push @{ $syms{structs} }, $1;
	}

	while ($line =~ s/^\s*typedef\s+(?:(?:struct|union)\s+)?(\w+)\s+(.*)//m) {
		my $name = $2;
		push @{ $syms{typedefs} }, $name =~ /(\w+)/;;
	}

	while ($line =~ s/^.*?(\w+)\s*\(\s*[^*].*//m) {
		push @{ $syms{funcs} }, $1;
	}

	while ($line =~ s/^.*?([a-zA-Z0-9_\[\]]+)\s*;\s*$//m) {
		my $var = $1;
		$var =~ s/\[.*//;
		push @{ $syms{vars} }, $var;
	}

#	$line =~ s/\n\n+/\n/gs;
#	print $line;
}

foreach my $key (keys %syms) {
	$syms{$key} = [ uniq sort @{ $syms{$key} } ];
}

$syms{defs} = [ excl $syms{defs}, $syms{undefs} ];
delete $syms{undefs};

foreach my $src (@srcs) {<commit_msg>handle enums a little better<commit_after>#!/usr/bin/perl -W
# $Id$
# %PSC_START_COPYRIGHT%
# -----------------------------------------------------------------------------
# Copyright (c) 2009-2010, Pittsburgh Supercomputing Center (PSC).
#
# Permission to use, copy, and modify this software and its documentation
# without fee for personal use or non-commercial use within your organization
# is hereby granted, provided that the above copyright notice is preserved in
# all copies and that the copyright and this permission notice appear in
# supporting documentation.  Permission to redistribute this software to other
# organizations or individuals is not permitted without the written permission
# of the Pittsburgh Supercomputing Center.  PSC makes no representations about
# the suitability of this software for any purpose.  It is provided "as is"
# without express or implied warranty.
# -----------------------------------------------------------------------------
# %PSC_END_COPYRIGHT%

use Getopt::Std;
use strict;
use warnings;

sub usage {
	die "usage: $0 file ...\n";
}

sub errx {
	die "$0: ", @_, "\n";
}

sub err {
	die "$0: ", @_, ": $!\n";
}

sub uniq {
	local $_;
	my @uniq;
	my $last = undef;

	foreach (@_) {
		unless (defined $last and $last eq $_) {
			push @uniq, $_;
			$last = $_;
		}
	}
	return @uniq;
}

sub excl {
	my ($master, $mask) = @_;
	my @excl;

	my $j = 0;
	for (my $i = 0; $i < @$master; $i++) {
		for (; $j < @$mask && $master->[$i] gt $mask->[$j]; $j++) {
		}
		push @excl, $master->[$i] unless
		    $j < @$mask && $master->[$i] eq $mask->[$j];
	}
	return (@excl);
}

sub maxlen {
	my ($ra) = @_;
	my $maxlen = 0;
	foreach my $t (@$ra) {
		my $tlen = length $t;
		$maxlen = $tlen if $tlen > $maxlen;
	}
	return ($maxlen);
};

my %opts;
getopts("", \%opts) or usage();
usage() unless @ARGV;

my %syms = (
	defs		=> [],
	funcs		=> [],
	structs		=> [],
	typedefs	=> [],
	undefs		=> [],
	unions		=> [],
	vars		=> [],
);
my %msyms = (
	defs		=> [],
	funcs		=> [],
	structs		=> [],
	typedefs	=> [],
	undefs		=> [],
	unions		=> [],
	vars		=> [],
);
my (@srcs, @hdrs);

foreach my $file (@ARGV) {
	if ($file =~ /\.[cly]$/) {
		push @srcs, $file;
	} elsif ($file =~ /\.h$/) {
		push @hdrs, $file;
	} else {
		errx "$file: unknown file type";
	}
}

foreach my $hdr (@hdrs) {
	local $/;
	open HDR, "<", $hdr or err $hdr;
	my $line = <HDR>;
	close HDR;

	1 while $line =~ s/{[^{}]*?}//gs;
	$line =~ s/\\\n//gs;
	$line =~ s!/\*.*?\*/!!gs;
	$line =~ s!//.*!!gm;
	$line =~ s/^\s*#\s*(?:if|ifdef|ifndef|include|endif|else)\b.*//gm;
	$line =~ s/\n(?=\w+\s*\()/ /gs;
	$line =~ s/\n(?=\s+\*?\w+)/ /gs;

	while ($line =~ s/^\s*#\s*define\s+(\w+).*//m) {
		my $def = $1;
		push @{ $syms{defs} }, $def unless $def =~ /^_/;
	}

	while ($line =~ s/^\s*#\s*undef\s+(\w+)\s*$//m) {
		my $undef = $1;
		push @{ $syms{undefs} }, $undef unless $undef =~ /^_/;
	}

	while ($line =~ s/^\s*struct\s+(\w+)\s*;//m) {
		push @{ $syms{structs} }, $1;
	}

	while ($line =~ s/^\s*typedef\s+(?:(?:struct|union)\s+)?(\w+)\s+(.*)//m) {
		my $name = $2;
		push @{ $syms{typedefs} }, $name =~ /(\w+)/;;
	}

	while ($line =~ s/^\s*enum\s+(\w+)\s*;//m) {
		#push @{ $syms{enums} }, $1;
	}

	while ($line =~ s/^.*?(\w+)\s*\(\s*[^*].*//m) {
		push @{ $syms{funcs} }, $1;
	}

	while ($line =~ s/^.*?([a-zA-Z0-9_\[\]]+)\s*;\s*$//m) {
		my $var = $1;
		$var =~ s/\[.*//;
		push @{ $syms{vars} }, $var;
	}

#	$line =~ s/\n\n+/\n/gs;
#	print $line;
}

foreach my $key (keys %syms) {
	$syms{$key} = [ uniq sort @{ $syms{$key} } ];
}

$syms{defs} = [ excl $syms{defs}, $syms{undefs} ];
delete $syms{undefs};

foreach my $src (@srcs) {
{'fraction_non_alphanumeric': 0.2891498881431767, 'fraction_numerical': 0.006571588366890381, 'mean_word_length': 2.5445986124876114, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '720326', 'n_tokens_mistral': 3160, 'n_tokens_neox': 3005, 'n_words': 748}