<commit_before>#!/usr/bin/perl -w
#----------------------------------------------------------------------
#
# genbki.pl
#    Perl script that generates postgres.bki, postgres.description,
#    postgres.shdescription, and symbol definition headers from specially
#    formatted header files and data files.  The BKI files are used to
#    initialize the postgres template database.
#
# Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group
# Portions Copyright (c) 1994, Regents of the University of California
#
# src/backend/catalog/genbki.pl
#
#----------------------------------------------------------------------

use Catalog;

use strict;
use warnings;

my @input_files;
my $output_path = '';
my $major_version;

# Process command line switches.
while (@ARGV)
{
	my $arg = shift @ARGV;
	if ($arg !~ /^-/)
	{
		push @input_files, $arg;
	}
	elsif ($arg =~ /^-o/)
	{
		$output_path = length($arg) > 2 ? substr($arg, 2) : shift @ARGV;
	}
	elsif ($arg =~ /^--set-version=(.*)$/)
	{
		$major_version = $1;
		die "Invalid version string.\n"
		  if !($major_version =~ /^\d+$/);
	}
	else
	{
		usage();
	}
}

# Sanity check arguments.
die "No input files.\n" if !@input_files;
die "--set-version must be specified.\n" if !defined $major_version;

# Make sure output_path ends in a slash.
if ($output_path ne '' && substr($output_path, -1) ne '/')
{
	$output_path .= '/';
}

# Read all the files into internal data structures.
my @catnames;
my %catalogs;
my %catalog_data;
my @toast_decls;
my @index_decls;
my %oidcounts;

foreach my $header (@input_files)
{
	$header =~ /(.+)\.h$/
	  or die "Input files need to be header files.\n";
	my $datfile = "$1.dat";

	my $catalog = Catalog::ParseHeader($header);
	my $catname = $catalog->{catname};
	my $schema  = $catalog->{columns};

	if (defined $catname)
	{
		push @catnames, $catname;
		$catalogs{$catname} = $catalog;
	}

	# While checking for duplicated OIDs, we ignore the pg_class OID and
	# rowtype OID of bootstrap catalogs, as those are expected to appear
	# in the initial data for pg_class and pg_type.  For regular catalogs,
	# include these OIDs.  (See also Catalog::FindAllOidsFromHeaders
	# if you change this logic.)
	if (!$catalog->{bootstrap})
	{
		$oidcounts{ $catalog->{relation_oid} }++
		  if ($catalog->{relation_oid});
		$oidcounts{ $catalog->{rowtype_oid} }++
		  if ($catalog->{rowtype_oid});
	}

	# Not all catalogs have a data file.
	if (-e $datfile)
	{
		my $data = Catalog::ParseData($datfile, $schema, 0);
		$catalog_data{$catname} = $data;

		# Check for duplicated OIDs while we're at it.
		foreach my $row (@$data)
		{
			$oidcounts{ $row->{oid} }++ if defined $row->{oid};
		}
	}

	# If the header file contained toast or index info, build BKI
	# commands for those, which we'll output later.
	foreach my $toast (@{ $catalog->{toasting} })
	{
		push @toast_decls,
		  sprintf "declare toast %s %s on %s\n",
		  $toast->{toast_oid}, $toast->{toast_index_oid},
		  $toast->{parent_table};
		$oidcounts{ $toast->{toast_oid} }++;
		$oidcounts{ $toast->{toast_index_oid} }++;
	}
	foreach my $index (@{ $catalog->{indexing} })
	{
		push @index_decls,
		  sprintf "declare %sindex %s %s %s\n",
		  $index->{is_unique} ? 'unique ' : '',
		  $index->{index_name}, $index->{index_oid},
		  $index->{index_decl};
		$oidcounts{ $index->{index_oid} }++;
	}
}

# Complain and exit if we found any duplicate OIDs.
# While duplicate OIDs would only cause a failure if they appear in
# the same catalog, our project policy is that manually assigned OIDs
# should be globally unique, to avoid confusion.
my $found = 0;
foreach my $oid (keys %oidcounts)
{
	next unless $oidcounts{$oid} > 1;
	print "Duplicate oids detected:\n" if !$found;
	print "$oid\n";
	$found++;
}
die "found $found duplicate OID(s) in catalog data\n" if $found;

# Fetch some special data that we will substitute into the output file.
# CAUTION: be wary about what symbols you substitute into the .bki file here!
# It's okay to substitute things that are expected to be really constant
# within a given Postgres release, such as fixed OIDs.  Do not substitute
# anything that could depend on platform or configuration.  (The right place
# to handle those sorts of things is in initdb.c's bootstrap_template1().)
my $BOOTSTRAP_SUPERUSERID =
  Catalog::FindDefinedSymbolFromData($catalog_data{pg_authid},
	'BOOTSTRAP_SUPERUSERID');
my $PG_CATALOG_NAMESPACE =
  Catalog::FindDefinedSymbolFromData($catalog_data{pg_namespace},
	'PG_CATALOG_NAMESPACE');
<commit_msg>Write error messages about duplicate OIDs to stderr<commit_after>#!/usr/bin/perl -w
#----------------------------------------------------------------------
#
# genbki.pl
#    Perl script that generates postgres.bki, postgres.description,
#    postgres.shdescription, and symbol definition headers from specially
#    formatted header files and data files.  The BKI files are used to
#    initialize the postgres template database.
#
# Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group
# Portions Copyright (c) 1994, Regents of the University of California
#
# src/backend/catalog/genbki.pl
#
#----------------------------------------------------------------------

use Catalog;

use strict;
use warnings;

my @input_files;
my $output_path = '';
my $major_version;

# Process command line switches.
while (@ARGV)
{
	my $arg = shift @ARGV;
	if ($arg !~ /^-/)
	{
		push @input_files, $arg;
	}
	elsif ($arg =~ /^-o/)
	{
		$output_path = length($arg) > 2 ? substr($arg, 2) : shift @ARGV;
	}
	elsif ($arg =~ /^--set-version=(.*)$/)
	{
		$major_version = $1;
		die "Invalid version string.\n"
		  if !($major_version =~ /^\d+$/);
	}
	else
	{
		usage();
	}
}

# Sanity check arguments.
die "No input files.\n" if !@input_files;
die "--set-version must be specified.\n" if !defined $major_version;

# Make sure output_path ends in a slash.
if ($output_path ne '' && substr($output_path, -1) ne '/')
{
	$output_path .= '/';
}

# Read all the files into internal data structures.
my @catnames;
my %catalogs;
my %catalog_data;
my @toast_decls;
my @index_decls;
my %oidcounts;

foreach my $header (@input_files)
{
	$header =~ /(.+)\.h$/
	  or die "Input files need to be header files.\n";
	my $datfile = "$1.dat";

	my $catalog = Catalog::ParseHeader($header);
	my $catname = $catalog->{catname};
	my $schema  = $catalog->{columns};

	if (defined $catname)
	{
		push @catnames, $catname;
		$catalogs{$catname} = $catalog;
	}

	# While checking for duplicated OIDs, we ignore the pg_class OID and
	# rowtype OID of bootstrap catalogs, as those are expected to appear
	# in the initial data for pg_class and pg_type.  For regular catalogs,
	# include these OIDs.  (See also Catalog::FindAllOidsFromHeaders
	# if you change this logic.)
	if (!$catalog->{bootstrap})
	{
		$oidcounts{ $catalog->{relation_oid} }++
		  if ($catalog->{relation_oid});
		$oidcounts{ $catalog->{rowtype_oid} }++
		  if ($catalog->{rowtype_oid});
	}

	# Not all catalogs have a data file.
	if (-e $datfile)
	{
		my $data = Catalog::ParseData($datfile, $schema, 0);
		$catalog_data{$catname} = $data;

		# Check for duplicated OIDs while we're at it.
		foreach my $row (@$data)
		{
			$oidcounts{ $row->{oid} }++ if defined $row->{oid};
		}
	}

	# If the header file contained toast or index info, build BKI
	# commands for those, which we'll output later.
	foreach my $toast (@{ $catalog->{toasting} })
	{
		push @toast_decls,
		  sprintf "declare toast %s %s on %s\n",
		  $toast->{toast_oid}, $toast->{toast_index_oid},
		  $toast->{parent_table};
		$oidcounts{ $toast->{toast_oid} }++;
		$oidcounts{ $toast->{toast_index_oid} }++;
	}
	foreach my $index (@{ $catalog->{indexing} })
	{
		push @index_decls,
		  sprintf "declare %sindex %s %s %s\n",
		  $index->{is_unique} ? 'unique ' : '',
		  $index->{index_name}, $index->{index_oid},
		  $index->{index_decl};
		$oidcounts{ $index->{index_oid} }++;
	}
}

# Complain and exit if we found any duplicate OIDs.
# While duplicate OIDs would only cause a failure if they appear in
# the same catalog, our project policy is that manually assigned OIDs
# should be globally unique, to avoid confusion.
my $found = 0;
foreach my $oid (keys %oidcounts)
{
	next unless $oidcounts{$oid} > 1;
	print STDERR "Duplicate OIDs detected:\n" if !$found;
	print STDERR "$oid\n";
	$found++;
}
die "found $found duplicate OID(s) in catalog data\n" if $found;

# Fetch some special data that we will substitute into the output file.
# CAUTION: be wary about what symbols you substitute into the .bki file here!
# It's okay to substitute things that are expected to be really constant
# within a given Postgres release, such as fixed OIDs.  Do not substitute
# anything that could depend on platform or configuration.  (The right place
# to handle those sorts of things is in initdb.c's bootstrap_template1().)
my $BOOTSTRAP_SUPERUSERID =
  Catalog::FindDefinedSymbolFromData($catalog_data{pg_authid},
	'BOOTSTRAP_SUPERUSERID');
my $PG_CATALOG_NAMESPACE =
  Catalog::FindDefinedSymbolFromData($catalog_data{pg_namespace},
	'PG_CATALOG_NAMESPACE');

{'fraction_non_alphanumeric': 0.16743878226340173, 'fraction_numerical': 0.004632693580410324, 'mean_word_length': 3.5154382470119523, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3602028', 'n_tokens_mistral': 3244, 'n_tokens_neox': 3063, 'n_words': 973}