<commit_before>#!/usr/bin/perl

# Script to verify existence and correctness of source code headers.

# Usage: perl scripts/check-headers.pl [subdirectory ...]

use strict;

my $dir = `dirname "$0"`;
chop $dir;
require "$dir/subs.pl";

my %knownAuthors = (
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,
);

# parse command line arguments
my @args;
if (scalar @ARGV == 0) {
  @args = ('.');
}
else {
  @args = @ARGV;
}

# read copyright file
my @copyright = readFile("LICENSE.txt");
push(@copyright, '*/');
push(@copyright, '');

# find source files
my $cmd = "find @args -name '*.java'";
my @src = `$cmd`;

# process files
for my $file (@src) {
  chop $file;
  process($file);
}

sub process {
  my ($file) = @_;

  my $dir = `dirname "$file"`;
  chop $dir;
  my $base = `basename "$file"`;
  chop $base;
  my $class = substr($base, 0, length($base) - 5);

  # read in source file
  my @data = readFile($file);

  # check header comment
  my $i = 0;
  my @header = ('//', "// $base", '//', '', '/*');
  if (!match(\@header, \@data, $i)) {
    print "$file: invalid header\n";
    return;
  }
  $i += @header;

  # check copyright statement
  if (!match(\@copyright, \@data, $i)) {
    print "$file: invalid copyright\n";
    return;
  }
  $i += @copyright;

  # check package statement
  if ($data[$i++] !~ /^package .*;$/) {
    print "$file: invalid package\n";
    return;
  }

  # check import statements
  my $blank = 0;
  while (1) {
    my $line = trim($data[$i++]);
    if ($line =~ /^$/) {
      if ($blank) {
        print "$file: duplicate blank line at line #$i\n";
        return;
      }
      $blank = 1;
    }
    else {
      $blank = 0;
      if ($line eq '/**') {
        last;
      }
      elsif ($line !~ /\/\// && $line !~ /^import /) {
        print "$file: unexpected text at line #$i\n";
        return;
      }
    }
  }

  if ($data[$i] !~ /^ \* [^\s]/) {
    print "$file: malformed class comment at line #$i\n";
    return;
  }

  # check class comment
  my $author = 0;
  while (1) {<commit_msg>Allow additional comments in header checker.<commit_after>#!/usr/bin/perl

# Script to verify existence and correctness of source code headers.

# Usage: perl scripts/check-headers.pl [subdirectory ...]

use strict;

my $dir = `dirname "$0"`;
chop $dir;
require "$dir/subs.pl";

my %knownAuthors = (
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,
  "<NAME>" => 1,

  "<NAME>" => 1,
  "<NAME>shofer" => 1,
);

# parse command line arguments
my @args;
if (scalar @ARGV == 0) {
  @args = ('.');
}
else {
  @args = @ARGV;
}

# read copyright file
my @copyright = readFile("LICENSE.txt");
push(@copyright, '*/');
push(@copyright, '');

# find source files
my $cmd = "find @args -name '*.java'";
my @src = `$cmd`;

# process files
for my $file (@src) {
  chop $file;
  process($file);
}

sub process($) {
  my ($file) = @_;

  my $dir = `dirname "$file"`;
  chop $dir;
  my $base = `basename "$file"`;
  chop $base;
  my $class = substr($base, 0, length($base) - 5);

  # read in source file
  my @data = readFile($file);

  # check header comment
  my $i = 0;
  my @header = ('//', "// $base", '//', '', '/*');
  if (!match(\@header, \@data, $i)) {
    print "$file: invalid header\n";
    return;
  }
  $i += @header;

  # check copyright statement
  if (!match(\@copyright, \@data, $i)) {
    print "$file: invalid copyright\n";
    return;
  }
  $i += @copyright;

  # check for optional additional comments
  my $blank = 0;
  while (1) {
    if ($data[$i] =~ /^\/\//) {
      # single line comments are OK; continue
      $blank = 0;
    }
    elsif ($data[$i] =~ /^\/\*/) {
      # multi-line comments are OK; search for end of comment
      while ($i < @data && $data[$i++] !~ /\*\//) { }
      $blank = 0;
    }
    elsif ($data[$i] =~ /^\s*$/) {
      if ($blank) {
        print "$file: duplicate blank line at line #$i\n";
        return;
      }
      $blank = 1;
    }
    else {
      # not an additional comment; move on
      last;
    }
    $i++;
  }

  # check package statement
  if ($data[$i++] !~ /^package .*;$/) {
    print "$file: invalid package\n";
    return;
  }

  # check import statements
  $blank = 0;
  while (1) {
    my $line = trim($data[$i++]);
    if ($line =~ /^$/) {
      if ($blank) {
        print "$file: duplicate blank line at line #$i\n";
        return;
      }
      $blank = 1;
    }
    else {
      $blank = 0;
      if ($line eq '/**') {
        last;
      }
      elsif ($line !~ /\/\// && $line !~ /^import /) {
        print "$file: unexpected text at line #$i\n";
        return;
      }
    }
  }

  if ($data[$i] !~ /^ \* [^\s]/) {
    print "$file: malformed class comment at line #$i\n";
    return;
  }

  # check class comment
  my $author = 0;
  while (1) {
{'fraction_non_alphanumeric': 0.2222222222222222, 'fraction_numerical': 0.010515873015873017, 'mean_word_length': 2.312089356110381, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1172144', 'n_tokens_mistral': 1946, 'n_tokens_neox': 1901, 'n_words': 536}