<commit_before>#!/usr/bin/perl -w
#==============================================================================
#
#    Copyright (C) 2002 by F.Richter <<EMAIL>>
#
#    This program is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 2 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
#==============================================================================

# Purpose of this script: Create an HTMLHELP TOC out of the Crystal Space docs.
# HHW's automatic TOC generation is not entirely poor, but it does not handle
# the CS docs too well, thus we do it ourselves.  This script is not really
# useful for anything other than the CS docs.  The HH compiler will complain
# about unresolveable URLs in the TOC. Don't worry, it'll work fine, though.

use strict;
use HTMLEntities;

#
# the index needs some special treatment
#
sub doctoc_index {
  	(my $docroot, my $srcfile, my $TOCFILE) = @_;
  	my %indexfiles;
  	my %letterindexed;

  	open(my $INDEXFILE, "<$docroot/$srcfile");
  	while (<$INDEXFILE>) {
  			chomp $_;
				if ( /<A HREF=\"(.*)\#(.*)\" style=\"text-decoration:none\"><b>(.*)<\/b><\/A>/ ) { # a letter
						if (!exists($letterindexed{$3})) {
  							print $TOCFILE <<EOTOCENTRY;
<LI> <OBJECT type="text/sitemap">
  <param name="Name" value="$3">
  <param name="Local" value="$docroot\\$1#$2">
</OBJECT>
EOTOCENTRY
								$indexfiles{$1} = 1;
								$letterindexed{$3} = 1;
						}
				}
  	}
  	close($INDEXFILE);
  	
  	my %indexwords;
  	
  	foreach (keys %indexfiles) {
  			open(my $INDEXFILE, "<$docroot/$_");
  			while (<$INDEXFILE>) {
  					chomp $_;
  					if ( /<TR><TD><\/TD><TD valign=top><A HREF=\"(.*)\">(.*)<\/A><\/TD><TD valign=top><A HREF=\".*\">(.*)<\/A><\/TD><\/TR>/ ) {
		  					my $ilink = $1;
		  					my $igroup = $2;
		  					my $ititle = $3;
		  					$igroup =~ s/(<.*?>)//g; # strip html code
		  					$igroup = HTMLEntities::encode($igroup);
		  					$ititle =~ s/(<.*?>)//g; 
		  					#$ititle =~ s/^([[:digit:]\.]*?) //; # strip leading section number
		  					$ititle = HTMLEntities::encode($ititle);
 								push(@{$indexwords{$igroup}}, [$ititle, $ilink]);
  					}
  			}
  			close($INDEXFILE);
  	}
  	
  	open(my $HHKFILE, ">manualindex.hhk");
	  print $HHKFILE <<EOHHKHEAD;
<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN">
<HTML>
<HEAD>
<meta name="GENERATOR" content="gendoctoc.pl">
<!-- Sitemap 1.0 -->
</HEAD><BODY>
<UL>
EOHHKHEAD
 <commit_msg>Updated to correctly parse the "index" produced by texi2html.pl v1.76.<commit_after>#!/usr/bin/perl -w
#==============================================================================
#
#    Copyright (C) 2002 by F.Richter <<EMAIL>>
#
#    This program is free software; you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation; either version 2 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
#==============================================================================

# Purpose of this script: Create an HTMLHELP TOC out of the Crystal Space docs.
# HHW's automatic TOC generation is not entirely poor, but it does not handle
# the CS docs too well, thus we do it ourselves.  This script is not really
# useful for anything other than the CS docs.  The HH compiler will complain
# about unresolveable URLs in the TOC. Don't worry, it'll work fine, though.

use strict;
use HTMLEntities;

#
# the index needs some special treatment
#
sub doctoc_index {
  	(my $docroot, my $srcfile, my $TOCFILE) = @_;
  	my %indexfiles;
  	my %letterindexed;

  	open(my $INDEXFILE, "<$docroot/$srcfile");
  	while (<$INDEXFILE>) {
  			chomp $_;
				if ( /<A HREF=\"(.*?)\#(.*?)\" class=\"summary-letter\"><b>(.*?)<\/b><\/A>/i ) { # a letter
						if (!exists($letterindexed{$3})) {
  							print $TOCFILE <<EOTOCENTRY;
<LI> <OBJECT type="text/sitemap">
  <param name="Name" value="$3">
  <param name="Local" value="$docroot\\$1#$2">
</OBJECT>
EOTOCENTRY
								$indexfiles{$1} = 1;
								$letterindexed{$3} = 1;
						}
				}
  	}
  	close($INDEXFILE);
  	
  	my %indexwords;
  	
  	foreach (keys %indexfiles) {
  			open(my $INDEXFILE, "<$docroot/$_");
  			while (<$INDEXFILE>) {
  					chomp $_;
  					if ( /<TR><TD><\/TD><TD valign=\"top\"><A HREF=\"(.*?)\">(.*?)<\/A><\/TD><TD valign=\"top\"><A HREF=\".*?\">(.*?)<\/A><\/TD><\/TR>/i ) {
		  					my $ilink = $1;
		  					my $igroup = $2;
		  					my $ititle = $3;
		  					$igroup =~ s/(<.*?>)//g; # strip html code
		  					$igroup = HTMLEntities::encode($igroup);
		  					$ititle =~ s/(<.*?>)//g; 
		  					#$ititle =~ s/^([[:digit:]\.]*?) //; # strip leading section number
		  					$ititle = HTMLEntities::encode($ititle);
 								push(@{$indexwords{$igroup}}, [$ititle, $ilink]);
  					}
  			}
  			close($INDEXFILE);
  	}
  	
  	open(my $HHKFILE, ">manualindex.hhk");
	  print $HHKFILE <<EOHHKHEAD;
<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN">
<HTML>
<HEAD>
<meta name="GENERATOR" content="gendoctoc.pl">
<!-- Sitemap 1.0 -->
</HEAD><BODY>
<UL>
EOHHKHEAD
 
{'fraction_non_alphanumeric': 0.20774762994442628, 'fraction_numerical': 0.011114743380189604, 'mean_word_length': 3.1372549019607843, 'pattern_counts': {'":': 0, '<': 85, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3083026', 'n_tokens_mistral': 2354, 'n_tokens_neox': 2036, 'n_words': 631}