<commit_before>#! /usr/bin/env python

"""Script to convert raw module index data to module index."""

import os
import string
import sys


def parse_line(input):
    lineno = string.split(input)[-1]
    module = string.strip(input[:len(input)-(len(lineno)+1)])
    return module, lineno


def cmp_items((s1, line1), (s2, line2)):
    rc = cmp(string.lower(s1), string.lower(s2))
    if rc == 0:
	# Make the lower-case version come first since the upper-case
	# version is usually a helper module for constants and such.
	rc = cmp(s2, s1)
    return rc



def main():
    if sys.argv[1:]:
	infile = sys.argv[1]
    else:
	infile = "-"
    if infile == "-":
	ifp = sys.stdin
	ofp = sys.stdout
	sys.stdout = sys.stderr
    else:
	ifp = open(infile)
	base, ext = os.path.splitext(infile)
	outfile = base + ".ind"
	ofp = open(outfile, "w")
    ofp.write("\\begin{theindex}\n\label{modindex}\n\n")
    lines = ifp.readlines()
    for i in range(len(lines)):
	if lines[i][0] == '\\':
	    lines[i] = '\1' + lines[i]
    lines = map(parse_line, lines)
    lines.sort(cmp_items)
    prev_letter = lines[0][0][0]
    if prev_letter == '\1':<commit_msg>Don't generate a \label{} here.<commit_after>#! /usr/bin/env python

"""Script to convert raw module index data to module index."""

import os
import string
import sys


def parse_line(input):
    lineno = string.split(input)[-1]
    module = string.strip(input[:len(input)-(len(lineno)+1)])
    return module, lineno


def cmp_items((s1, line1), (s2, line2)):
    rc = cmp(string.lower(s1), string.lower(s2))
    if rc == 0:
	# Make the lower-case version come first since the upper-case
	# version is usually a helper module for constants and such.
	rc = cmp(s2, s1)
    return rc



def main():
    if sys.argv[1:]:
	infile = sys.argv[1]
    else:
	infile = "-"
    if infile == "-":
	ifp = sys.stdin
	ofp = sys.stdout
	sys.stdout = sys.stderr
    else:
	ifp = open(infile)
	base, ext = os.path.splitext(infile)
	outfile = base + ".ind"
	ofp = open(outfile, "w")
    ofp.write("\\begin{theindex}\n\n")
    lines = ifp.readlines()
    for i in range(len(lines)):
	if lines[i][0] == '\\':
	    lines[i] = '\1' + lines[i]
    lines = map(parse_line, lines)
    lines.sort(cmp_items)
    prev_letter = lines[0][0][0]
    if prev_letter == '\1':
{'fraction_non_alphanumeric': 0.16491688538932633, 'fraction_numerical': 0.016622922134733157, 'mean_word_length': 3.1809872029250457, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7619646', 'n_tokens_mistral': 886, 'n_tokens_neox': 866, 'n_words': 227}