<commit_before>import json
import os,os.path
import re
import sys

import xml.etree
import xml.etree.ElementTree as ET

import xml.dom.minidom

shre = re.compile("^[a-f0-9]+$")
def studio_hash(s):
    ''' Check if a string is a Studio-generated hash '''
    if len(s) == len("d750387a715f4c0e981efebe128ff754") and shre.match(s):
        return True
    return False

def url_slug(unsluggified, unique=True):
    ''' Return a sluggified string appropriate for embedding in a URL. 
    For example, "Hello, <NAME>!" will convert to "Hello_Mr._Rogers"
    The string is guaranteed to have only letters, numbers, and underscores. 

    It is not guaranteed to be unique. 
    '''
    if unique: 
        return _make_unique_url_slug(unsluggified)
    else: 
        return _url_slug_encode(unsluggified)

def save_url_name_slugs(tree):
    ''' Go through the tree. Save all of the url_names, so when we do
    unique slug encodes, we don't reuse them. 
    '''
    for e in tree.iter():
        if 'url_name' in e.attrib:
            _make_unique_url_slug(e.attrib['url_name'])

def propagate_display_to_url_name(tree):
    ''' If we have a Studio-assigned URL name, but we do have a display name,
    change the URL name to be a sluggification of the display name '''
    for e in tree.iter():
        if 'display_name' in e.attrib and studio_hash(e.attrib['url_name']):
            set_url_name_slug(e, e.attrib['display_name'])

display_map = {}
def set_url_name_slug(e, new_name):
    ''' Maintain a mapping from new URL names to old ones for e.g. analytics. 
    We'll keep the mapping in static files. 
    '''
    new_name = url_slug(new_name, unique=True)
    display_map[new_name] = e.attrib['url_name']
    e.attrib['url_name'] = new_name

def save_url_name_map(basepath):
    ''' Save a mapping from old url names to new url names.
    This way, we can resconstruct between processed/unprocessed courses between 
    runs. '''

    mapping_filename = os.path.join(basepath, 'static/urlname_mapping.json')
    i = 0
    while os.path.exists(mapping_filename):
        mapping_filename = os.path.join(basepath, 'static/urlname_mapping_{i}.json'.format(i=i))
        i = i+1

    mapping_file = open(mapping_filename, "w")
    mapping_file.write(json.dumps(display_map, indent=2))
    mapping_file.close()

def propagate_urlname_to_filename(tree, basepath):
    ''' Rename horrific Studio names for files to be the same as nice new url_names''' 
    for e in tree.iter():
        if 'filename' in e.attrib and  \
                os.path.exists(os.path.join(basepath, 'html', e.attrib['filename'])+".html") and \
                studio_hash(e.attrib['filename']):
            oldpath = os.path.join(basepath, 'html', e.attrib['filename'])+".html"
            if 'url_name' in e.attrib:
                slug = e.attrib['url_name']
            newpath = os.path.join(basepath, 'html', slug)+".html"
            if not os.path.exists(newpath):
                os.rename(oldpath, newpath)
                e.attrib['filename'] = slug

def left_sibling_node(node):
    ''' Find the node directly above a given node. 
    '''
    related_index = node.parent._children.index(node)-1
    if related_index >= 0: 
        related_node = node.parent._children[related_index]
        return related_node
    return None

def propagate_sibling_tags(tree):
    ''' If a discussion node has an automatic name, assume it is about
    the node above it, and use that as a URL name with _discussion at the end. 

    Use display names for discussion targets. 
    '''
    for e in tree.iter():
        if e.tag in ['discussion']: 
            related_node = left_sibling_node(e)
            if related_node == None:
                continue
            if studio_hash(e.attrib['url_name']) and not studio_hash(related_node.attrib['url_name']):
                set_url_name_slug(e, related_node.attrib['url_name'] + '_' + e.tag)
            if not 'discussion_target' in e.attrib or studio_hash(e.attrib['discussion_target']):
                if 'display_name' in related_node.attrib and not studio_hash(related_node.attrib['display_name']):
                    e.attrib['discussion_target'] = related_node.attrib['display_name']

def _url_slug_encode(s):
    ''' Return a sluggified string appropriate for embedding in a URL. 
    For example, "Hello, <NAME>!" will convert to "Hello_Mr._Rogers"
    The string is guaranteed to have only letters, numbers, and underscores. 

    It is not guaranteed to be unique. 
    '''

    # Step 1: Replace non-alphanumeric characters with underscores
    new_string = str()
    for i in range(len(s)):
        if s[i].isalnum():
            new_string = new_string + s[i]
        else:
            new_string = new_string + '_'

    # Step 2: Remove underscores at the end
    
    while len(new_string) and new_string[-1] == '_':
        new_string = new_string[:-1]
    if len(new_string) == 0:
        new_string = "_"

    # Step 3: Consolidate repeated underscores
    new_string = new_string.replace("_____", "_")
    new_string = new_string.replace("___", "_")
    new_string = new_string.replace("__", "_")

    return new_string

used_url_names = set()
def _make_unique_url_slug(s):<commit_msg>Fixed voodoo bug<commit_after>import json
import os,os.path
import re
import sys

import xml.etree
import xml.etree.ElementTree as ET

import xml.dom.minidom

shre = re.compile("^[a-f0-9]+$")
def studio_hash(s):
    ''' Check if a string is a Studio-generated hash '''
    if len(s) == len("d750387a715f4c0e981efebe128ff754") and shre.match(s):
        return True
    return False

def url_slug(unsluggified, unique=True):
    ''' Return a sluggified string appropriate for embedding in a URL. 
    For example, "Hello, Mr. Rogers!" will convert to "Hello_Mr._Rogers"
    The string is guaranteed to have only letters, numbers, and underscores. 

    It is not guaranteed to be unique. 
    '''
    if unique: 
        return _make_unique_url_slug(unsluggified)
    else: 
        return _url_slug_encode(unsluggified)

def save_url_name_slugs(tree):
    ''' Go through the tree. Save all of the url_names, so when we do
    unique slug encodes, we don't reuse them. 
    '''
    for e in tree.iter():
        if 'url_name' in e.attrib:
            _make_unique_url_slug(e.attrib['url_name'])

def propagate_display_to_url_name(tree):
    ''' If we have a Studio-assigned URL name, but we do have a display name,
    change the URL name to be a sluggification of the display name '''
    for e in tree.iter():
        if 'display_name' in e.attrib and studio_hash(e.attrib['url_name']):
            set_url_name_slug(e, e.attrib['display_name'])

display_map = {}
def set_url_name_slug(e, new_name):
    ''' Maintain a mapping from new URL names to old ones for e.g. analytics. 
    We'll keep the mapping in static files. 
    '''
    new_name = url_slug(new_name, unique=True)
    display_map[new_name] = e.attrib['url_name']
    e.attrib['url_name'] = new_name

def save_url_name_map(basepath):
    ''' Save a mapping from old url names to new url names.
    This way, we can resconstruct between processed/unprocessed courses between 
    runs. '''

    mapping_filename = os.path.join(basepath, 'static/urlname_mapping.json')
    i = 0
    while os.path.exists(mapping_filename):
        mapping_filename = os.path.join(basepath, 'static/urlname_mapping_{i}.json'.format(i=i))
        i = i+1

    mapping_file = open(mapping_filename, "w")
    mapping_file.write(json.dumps(display_map, indent=2))
    mapping_file.close()

def propagate_urlname_to_filename(tree, basepath):
    ''' Rename horrific Studio names for files to be the same as nice new url_names''' 
    for e in tree.iter():
        if 'filename' in e.attrib and  \
                os.path.exists(os.path.join(basepath, 'html', e.attrib['filename'])+".html") and \
                studio_hash(e.attrib['filename']):
            oldpath = os.path.join(basepath, 'html', e.attrib['filename'])+".html"
            if 'url_name' in e.attrib:
                slug = e.attrib['url_name']
            newpath = os.path.join(basepath, 'html', slug)+".html"
            if not os.path.exists(newpath):
                os.rename(oldpath, newpath)
                e.attrib['filename'] = slug

def left_sibling_node(node):
    ''' Find the node directly above a given node. 
    '''
    related_index = node.parent._children.index(node)-1
    if related_index >= 0: 
        related_node = node.parent._children[related_index]
        return related_node
    return None

def propagate_sibling_tags(tree):
    ''' If a discussion node has an automatic name, assume it is about
    the node above it, and use that as a URL name with _discussion at the end. 

    Use display names for discussion targets. 
    '''
    for e in tree.iter():
        if e.tag in ['discussion']: 
            related_node = left_sibling_node(e)
            if related_node == None:
                continue
            if studio_hash(e.attrib['url_name']) and not studio_hash(related_node.attrib['url_name']):
                set_url_name_slug(e, related_node.attrib['url_name'] + '_' + e.tag)
            if not 'discussion_target' in e.attrib or studio_hash(e.attrib['discussion_target']):
                if 'display_name' in related_node.attrib and not studio_hash(related_node.attrib['display_name']):
                    e.attrib['discussion_target'] = related_node.attrib['display_name']

def _url_slug_encode(s):
    ''' Return a sluggified string appropriate for embedding in a URL. 
    For example, "Hello, <NAME>!" will convert to "Hello_Mr._Rogers"
    The string is guaranteed to have only letters, numbers, and underscores. 

    It is not guaranteed to be unique. 
    '''

    # Step 1: Replace non-alphanumeric characters with underscores
    new_string = str()
    for i in range(len(s)):
        if s[i].isalnum():
            new_string = new_string + s[i]
        else:
            new_string = new_string + '_'

    # Step 2: Remove underscores at the end
    
    while len(new_string)>0 and new_string[-1] == '_':
        new_string = new_string[:-1]
    if len(new_string) == 0:
        new_string = "_"

    # Step 3: Consolidate repeated underscores
    new_string = new_string.replace("_____", "_")
    new_string = new_string.replace("___", "_")
    new_string = new_string.replace("__", "_")

    return new_string

used_url_names = set()
def _make_unique_url_slug(s):
{'fraction_non_alphanumeric': 0.10912565695174391, 'fraction_numerical': 0.006402293358815098, 'mean_word_length': 2.912523364485981, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '281683', 'n_tokens_mistral': 3418, 'n_tokens_neox': 3295, 'n_words': 934}