<commit_before># utilities.py
#
# some utilities like text processing

import string
import itertools
import re

def list_append(xs) :
    #return itertools.chain.from_iterable(xs)
    return [a for x in xs for a in x]

def join_with_spaces(xs) :
    return " ".join(xs)
def join(xs) :
    return "".join(xs)

def serial_comma(nouns, conj="and", comma=",", force_comma=False) :
    conj = " " + conj + " "
    if len(nouns) == 0 :
        return "nothing"
    elif len(nouns) == 1 :
        return nouns[0]
    elif len(nouns) == 2 :
        if force_comma :
            return nouns[0] + comma + conj + nouns[1]
        else :
            return nouns[0] + conj + nouns[1]
    else :
        comma_sp = comma + " "
        return comma_sp.join(nouns[:-1]) + comma + conj + nouns[-1]

def is_are_list(nouns) :
    if len(nouns) == 0 :
        return "is nothing"
    elif len(nouns) == 1 :
        return "is "+nouns[0]
    else :
        return "are "+serial_comma(nouns)

DIRECTION_INVERSES = {"north" : "south",
                      "south" : "north",
                      "east" : "west",
                      "west" : "east",
                      "northwest" : "southeast",
                      "northeast" : "southwest",
                      "southwest" : "northeast",
                      "southeast" : "northwest",
                      "up" : "down",
                      "down" : "up",
                      "in" : "out",
                      "out" : "in"}

def inverse_direction(direction) :
    return DIRECTION_INVERSES[direction]

def docstring(s) :
    def _docstring(f) :
        f.__doc__ = s
        return f
    return _docstring


html_escape_table = {
    "&": "&amp;",
    '"': "&quot;",
    "'": "&apos;",
    ">": "&gt;",
    "<": "&lt;",
    }

def html_escape(text):
    """Produce entities within text."""
    if text :
        return "".join(html_escape_table.get(c,c) for c in text)
    else :
        return None


###
### Web interface stuff<commit_msg>added hack hook for inverse directions<commit_after># utilities.py
#
# some utilities like text processing

import string
import itertools
import re

def list_append(xs) :
    #return itertools.chain.from_iterable(xs)
    return [a for x in xs for a in x]

def join_with_spaces(xs) :
    return " ".join(xs)
def join(xs) :
    return "".join(xs)

def serial_comma(nouns, conj="and", comma=",", force_comma=False) :
    conj = " " + conj + " "
    if len(nouns) == 0 :
        return "nothing"
    elif len(nouns) == 1 :
        return nouns[0]
    elif len(nouns) == 2 :
        if force_comma :
            return nouns[0] + comma + conj + nouns[1]
        else :
            return nouns[0] + conj + nouns[1]
    else :
        comma_sp = comma + " "
        return comma_sp.join(nouns[:-1]) + comma + conj + nouns[-1]

def is_are_list(nouns) :
    if len(nouns) == 0 :
        return "is nothing"
    elif len(nouns) == 1 :
        return "is "+nouns[0]
    else :
        return "are "+serial_comma(nouns)

DIRECTION_INVERSES = {"north" : "south",
                      "south" : "north",
                      "east" : "west",
                      "west" : "east",
                      "northwest" : "southeast",
                      "northeast" : "southwest",
                      "southwest" : "northeast",
                      "southeast" : "northwest",
                      "up" : "down",
                      "down" : "up",
                      "in" : "out",
                      "out" : "in"}

def inverse_direction(direction) :
    return DIRECTION_INVERSES[direction]

def add_direction_pair(dir, opp) :
    DIRECTION_INVERSES[dir] = opp
    DIRECTION_INVERSES[opp] = dir

def docstring(s) :
    def _docstring(f) :
        f.__doc__ = s
        return f
    return _docstring


html_escape_table = {
    "&": "&amp;",
    '"': "&quot;",
    "'": "&apos;",
    ">": "&gt;",
    "<": "&lt;",
    }

def html_escape(text):
    """Produce entities within text."""
    if text :
        return "".join(html_escape_table.get(c,c) for c in text)
    else :
        return None


###
### Web interface stuff
{'fraction_non_alphanumeric': 0.14532199270959903, 'fraction_numerical': 0.006318347509113001, 'mean_word_length': 1.8172484599589322, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2208032', 'n_tokens_mistral': 1480, 'n_tokens_neox': 1395, 'n_words': 327}