<commit_before>#!/usr/bin/env python3
#
# RESAMPLE.py: Generates a stream of strings (roughly) uniformly sampled
# from the set of strings matching a given regex
#
# anrosent (<EMAIL>)

import sys, string
from random import randint, choice
from re import sre_parse

# Parse the regex so we can traverse the tree
def parse(s):
    return sre_parse.parse(s)

# Predefined character categories
CATEGORIES = { 'category_digit' : string.digits, 'category_word': string.ascii_letters }

# Sampler function in case we run into an unknown type tag
UNK = lambda x: ''


# Component Sampler functions

# Trival sample from literally matching strings
def sample_literal(x): return chr(x)

# Sample from on of the predefined character categories
def sample_category(c):
    return choice(CATEGORIES[c])

# Sample from a range of ASCII values 
# e.g. a-z, A-Z, 0-9
def sample_range(rg):
    s, e = rg
    return chr(randint(s, e))

# Sample from one of the regexes in the matching set
def sample_branch(bs):
    _, d = bs
    return sample(choice(d))

# Sample from the set of matching characters 
def sample_in(opts):
    return sample_single(choice(opts))


# Sample bounded repititions of given regex
#FIXME: geometric dist for big max repeat
def sample_max_repeat(node):

    # Unpack start, end of repitition interval and regex to repeat
    s, e, d = node
    return ''.join(sample(d) for _ in range(randint(s, min(e, 25))))

# NOTE: considered equivalent to sampling from greedy repeat matcher.
# May or may not be completely equivalent
def sample_min_repeat(node):
    return sample_max_repeat(node)

# Match any character
def sample_any(d):
    return choice(string.printable)

# FIXME: subpatterns only used for grouping now - no lookahead or negative matching
def sample_subpattern(p):
    t, d = p
    return sample(d)


# Mapping from regex node type tag to sampler function
SAMPLERS = {                                \
        "literal"    : sample_literal,      \
        "branch"     : sample_branch,       \
        "in"         : sample_in,           \
        "range"      : sample_range,        \
        "category"   : sample_category,     \
        "max_repeat" : sample_max_repeat ,      \
        "any"        : sample_any,          \
        "subpattern" : sample_subpattern,   \
        "min_repeat" : sample_min_repeat
}

# Recursive Sampling functions 

# Sample a single node in the regex
def sample_single(node):

    # Unpack node type and data
    t, data = node

    # Use sampler specified by node type on data
    return SAMPLERS.get(t, UNK)(data)

# Generates one sample from the uniform distribution on the set of strings matching
# the regex whose parse tree is given as argument
def sample(parsed):
    return ''.join(map(sample_single, parsed))



# A generator of roughly uniform samples from the set of strings matching the regex<commit_msg>subpattern capture and backreferences now work<commit_after>#!/usr/bin/env python3
#
# RESAMPLE.py: Generates a stream of strings (roughly) uniformly sampled
# from the set of strings matching a given regex
#
# anrosent (<EMAIL>)

import sys, string
from random import randint, choice
from re import sre_parse

# Parse the regex so we can traverse the tree
def parse(s):
    return sre_parse.parse(s)

# Predefined character categories
CATEGORIES = { 'category_digit' : string.digits, 'category_word': string.ascii_letters }

# Sampler function in case we run into an unknown type tag
UNK = lambda x: ''


# Component Sampler functions

# Trival sample from literally matching strings
def sample_literal(x): return chr(x)

# Sample from on of the predefined character categories
def sample_category(c):
    return choice(CATEGORIES[c])

# Sample from a range of ASCII values 
# e.g. a-z, A-Z, 0-9
def sample_range(rg):
    s, e = rg
    return chr(randint(s, e))

# Sample from one of the regexes in the matching set
def sample_branch(bs):
    _, d = bs
    return sample(choice(d))

# Sample from the set of matching characters 
def sample_in(opts):
    return sample_single(choice(opts))


# Sample bounded repititions of given regex
#FIXME: geometric dist for big max repeat
def sample_max_repeat(node):

    # Unpack start, end of repitition interval and regex to repeat
    s, e, d = node
    return ''.join(sample(d) for _ in range(randint(s, min(e, 25))))

# NOTE: considered equivalent to sampling from greedy repeat matcher.
# May or may not be completely equivalent
def sample_min_repeat(node):
    return sample_max_repeat(node)

# Match any character
def sample_any(d):
    return choice(string.printable)

# Sample from subpattern and cache for backreferences
# FIXME: still doesn't handle assert and assert_not subpatterns
def sample_subpattern(p):
    subpattern_id, d = p
   
    # Sample subpattern and cache
    sampled = sample(d)

    # Check if captured subpattern
    if subpattern_id:
        SUBPATTERNS[subpattern_id] = sampled

    return sampled 

# Lookup referenced group and sample from it
def sample_groupref(ref):
    return SUBPATTERNS[ref]

# Mapping from regex node type tag to sampler function
SAMPLERS = {                                \
        "literal"    : sample_literal,      \
        "branch"     : sample_branch,       \
        "in"         : sample_in,           \
        "range"      : sample_range,        \
        "category"   : sample_category,     \
        "max_repeat" : sample_max_repeat ,      \
        "any"        : sample_any,          \
        "subpattern" : sample_subpattern,   \
        "min_repeat" : sample_min_repeat,   \
        'groupref'   : sample_groupref
}

# Map to keep track of subpatterns we see
SUBPATTERNS = {}

# Recursive Sampling functions 

# Sample a single node in the regex
def sample_single(node):

    # Unpack node type and data
    t, data = node

    # Use sampler specified by node type on data
    return SAMPLERS.get(t, UNK)(data)

# Generates one sample from the uniform distribution on the set of strings matching
# the regex whose parse tree is given as argument
def sample(parsed):
    return ''.join(map(sample_single, parsed))



# A generator of roughly uniform samples from the set of strings matching the regex
{'fraction_non_alphanumeric': 0.07771853288091776, 'fraction_numerical': 0.0016157699143641946, 'mean_word_length': 3.1239173884077283, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1616807', 'n_tokens_mistral': 1873, 'n_tokens_neox': 1810, 'n_words': 687}