<jupyter_start><jupyter_text>each input-output-program in single row<jupyter_code>def apply_func(x, func):
    d = {"x": x}
    exec(func, d)
    return d["x"]

def apply_func_on_group(group, func):
    group_2 = group.copy()
    group_2["input"] = group["input"].apply(apply_func, args=(func,))
    group_2["program"] += "\n" + func
    return group_2

def apply_meta_func_on_group(group, meta_func):
    func_list = meta_func(group)
#     print(meta_func.__name__, len(func_list))
    updated_group = pd.concat([apply_func_on_group(group, func) for func in func_list])
    return updated_group

def apply_meta_func(input_output_data_frame, meta_func):
    updated_input_output_data_frame = input_output_data_frame.groupby("program").apply(apply_meta_func_on_group, meta_func).reset_index(drop=True)
    return updated_input_output_data_frame
def split_string(group):
    input_string_list = group["input"].values.tolist()
    list_of_set_of_separators = [utils.get_all_substrings(string) for string in input_string_list]
    set_of_separators = utils.get_intersection_of_list_of_sets(list_of_set_of_separators)
    candidate_functions_list = list(set(["x = x.split(\"{}\")".format(sep) for sep in set_of_separators]))
    return candidate_functions_list
def get_select_combinations(group):
    min_length = group["input"].apply(len).min()
    list_of_list = [list(itertools.combinations(range(min_length), k)) for k in range(1, min_length + 1)]
    reverse_list_of_list = [list(itertools.combinations(np.linspace(-min_length, -1, min_length).astype(int), k)) for k in range(1, min_length + 1)]
    select_list = list(itertools.chain.from_iterable(list_of_list + reverse_list_of_list))
    func_str_list = []
    for selected in select_list:
        if len(selected) <= 3 and np.ptp(selected) < 3:
            if len(selected) > -1:
                func = "x = [x[i] for i in {}]".format(selected)
            else:
                func = "x = x[{}]".format(selected[0])
            func_str_list.append(func)
    return func_str_list
def get_permutations(group):
    min_length = group["input"].apply(len).min()
    permutations_list = list(itertools.permutations(range(min_length)))
    func_str_list = ["x = [x[i] for i in {}]".format(permutation) for permutation in permutations_list]
    return func_str_list
def join_list_to_string(group):
    input_string_list = group["output"].values.tolist()
    list_of_set_of_separators = [utils.get_all_substrings(string) for string in input_string_list]
    set_of_separators = utils.get_intersection_of_list_of_sets(list_of_set_of_separators)
    candidate_functions_list = ["x = \"{}\".join(x)".format(sep) for sep in list(set_of_separators) + [""] if len(sep) <= 2]
    return candidate_functions_list
def is_program_correct_on_all_examples(data_frame):
    return (data_frame["input"] == data_frame["output"]).all()

def get_correct_programs(data_frame):
    correctness = pd.DataFrame(data_frame.groupby("program").apply(is_program_correct_on_all_examples)).reset_index()
    correct_programs = correctness[correctness[0]]["program"]
    return correct_programs

def code(input_output_data_frame, is_log=False):
    input_output_data_frame["program"] = ""
    if is_log:
        meta_func_list = [split_string, get_select_combinations, join_list_to_string]
    else:
        meta_func_list = [split_string, get_select_combinations, get_permutations, join_list_to_string]
    for meta_func in meta_func_list:
        input_output_data_frame = apply_meta_func(input_output_data_frame, meta_func)
        
    correct_programs = get_correct_programs(input_output_data_frame)
    return correct_programs
input_output_examples_list = [("aa bb cc", "bbmmccmmaa"), ("ll dd ff", "ddmmffmmll")]
input_output_data_frame = pd.DataFrame(input_output_examples_list, columns=["input", "output"])
input_output_data_frame
correct_series = code(input_output_data_frame)
correct_series.apply(print)
correct_series<jupyter_output>x = x.split(" ")
x = [x[i] for i in (-3, -2, -1)]
x = [x[i] for i in (1, 2, 0)]
x = "mm".join(x)

x = x.split(" ")
x = [x[i] for i in (0, 1, 2)]
x = [x[i] for i in (1, 2, 0)]
x = "mm".join(x)<jupyter_text>logs<jupyter_code>def get_function(input_output_data_frame):
    prefix_list = []
    suffix_list = []
    for index, row in input_output_data_frame.iterrows():
        sub_string = row["output"]
        line = row["input"]
        start_index = line.find(sub_string)
        end_index = start_index + len(sub_string)
        prefix = line[:start_index][-15:]
        suffix = line[end_index:][:15]
        prefix_list.append(prefix)
        suffix_list.append(suffix)
    common_prefix = utils.get_max_length_common_string_of_list(prefix_list)
    common_suffix = utils.get_max_length_common_string_of_list(suffix_list)
    func_str = "x = x.split(\"{}\")[1].split(\"{}\")[0]".format(common_prefix, common_suffix)
    return func_str

def apply_func_on_log_lines(log_lines, func_str):
    log_lines = [line.replace("\n", "") for line in log_lines]
    d = {"log_lines": log_lines}
    exec(func_str, d)
    return d["output_list"]
def get_info_from_log(log_lines, output_strings_list):
    log_lines = [line.replace("\n", "") for line in log_lines]
    interesting_lines = []
    interesting_input_output_list = []
    for line in log_lines:
        for output_string in output_strings_list:
            if output_string in line:
                interesting_lines.append(line)
                interesting_input_output_list.append((line, output_string))

    common_string = utils.get_max_length_common_string_of_list(interesting_lines)

    common_string_no_numeric_trail = \
        common_string[:(-pd.Series(list(common_string)).str.isnumeric().values[::-1].argmin())]

    input_output_data_frame = pd.DataFrame(interesting_input_output_list, columns=["input", "output"])

    func_str = get_function(input_output_data_frame)

    program_str = """
output_list = []
for line in log_lines:
    if \"{}\" in line:
        x = line
        {}
        output_list.append(x)
""".format(common_string_no_numeric_trail, func_str)

    output_list = apply_func_on_log_lines(log_lines, program_str)

    return output_list, program_str
file_path = r".\log_example.txt"
log_lines = open(file_path, "r").readlines()
log_lines
output_strings_list = ["port 40009", "port 48849"]

output_list, program_str = get_info_from_log(log_lines, output_strings_list)

display(output_list)
print(program_str)
utils.create_script(program_str)
%run -i .\machine_induced_script .\log_example.txt .\log_output_example.txt<jupyter_output>done writing .\log_output_example.txt
{'fraction_non_alphanumeric': 0.12205438066465257, 'fraction_numerical': 0.006646525679758308, 'mean_word_length': 4.742411101474414, 'pattern_counts': {'":': 2, '<': 10, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '552903', 'license': 'Apache-2.0', 'n_tokens_mistral': 2340, 'n_tokens_neox': 2284, 'n_words': 397, 'path': 'PBE.ipynb', 'repo_name': 'erez-aharonov/auto-coder', 'size': '6620'}