<commit_before>import os
import argparse
import datetime
import difflib
import pprint


def filelines(fp):
    with open(fp, 'r') as f:
        fl = [line.rstrip() for line in f.readlines()]
    return fl


def hostname(f):
    fl = filelines(f)
    hn_lines = list(ln for ln in fl if 'hostname' in ln.lower())
    return hn_lines[0].split()[1]


def compare(cases=None, candidate=None):
    if not cases:
        cases = "./cases"
    if not candidate:
        candidate = "./candidate.txt"
    res = "\n{0:=<69} [ Start ]\n".format("")
    # d = {}
    for f in os.listdir(cases):
        case = os.path.join("./cases", f)
        hn = hostname(case)
        # d[hn] = {}
        diff = list(difflib.context_diff(filelines(candidate), filelines(case)))
        missing = '\n'.join(x[2:] for x in diff if x.startswith('- '))
        additional = '\n'.join(x[2:] for x in diff if x.startswith('+ '))
        # d[hn]["-"] = missing
        # d[hn]["+"] = additional
        title = "\n[ Host: {0} ] {1:=<{2}}".format(hn, "", (79 - (len(hn) + 11)))
        minus = "\n\n-\n\n{0}".format(missing)
        plus = "\n\n+\n\n{0}".format(additional)
        res += "{0}{1}{2}\n".format(title, minus, plus)
    res += "{0:=<68} [ Finish ]\n".format("")
    # pprint.pprint(d)
    return res

def diff_to_file(diff):
    timestamp = datetime.datetime.now().isoformat().split(
            '.')[0].replace(':', '').replace('T', '-')
    fout = 'diff-{0}.txt'.format(timestamp)
    with open(fout, 'a') as output:
        output.write(diff)
    return diff

compare()
# print(diff_to_file(compare()))

<commit_msg>add context_list function<commit_after>import os
import argparse
import datetime
import difflib
from pprint import pprint


def filelines(fp):
    with open(fp, 'r') as f:
        fl = [line.rstrip() for line in f.readlines()]
    return fl


def hostname(f):
    fl = filelines(f)
    hn_lines = list(ln for ln in fl if 'hostname' in ln.lower())
    return hn_lines[0].split()[1]


def context_list(a):
    normalized_a, prev, context_a = [], [], []
    a = open(a).readlines()
    for line in a:
        if len(line) > 3:
            normalized_a.append(line.rstrip())
    for line in normalized_a:
        if line.startswith(" "):
            prev.append(line)
        else:
            context_a.append(prev)
            prev = [line]
    return context_a


pprint(context_list("./jon_candidate.conf"))

# def compare(cases=None, candidate=None):
#     if not cases:
#         cases = "./cases"
#     if not candidate:
#         candidate = "./candidate.txt"
#     res = "\n{0:=<69} [ Start ]\n".format("")
#     # d = {}
#     for f in os.listdir(cases):
#         d = difflib.Differ(linejunk=lambda x : x == "!")
#         case = os.path.join(cases, f)
#         hn = hostname(case)
#         # d[hn] = {}
#         out = d.compare(filelines(candidate), filelines(case))
#         pprint.pprint(list(out))
#         diff = list(difflib.unified_diff(filelines(candidate), filelines(case), n=0, lineterm="", fromfile=str(candidate), tofile=str(case)))
#         # for line in diff:
#         #     print(line)
#         missing = '\n'.join(x[2:] for x in diff if x.startswith('- '))
#         additional = '\n'.join(x[2:] for x in diff if x.startswith('+ '))
#         semi = '\n'.join(x[2:] for x in diff if x.startswith('! '))
#         # d[hn]["-"] = missing
#         # d[hn]["+"] = additional
#         title = "\n[ Host: {0} ] {1:=<{2}}".format(hn, "", (79 - (len(hn) + 11)))
#         minus = "\n\nminus\n\n{0}".format(missing)
#         plus = "\n\nplus\n\n{0}".format(additional)
#         var = "\n\nvar\n\n{0}".format(semi)
#         res += "{0}{1}{2}{3}\n".format(title, minus, plus, var)
#     res += "{0:=<68} [ Finish ]\n".format("")
#     # pprint.pprint(d)
#     return res
#
# def diff_to_file(diff):
#     timestamp = datetime.datetime.now().isoformat().split(
#             '.')[0].replace(':', '').replace('T', '-')
#     fout = 'diff-{0}.txt'.format(timestamp)
#     with open(fout, 'a') as output:
#         output.write(diff)
#     return diff
#
# # compare()
# # diff_to_file(compare(cases="./jon_cases", candidate="./jon_candidate.conf"))
# compare(cases="./jon_cases", candidate="./jon_candidate.conf")


{'fraction_non_alphanumeric': 0.1989615293839981, 'fraction_numerical': 0.012508850601840924, 'mean_word_length': 2.5375626043405677, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4752186', 'n_tokens_mistral': 1577, 'n_tokens_neox': 1535, 'n_words': 330}