<commit_before>import os
import os.path
import json
import hashlib
import git

PROTOCOL_VERSION = 1.1

gitRepo = git.Repo("./")

mods = {}

current_commit = gitRepo.rev_parse("HEAD")

url_base = "https://cdn.rawgit.com/Mrmaxmeier/BombSquad-Community-Mod-Manager/"
modurl = url_base + current_commit.hexsha + "/mods/"

old_data = None

for blob in gitRepo.head.object.tree.traverse():
    if blob.path.startswith("mods/") and blob.path.endswith(".py"):
        filename = blob.path[5:]
        base = filename[:-3]
        data = blob.data_stream.read()
        md5 = hashlib.md5(data).hexdigest()
        mod = {
            "changelog": [],
            "md5": md5,
            "url": modurl + filename,  # TODO: remove url field and bump version to 1.6
            "filename": filename,
            "commit_sha": current_commit.hexsha,
            "old_md5s": [],
        }
        if os.path.isfile("mods/" + base + ".json"):
            with open("mods/" + base + ".json", "r") as json_file:
                mod.update(json.load(json_file))
        if mod.get("index", True):
            mods[base] = mod
    elif blob.path == "index.json":
        old_data = json.loads(blob.data_stream.read().decode("UTF-8"))

specific_sha = set()

for commit in gitRepo.iter_commits(max_count=1000, paths="mods/"):
    for filename in commit.stats.files:
        if filename.startswith("mods/"):
            filename = filename[5:]
            if filename.endswith(".py"):
                if not filename[:-3] in mods:
                    continue
                txt = commit.message
                txt = txt.replace("\n", "")
                mod_slug = filename[:-3]
                mods[mod_slug]["changelog"].append(txt)
                if filename not in specific_sha:
                    # TODO: remove url field and bump version to 1.6
                    mods[mod_slug]["url"] = url_base + commit.hexsha + "/mods/" + filename
                    mods[mod_slug]["commit_sha"] = commit.hexsha
                    specific_sha.add(filename)
    for blob in commit.tree["mods"].blobs:
        if not blob.path.endswith(".py"):
            continue
        name = blob.path[5:-3]
        if name in mods:
            data = blob.data_stream.read()
            md5 = hashlib.md5(data).hexdigest()
            if md5 not in mods[name]["old_md5s"] and md5 != mods[name]["md5"]:
                mods[name]["old_md5s"].append(md5)

for mod in mods:
    if not mod + ".py" in specific_sha:
        print("didnt find latest commit for", mod + ", head is used")

for mod in mods.values():
    mod["changelog"] = mod["changelog"][:2]
    # TODO: if the index.json gets too big
    # mod["old_md5s"] = [md5[:10] for md5 in mod["old_md5s"]]

index_data = {"mods": mods, "version": PROTOCOL_VERSION}
with open("index.json", "w") as f:
    json.dump(index_data, f, indent=4, sort_keys=True)

if old_data:
    old_mods = old_data["mods"]
    text = ""

    def add(text, mod, spacer, *args):
        if spacer:
            text += " " * (spacer + 2)<commit_msg>limit old_md5 count to 5<commit_after>import os
import os.path
import json
import hashlib
import git

PROTOCOL_VERSION = 1.1

gitRepo = git.Repo("./")

mods = {}

current_commit = gitRepo.rev_parse("HEAD")

url_base = "https://cdn.rawgit.com/Mrmaxmeier/BombSquad-Community-Mod-Manager/"
modurl = url_base + current_commit.hexsha + "/mods/"

old_data = None

for blob in gitRepo.head.object.tree.traverse():
    if blob.path.startswith("mods/") and blob.path.endswith(".py"):
        filename = blob.path[5:]
        base = filename[:-3]
        data = blob.data_stream.read()
        md5 = hashlib.md5(data).hexdigest()
        mod = {
            "changelog": [],
            "md5": md5,
            "url": modurl + filename,  # TODO: remove url field and bump version to 1.6
            "filename": filename,
            "commit_sha": current_commit.hexsha,
            "old_md5s": [],
        }
        if os.path.isfile("mods/" + base + ".json"):
            with open("mods/" + base + ".json", "r") as json_file:
                mod.update(json.load(json_file))
        if mod.get("index", True):
            mods[base] = mod
    elif blob.path == "index.json":
        old_data = json.loads(blob.data_stream.read().decode("UTF-8"))

specific_sha = set()

for commit in gitRepo.iter_commits(max_count=1000, paths="mods/"):
    for filename in commit.stats.files:
        if filename.startswith("mods/"):
            filename = filename[5:]
            if filename.endswith(".py"):
                if not filename[:-3] in mods:
                    continue
                txt = commit.message
                txt = txt.replace("\n", "")
                mod_slug = filename[:-3]
                mods[mod_slug]["changelog"].append(txt)
                if filename not in specific_sha:
                    # TODO: remove url field and bump version to 1.6
                    mods[mod_slug]["url"] = url_base + commit.hexsha + "/mods/" + filename
                    mods[mod_slug]["commit_sha"] = commit.hexsha
                    specific_sha.add(filename)
    for blob in commit.tree["mods"].blobs:
        if not blob.path.endswith(".py"):
            continue
        name = blob.path[5:-3]
        if name in mods:
            data = blob.data_stream.read()
            md5 = hashlib.md5(data).hexdigest()
            if md5 not in mods[name]["old_md5s"] and md5 != mods[name]["md5"] and len(mods[name]['old_md5s']) < 5:
                mods[name]["old_md5s"].append(md5)

for mod in mods:
    if not mod + ".py" in specific_sha:
        print("didnt find latest commit for", mod + ", head is used")

for mod in mods.values():
    mod["changelog"] = mod["changelog"][:2]
    # TODO: if the index.json gets too big
    # mod["old_md5s"] = [md5[:10] for md5 in mod["old_md5s"]]

index_data = {"mods": mods, "version": PROTOCOL_VERSION}
with open("index.json", "w") as f:
    json.dump(index_data, f, indent=4, sort_keys=True)

if old_data:
    old_mods = old_data["mods"]
    text = ""

    def add(text, mod, spacer, *args):
        if spacer:
            text += " " * (spacer + 2)
{'fraction_non_alphanumeric': 0.14635340186000978, 'fraction_numerical': 0.013705335291238374, 'mean_word_length': 2.14520266803489, 'pattern_counts': {'":': 18, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2434073', 'n_tokens_mistral': 2060, 'n_tokens_neox': 2024, 'n_words': 466}