<commit_before># version: 2
import os
import glob
import hashlib
import sys

matches = []

exlude_dirs = set(['.git', 'docs'])

def get_subdirs(path):
    return set([name for name in os.listdir(path)
        if os.path.isdir(os.path.join(path, name))])

def find_in(path):
    # print(path)
    out = []
    out += glob.glob(path + "/*.pro")
    out += glob.glob(path + "/CMakeLists.txt")
    out += glob.glob(path + "/Info.plist")
    subs = get_subdirs(path) - exlude_dirs
    for s in subs:
        out += find_in(os.path.join(path, s))
    out.sort()
    return out
	
pros = find_in(".")
# print(pros)

hasher = hashlib.md5()
for pro in pros:
    with open(pro) as f: s = f.read()
    hasher.update(s.encode('utf8'))

current = hasher.hexdigest()
if os.path.isfile("projectfiles.md5.tmp"):
    with open("projectfiles.md5.tmp") as f: old = f.read()
else:
    old = ""

if current.strip() == old.strip():
    sys.exit(0)
else:
    with open("projectfiles.md5.tmp", "w") as f: print(current, file=f)
    sys.exit(1)<commit_msg>Update projectfiles_unchanged script<commit_after>#!/usr/bin/env python3
#
# This script is used on Linux, OS X and Windows.
# Python 3 required.
# Returns 0 if project files are unchanged and 1 else.
#
# Script version: 3
import os
import glob
import hashlib
import sys

matches = []

tmp_file = "projectfiles.md5.tmp"
exlude_dirs = set(['.git', 'docs'])

def get_subdirs(path):
    return set([name for name in os.listdir(path)
        if os.path.isdir(os.path.join(path, name))])

def find_in(path):
    # print(path)
    out = []
    out += glob.glob(path + "/*.pro")
    out += glob.glob(path + "/CMakeLists.txt")
    out += glob.glob(path + "/Info.plist")
    subs = get_subdirs(path) - exlude_dirs
    for s in subs:
        out += find_in(os.path.join(path, s))
    out.sort()
    return out
	
pros = find_in(".")
# print(pros)

hasher = hashlib.md5()
for pro in pros:
    with open(pro) as f: s = f.read()
    hasher.update(s.encode('utf8'))

current = hasher.hexdigest()
if os.path.isfile(tmp_file):
    with open(tmp_file) as f: old = f.read()
else:
    old = ""

if current.strip() == old.strip():
    sys.exit(0)
else:
    with open(tmp_file, "w") as f: print(current, file=f)
    sys.exit(1)
{'fraction_non_alphanumeric': 0.16599552572706935, 'fraction_numerical': 0.008053691275167786, 'mean_word_length': 3.1951219512195124, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '208228', 'n_tokens_mistral': 864, 'n_tokens_neox': 850, 'n_words': 214}