<commit_before>import os
import ast

OLD_DICT_FILENAME = 'old_dict.txt'

def _old_dict_filename(persist_dir):
    return os.path.join(persist_dir, OLD_DICT_FILENAME)

def _write_old_dict(input, persist_dir):
    with open(_old_dict_filename(persist_dir), 'w+') as f:
        f.write(str(input))

def _load_old_dict(persist_dir):
    try:
        return ast.literal_eval(open(_old_dict_filename(persist_dir)).read())
    except IOError:
        return {}

def _dict_compare(d1, d2):
    """
    We care if one of two things happens:

      * d2 has added a new key
      * a (value for the same key) in d2 has a different value than d1

    We don't care if this stuff happens:

      * A key is deleted from the dict

    Should return a list of keys that either have been added or have a different value than they used to

    """
    keys_added = set(d2.keys()) - set(d1.keys())
    keys_changed = [k for k in d1.keys() if k in d2.keys() and d1[k] != d2[k]]
    return list(keys_added) + keys_changed

def changed_keys(new_dict, persist_dir):<commit_msg>expanduser on hash_dir<commit_after>import os
import ast

OLD_DICT_FILENAME = 'old_dict.txt'

def _old_dict_filename(persist_dir):
    return os.path.join(os.path.expanduser(persist_dir), OLD_DICT_FILENAME)

def _write_old_dict(input, persist_dir):
    with open(_old_dict_filename(persist_dir), 'w+') as f:
        f.write(str(input))

def _load_old_dict(persist_dir):
    try:
        return ast.literal_eval(open(_old_dict_filename(persist_dir)).read())
    except IOError:
        return {}

def _dict_compare(d1, d2):
    """
    We care if one of two things happens:

      * d2 has added a new key
      * a (value for the same key) in d2 has a different value than d1

    We don't care if this stuff happens:

      * A key is deleted from the dict

    Should return a list of keys that either have been added or have a different value than they used to

    """
    keys_added = set(d2.keys()) - set(d1.keys())
    keys_changed = [k for k in d1.keys() if k in d2.keys() and d1[k] != d2[k]]
    return list(keys_added) + keys_changed

def changed_keys(new_dict, persist_dir):
{'fraction_non_alphanumeric': 0.09715086408220458, 'fraction_numerical': 0.010275572162540868, 'mean_word_length': 3.453222453222453, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7351827', 'n_tokens_mistral': 753, 'n_tokens_neox': 732, 'n_words': 231}