<commit_before>import json
import logging

if __name__ == "__main__":
    input = sys.argv[1]
    output = sys.argv[2]
    processor = fix_changeline_budget_titles().process(input,output,[])

class fix_support_budget_titles(object):

    def process(self,inputs,output):

        out = []

        budgets = {}
        budgets2 = {}
        supports = {}

        supports_jsons, budget_jsons = inputs

        for line in file(budget_jsons):
            line = json.loads(line.strip())
            budgets["%(year)s/%(code)s" % line] = line['title']
            if line.get('title','') != '':
                budgets2.setdefault("%(year)s/%(title)s" % line,[]).append(line['code'])

        for line in file(supports_jsons):
            line = json.loads(line.strip())
            if line.get('title','') != '':
                supports.setdefault("%(year)s/%(title)s" % line,set()).add(line['code'])

        errors = {}

        outfile = file(output,"w")
        changed_num = 0
        for line in file(supports_jsons):
            datum = json.loads(line.strip())
            year = datum['year']
            key_code = "%s/%s" % (year, datum['code'])
            title = budgets.get(key_code)
            if title != None:
                if title != datum.get('title',''):
                    datum['title'] = title
                    changed_num += 1
            else:
                key_title = "%s/%s" % (year, datum['title'])
                possible_codes = budgets2.get(key_title,[])
                if len(possible_codes) == 1:
                    datum['code'] = possible_codes[0]
                    changed_num += 1
                else:
                    all_codes_for_title = list(supports.get(key_title,[]))
                    all_valid_codes = [ x for x in all_codes_for_title if "%s/%s" % (year,x) in budgets ]
                    if len(all_valid_codes) == 1:
                        code = all_valid_codes[0]
                        datum['code'] = code
                        datum['title'] = budgets[ "%s/%s" % (year,code) ]
                        changed_num += 1
                    else:
                        errors[key_code] = (key_code, possible_codes, all_codes_for_title, all_valid_codes)
            outfile.write(json.dumps(datum,sort_keys=True)+"\n")

        for error in errors.values():
            logging.error("Failed to find title for support with key %s: pv=%r, ac=%r, vc=%r" % error)

        logging.info("updated %d entries" % changed_num)
<commit_msg>better fixing of support titles<commit_after>import json
import logging

if __name__ == "__main__":
    input = sys.argv[1]
    output = sys.argv[2]
    processor = fix_changeline_budget_titles().process(input,output,[])

class fix_support_budget_titles(object):

    def process(self,inputs,output):

        out = []

        budgets = {}
        budgets2 = {}
        supports = {}
        supports2 = {}

        supports_jsons, budget_jsons = inputs

        for line in file(budget_jsons):
            line = json.loads(line.strip())
            budgets["%(year)s/%(code)s" % line] = line['title']
            # if line.get('title','') != '':
            #     budgets2.setdefault("%(year)s/%(title)s" % line,[]).append(line['code'])

        for line in file(supports_jsons):
            line = json.loads(line.strip())
            if line.get('title','') != '':
                if budgets.get("%(year)s/%(code)s" % year) is not None:
                    supports.setdefault("%(year)s/%(title)s" % line,set()).add(line['code'])
            if line.get('subject','') != '':
                if budgets.get("%(year)s/%(code)s" % year) is not None:
                    supports2.setdefault("%(year)s/%(subject)s" % line,set()).add(line['code'])

        errors = {}

        outfile = file(output,"w")
        changed_num = 0
        for line in file(supports_jsons):
            datum = json.loads(line.strip())
            year = datum['year']
            key_code = "%s/%s" % (year, datum['code'])
            title = budgets.get(key_code)
            if title is not None:
                # Code is valid, just fix title
                if title != datum.get('title',''):
                    datum['title'] = title
                    changed_num += 1
            else:
                key_title = "%s/%s" % (year, datum['title'])
                possible_codes = supports.get(key_title,[])
                if len(possible_codes) == 1:
                    # There's only one valid budget code for this support title
                    datum['code'] = possible_codes[0]
                    key_code = "%s/%s" % (year, datum['code'])
                    title = budgets.get(key_code)
                    if title is not None:
                        datum['title'] = title
                    changed_num += 1
                else:
                    key_subject = "%s/%s" % (year, datum['subject'])
                    possible_codes = supports2.get(key_subject,[])
                    if len(possible_codes) == 1:
                        # There's only one valid budget code for this support subject
                        datum['code'] = possible_codes[0]
                        key_code = "%s/%s" % (year, datum['code'])
                        title = budgets.get(key_code)
                        if title is not None:
                            datum['title'] = title
                        changed_num += 1
                    else:
                        errors.append((key_code,possible_codes))
            outfile.write(json.dumps(datum,sort_keys=True)+"\n")

        for error in errors.values():
            logging.error("Failed to find title for support with key %s: pc=%r" % error)

        logging.info("updated %d entries" % changed_num)

{'fraction_non_alphanumeric': 0.14053305642090688, 'fraction_numerical': 0.004845967462789893, 'mean_word_length': 1.649701971572673, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3616397', 'n_tokens_mistral': 1720, 'n_tokens_neox': 1637, 'n_words': 346}