<commit_before># coding=utf-8

from collections import OrderedDict
from six import iteritems

import parseJATS as parser
import utils

def rewrite_json(rewrite_type, soup, json_content):
    """
    Due to XML content that will not conform with the strict JSON schema validation rules,
    for elife articles only, rewrite the JSON to make it valid
    """
    if not soup:
        return json_content
    if not parser.doi(soup) or not parser.journal_id(soup):
        return json_content

    # Hook only onto elife articles for rewriting currently
    if parser.journal_id(soup).lower() == "elife":
        function_name = rewrite_function_name(parser.journal_id(soup), rewrite_type)
        if function_name:
            try:
                json_content = globals()[function_name](json_content, parser.doi(soup))
            except KeyError:
                pass
    return json_content

def rewrite_function_name(journal_id, rewrite_type):
    """ concatenate the name of the rewriting function """
    if not journal_id or not rewrite_type:
        return None
    return "rewrite_" + journal_id.lower() + "_" + rewrite_type

def rewrite_elife_references_json(json_content, doi):
    """ this does the work of rewriting elife references json """
    references_rewrite_json = elife_references_rewrite_json()
    if doi in references_rewrite_json:
        json_content = rewrite_references_json(json_content, references_rewrite_json[doi])

    # Edge case delete one reference
    if doi == "10.7554/eLife.12125":
        for i, ref in enumerate(json_content):
            if ref.get("id") and ref.get("id") == "bib11":
                del json_content[i]

    return json_content

def rewrite_references_json(json_content, rewrite_json):
    """ general purpose references json rewriting by matching the id value """
    for ref in json_content:
        if ref.get("id") and ref.get("id") in rewrite_json:
            for key, value in iteritems(rewrite_json.get(ref.get("id"))):
                ref[key] = value
    return json_content

def elife_references_rewrite_json():
    """ Here is the DOI and references json replacements data for elife """
    references_rewrite_json = {}

    references_rewrite_json["10.7554/eLife.00051"] = {"bib25": {"date": "2012"}}
    references_rewrite_json["10.7554/eLife.00278"] = {"bib11": {"date": "2013"}}
    references_rewrite_json["10.7554/eLife.00444"] = {"bib2": {"date": "2013"}}
    references_rewrite_json["10.7554/eLife.00569"] = {"bib74": {"date": "1996"}}
    references_rewrite_json["10.7554/eLife.00592"] = {"bib8": {"date": "2013"}}
    references_rewrite_json["10.7554/eLife.00633"] = {"bib38": {"date": "2004"}}
    references_rewrite_json["10.7554/eLife.00646"] = {"bib1": {"date": "2012"}}
    references_rewrite_json["10.7554/eLife.00813"] = {"bib33": {"date": "2007"}}
    references_rewrite_json["10.7554/eLife.01355"] = {"bib9": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.01530"] = {"bib12": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.01681"] = {"bib5": {"date": "2000"}}
    references_rewrite_json["10.7554/eLife.01917"] = {"bib35": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.02030"] = {"bib53": {"date": "2013"}, "bib56": {"date": "2013"}}
    references_rewrite_json["10.7554/eLife.02076"] = {"bib93a": {"date": "1990"}}
    references_rewrite_json["10.7554/eLife.02217"] = {"bib27": {"date": "2009"}}
    references_rewrite_json["10.7554/eLife.02535"] = {"bib12": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.02862"] = {"bib8": {"date": "2010"}}
    references_rewrite_json["10.7554/eLife.03711"] = {"bib35": {"date": "2012"}}
    references_rewrite_json["10.7554/eLife.03819"] = {"bib37": {"date": "2008"}}
    references_rewrite_json["10.7554/eLife.04069"] = {"bib8": {"date": "2011"}}
    references_rewrite_json["10.7554/eLife.04247"] = {"bib19a": {"date": "2015"}}
    references_rewrite_json["10.7554/eLife.04333"] = {
        "bib3": {"date": "1859"},
        "bib37": {"date": "1959"}}
    references_rewrite_json["10.7554/eLife.04478"] = {"bib49": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.04580"] = {"bib139": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.05042"] = {"bib78": {"date": "2015"}}
    references_rewrite_json["10.7554/eLife.05323"] = {"bib102": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.05423"] = {"bib102": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.05503"] = {"bib94": {"date": "2016"}}
    references_rewrite_json["10.7554/eLife.05849"] = {"bib82": {"date": "2005"}}
    references_rewrite_json["10.7554/eLife.06072"] = {"bib17": {"date": "2003"}}
    references_rewrite_json["10.7554/eLife.06315"] = {"bib19": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.06426"] = {"bib39": {"date": "2015"}}
    references_rewrite_json["10.7554/eLife.07361"] = {"bib76": {"date": "2011"}}
    references_rewrite_json["10.7554/eLife.07460"] = {
        "bib1": {"date": "2013"},
        "bib2": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.08500"] = {"bib55": {"date": "2015"}}
    references_rewrite_json["10.7554/eLife.09066"] = {"bib46": {"date": "2015"}}
    references_rewrite_json["10.7554/eLife.09100"] = {"bib50": {"date": "2011"}}
    references_rewrite_json["10.7554/eLife.09148"] = {
        "bib47": {"articleTitle": "97–104"},
        "bib59": {"articleTitle": "1913–1918"}}
    references_rewrite_json["10.7554/eLife.09186"] = {
        "bib31": {"date": "2015"},
        "bib54": {"date": "2014"},
        "bib56": {"date": "2014"},
        "bib65": {"date": "2015"}}
    references_rewrite_json["10.7554/eLife.09215"] = {"bib5": {"date": "2012"}}

    references_rewrite_json["10.7554/eLife.09520"] = {
        "bib35": {
            "conference": {
                "name": ["WHO Expert Committee on Malaria"]},
            "articleTitle": "WHO Expert Committee on Malaria [meeting held in Geneva from 19 to 30 October 1970]: fifteenth report",
            "publisher": OrderedDict([
                ("name", ["World Health Organization"]),
                ("address", OrderedDict([
                    ("formatted", ["Geneva"]),
                    ("components", OrderedDict([
                        ("locality", ["Geneva"])
                    ])),
                ])),
            ])
        }
    }

    references_rewrite_json["10.7554/eLife.09579"] = {
        "bib19": {"date": "2007"},
        "bib49": {"date": "2002"}}
    references_rewrite_json["10.7554/eLife.09600"] = {"bib13": {"date": "2009"}}
    references_rewrite_json["10.7554/eLife.09672"] = {
        "bib25": {"conference": {"name": ["Seventeenth Meeting of the RBM Partnership Monitoring and Evaluation Reference Group (MERG)"]}}}
    references_rewrite_json["10.7554/eLife.09771"] = {"bib22": {"date": "2012"}}
    references_rewrite_json["10.7554/eLife.09972"] = {"bib61": {"date": "2007", "discriminator": "a"}}
    references_rewrite_json["10.7554/eLife.09977"] = {"bib41": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.10032"] = {"bib45": {"date": "2016"}}
    references_rewrite_json["10.7554/eLife.10042"] = {"bib14": {"date": "2015"}}
    references_rewrite_json["10.7554/eLife.10070"] = {"bib15": {"date": "2015"}, "bib38": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.10222"] = {"bib30": {"date": "2015"}}
    references_rewrite_json["10.7554/eLife.10670"] = {"bib7": {"date": "2015"}, "bib8": {"date": "2015"}}
    references_rewrite_json["10.7554/eLife.10781"] = {"bib32": {"date": "2003"}}<commit_msg>More OrderedDict formatting on the JSON rewrite edge case.<commit_after># coding=utf-8

from collections import OrderedDict
from six import iteritems

import parseJATS as parser
import utils

def rewrite_json(rewrite_type, soup, json_content):
    """
    Due to XML content that will not conform with the strict JSON schema validation rules,
    for elife articles only, rewrite the JSON to make it valid
    """
    if not soup:
        return json_content
    if not parser.doi(soup) or not parser.journal_id(soup):
        return json_content

    # Hook only onto elife articles for rewriting currently
    if parser.journal_id(soup).lower() == "elife":
        function_name = rewrite_function_name(parser.journal_id(soup), rewrite_type)
        if function_name:
            try:
                json_content = globals()[function_name](json_content, parser.doi(soup))
            except KeyError:
                pass
    return json_content

def rewrite_function_name(journal_id, rewrite_type):
    """ concatenate the name of the rewriting function """
    if not journal_id or not rewrite_type:
        return None
    return "rewrite_" + journal_id.lower() + "_" + rewrite_type

def rewrite_elife_references_json(json_content, doi):
    """ this does the work of rewriting elife references json """
    references_rewrite_json = elife_references_rewrite_json()
    if doi in references_rewrite_json:
        json_content = rewrite_references_json(json_content, references_rewrite_json[doi])

    # Edge case delete one reference
    if doi == "10.7554/eLife.12125":
        for i, ref in enumerate(json_content):
            if ref.get("id") and ref.get("id") == "bib11":
                del json_content[i]

    return json_content

def rewrite_references_json(json_content, rewrite_json):
    """ general purpose references json rewriting by matching the id value """
    for ref in json_content:
        if ref.get("id") and ref.get("id") in rewrite_json:
            for key, value in iteritems(rewrite_json.get(ref.get("id"))):
                ref[key] = value
    return json_content

def elife_references_rewrite_json():
    """ Here is the DOI and references json replacements data for elife """
    references_rewrite_json = {}

    references_rewrite_json["10.7554/eLife.00051"] = {"bib25": {"date": "2012"}}
    references_rewrite_json["10.7554/eLife.00278"] = {"bib11": {"date": "2013"}}
    references_rewrite_json["10.7554/eLife.00444"] = {"bib2": {"date": "2013"}}
    references_rewrite_json["10.7554/eLife.00569"] = {"bib74": {"date": "1996"}}
    references_rewrite_json["10.7554/eLife.00592"] = {"bib8": {"date": "2013"}}
    references_rewrite_json["10.7554/eLife.00633"] = {"bib38": {"date": "2004"}}
    references_rewrite_json["10.7554/eLife.00646"] = {"bib1": {"date": "2012"}}
    references_rewrite_json["10.7554/eLife.00813"] = {"bib33": {"date": "2007"}}
    references_rewrite_json["10.7554/eLife.01355"] = {"bib9": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.01530"] = {"bib12": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.01681"] = {"bib5": {"date": "2000"}}
    references_rewrite_json["10.7554/eLife.01917"] = {"bib35": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.02030"] = {"bib53": {"date": "2013"}, "bib56": {"date": "2013"}}
    references_rewrite_json["10.7554/eLife.02076"] = {"bib93a": {"date": "1990"}}
    references_rewrite_json["10.7554/eLife.02217"] = {"bib27": {"date": "2009"}}
    references_rewrite_json["10.7554/eLife.02535"] = {"bib12": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.02862"] = {"bib8": {"date": "2010"}}
    references_rewrite_json["10.7554/eLife.03711"] = {"bib35": {"date": "2012"}}
    references_rewrite_json["10.7554/eLife.03819"] = {"bib37": {"date": "2008"}}
    references_rewrite_json["10.7554/eLife.04069"] = {"bib8": {"date": "2011"}}
    references_rewrite_json["10.7554/eLife.04247"] = {"bib19a": {"date": "2015"}}
    references_rewrite_json["10.7554/eLife.04333"] = {
        "bib3": {"date": "1859"},
        "bib37": {"date": "1959"}}
    references_rewrite_json["10.7554/eLife.04478"] = {"bib49": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.04580"] = {"bib139": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.05042"] = {"bib78": {"date": "2015"}}
    references_rewrite_json["10.7554/eLife.05323"] = {"bib102": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.05423"] = {"bib102": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.05503"] = {"bib94": {"date": "2016"}}
    references_rewrite_json["10.7554/eLife.05849"] = {"bib82": {"date": "2005"}}
    references_rewrite_json["10.7554/eLife.06072"] = {"bib17": {"date": "2003"}}
    references_rewrite_json["10.7554/eLife.06315"] = {"bib19": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.06426"] = {"bib39": {"date": "2015"}}
    references_rewrite_json["10.7554/eLife.07361"] = {"bib76": {"date": "2011"}}
    references_rewrite_json["10.7554/eLife.07460"] = {
        "bib1": {"date": "2013"},
        "bib2": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.08500"] = {"bib55": {"date": "2015"}}
    references_rewrite_json["10.7554/eLife.09066"] = {"bib46": {"date": "2015"}}
    references_rewrite_json["10.7554/eLife.09100"] = {"bib50": {"date": "2011"}}
    references_rewrite_json["10.7554/eLife.09148"] = {
        "bib47": {"articleTitle": "97–104"},
        "bib59": {"articleTitle": "1913–1918"}}
    references_rewrite_json["10.7554/eLife.09186"] = {
        "bib31": {"date": "2015"},
        "bib54": {"date": "2014"},
        "bib56": {"date": "2014"},
        "bib65": {"date": "2015"}}
    references_rewrite_json["10.7554/eLife.09215"] = {"bib5": {"date": "2012"}}

    references_rewrite_json["10.7554/eLife.09520"] = {
        "bib35": OrderedDict([
            ("conference", OrderedDict([
                ("name", ["WHO Expert Committee on Malaria"])
            ])),
            ("articleTitle", "WHO Expert Committee on Malaria [meeting held in Geneva from 19 to 30 October 1970]: fifteenth report"),
            ("publisher", OrderedDict([
                ("name", ["World Health Organization"]),
                ("address", OrderedDict([
                    ("formatted", ["Geneva"]),
                    ("components", OrderedDict([
                        ("locality", ["Geneva"])
                    ])),
                ])),
            ])),
        ])
    }

    references_rewrite_json["10.7554/eLife.09579"] = {
        "bib19": {"date": "2007"},
        "bib49": {"date": "2002"}}
    references_rewrite_json["10.7554/eLife.09600"] = {"bib13": {"date": "2009"}}
    references_rewrite_json["10.7554/eLife.09672"] = {
        "bib25": {"conference": {"name": ["Seventeenth Meeting of the RBM Partnership Monitoring and Evaluation Reference Group (MERG)"]}}}
    references_rewrite_json["10.7554/eLife.09771"] = {"bib22": {"date": "2012"}}
    references_rewrite_json["10.7554/eLife.09972"] = {"bib61": {"date": "2007", "discriminator": "a"}}
    references_rewrite_json["10.7554/eLife.09977"] = {"bib41": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.10032"] = {"bib45": {"date": "2016"}}
    references_rewrite_json["10.7554/eLife.10042"] = {"bib14": {"date": "2015"}}
    references_rewrite_json["10.7554/eLife.10070"] = {"bib15": {"date": "2015"}, "bib38": {"date": "2014"}}
    references_rewrite_json["10.7554/eLife.10222"] = {"bib30": {"date": "2015"}}
    references_rewrite_json["10.7554/eLife.10670"] = {"bib7": {"date": "2015"}, "bib8": {"date": "2015"}}
    references_rewrite_json["10.7554/eLife.10781"] = {"bib32": {"date": "2003"}}
{'fraction_non_alphanumeric': 0.18933595027805036, 'fraction_numerical': 0.12692181877657835, 'mean_word_length': 3.835811452072129, 'pattern_counts': {'":': 264, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7118903', 'n_tokens_mistral': 6615, 'n_tokens_neox': 5136, 'n_words': 985}