<commit_before># Copyright (c) 2017 The Regents of the University of Michigan.
# All Rights Reserved. Licensed according to the terms of the Revised
# BSD License. See LICENSE.txt for details.
import json

from .common.read_only_data_structure import ReadOnlyDataStructure

class HathiData (ReadOnlyDataStructure):

    @property
    def titles (self):
        return self.get("titles", ())

    @property
    def htids (self):
        return self.get("htids", ())

def get_counts_from_item_list (items, htid):
    a = len([x for x in items if x["htid"] == htid])
    b = len(items) - a

    return a, b

def get_oclc_counts_from_json (json_data, htid = ""):
    try:
        data = json.loads(json_data)
        return get_counts_from_item_list(data["items"], htid)

    except:
        return 0, 0

def load_json (json_data, default):
    try:
        return json.loads(json_data)

    except:
        return default

def get_None_if_empty (container):
    return container if container else None

def get_titles_from_data (data):
    result = [ ]
    for x in data.get("records", { }).values():
        result.extend(x.get("titles", ()))

    return get_None_if_empty(result)

def get_htids_from_data (data):
    result = [ ]
    for x in data.get("items", [ ]):
        if "htid" in x:
            result.append(x["htid"])

    return get_None_if_empty(result)

def get_hathi_data_from_json (json_data = ""):
    data = load_json(json_data, { })
    return HathiData(titles=get_titles_from_data(data),
                     htids=get_htids_from_data(data))<commit_msg>Extract method title_lists_in_data<commit_after># Copyright (c) 2017 The Regents of the University of Michigan.
# All Rights Reserved. Licensed according to the terms of the Revised
# BSD License. See LICENSE.txt for details.
import json

from .common.read_only_data_structure import ReadOnlyDataStructure

class HathiData (ReadOnlyDataStructure):

    @property
    def titles (self):
        return self.get("titles", ())

    @property
    def htids (self):
        return self.get("htids", ())

def get_counts_from_item_list (items, htid):
    a = len([x for x in items if x["htid"] == htid])
    b = len(items) - a

    return a, b

def get_oclc_counts_from_json (json_data, htid = ""):
    try:
        data = json.loads(json_data)
        return get_counts_from_item_list(data["items"], htid)

    except:
        return 0, 0

def load_json (json_data, default):
    try:
        return json.loads(json_data)

    except:
        return default

def get_None_if_empty (container):
    return container if container else None

def title_lists_in_data (data):
    return (x.get("titles", ()) for x in data.get("records", {}).values())

def get_titles_from_data (data):
    result = [ ]
    for title_list in title_lists_in_data(data):
        result.extend(title_list)

    return get_None_if_empty(result)

def get_htids_from_data (data):
    result = [ ]
    for x in data.get("items", [ ]):
        if "htid" in x:
            result.append(x["htid"])

    return get_None_if_empty(result)

def get_hathi_data_from_json (json_data = ""):
    data = load_json(json_data, { })
    return HathiData(titles=get_titles_from_data(data),
                     htids=get_htids_from_data(data))
{'fraction_non_alphanumeric': 0.10879558688323629, 'fraction_numerical': 0.003677597303095311, 'mean_word_length': 3.2061855670103094, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2534158', 'n_tokens_mistral': 1113, 'n_tokens_neox': 1081, 'n_words': 282}