<commit_before>from opencivicdata.models import (Jurisdiction, Organization, Person,
                                  Bill, VoteEvent, Event)
from .helpers import PublicListEndpoint, PublicDetailEndpoint, get_field_list
import copy
import pytz


def dout(obj):
    """
    Helper function to ensure that we're always printing internal
    datetime objects in a fully qualified UTC timezone. This will
    let 'None' values pass through untouched.
    """
    if obj is None:
        return
    return pytz.UTC.localize(obj).isoformat()


def sfilter(obj, blacklist):
    """
    Helper function to deep copy a dict, and pop elements off.
    """
    ret = copy.deepcopy(obj)
    for el in blacklist:
        ret.pop(el)
    return ret


#
#
#
# OK. This is where we define *what* we allow to be public.
#
#
#

DIVISION_SERIALIZE = dict([
    ("id", {}),
    ("display_name", {}),
])

SOURCES_SERIALIZE = {"note": {}, "url": {},}

JURISDICTION_SERIALIZE = dict([
    ("id", {}),
    ("name", {}),
    ("url", {}),
    ("classification", {}),
    ("extras", lambda x: x.extras),
    ("feature_flags", lambda x: x.feature_flags),
    ("division", DIVISION_SERIALIZE),
])

LEGISLATIVE_SESSION_SERIALIZE = dict([
    ('identifier', {}),
    ('classification', {}),
    ('jurisdiction', JURISDICTION_SERIALIZE),
])


CONTACT_DETAIL_SERIALIZE = dict([
    ("type", {}),
    ("value", {}),
    ("note", {}),
    ("label", {}),
])


LINK_SERALIZE = dict([
    ("note", {}),
    ("url", {}),
])

IDENTIFIERS_SERIALIZE = {
    "identifier": {},
    "scheme": {},
}

OTHER_NAMES_SERIALIZE = {
    "name": {},
    "note": {},
    "start_date": {},
    "end_date": {},
}

ORGANIZATION_SERIALIZE = dict([
    ("id", {}),
    ("name", {}),
    ("image", {}),
    ("identifiers", IDENTIFIERS_SERIALIZE),
    ("links", LINK_SERALIZE),
    ("contact_details", CONTACT_DETAIL_SERIALIZE),
    ("other_names", OTHER_NAMES_SERIALIZE),
    ("classification", {}),
    ("founding_date", {}),
    ("dissolution_date", {}),
    ("jurisdiction", JURISDICTION_SERIALIZE),
    ("sources", SOURCES_SERIALIZE),
])

ORGANIZATION_SERIALIZE['parent'] = ORGANIZATION_SERIALIZE
ORGANIZATION_SERIALIZE['children'] = sfilter(
    ORGANIZATION_SERIALIZE,
    blacklist=['parent']
)

ORGANIZATION_SERIALIZE['identifiers'] = {
    # Don't leak 'id'
    "identifier": {},
    # Don't allow recuse into our own org.
    "scheme": {},
}

PERSON_SERIALIZE = dict([
    ("id", {}),
    ("name", {}),
    ("sort_name", {}),
    ("image", {}),
    ("gender", {}),
    ("summary", {}),
    ("national_identity", {}),
    ("biography", {}),
    ("birth_date", {}),
    ("death_date", {}),

    ("identifiers", IDENTIFIERS_SERIALIZE),
    ("other_names", OTHER_NAMES_SERIALIZE),
    ("contact_details", CONTACT_DETAIL_SERIALIZE),
    ("links", LINK_SERALIZE),
    ("sources", SOURCES_SERIALIZE),
])

POST_SERIALIZE = dict([
    ("id", {}),
    ("label", {}),
    ("role", {}),
    ("start_date", {}),
    ("end_date", {}),
    ("links", LINK_SERALIZE),
    ("contact_details", CONTACT_DETAIL_SERIALIZE),
    ("organization", ORGANIZATION_SERIALIZE),
    ("division", DIVISION_SERIALIZE),
])

ORGANIZATION_SERIALIZE['posts'] = sfilter(
    POST_SERIALIZE,
    blacklist=["organization"]
)

MEMBERSHIP_SERIALIZE = {
    # Explicit to avoid letting `id' leak out.
    "start_date": {},
    "end_date": {},
    "role": {},
    "label": {},
    "organization": ORGANIZATION_SERIALIZE,
    "person": PERSON_SERIALIZE,
    "post": POST_SERIALIZE,
    "on_behalf_of": ORGANIZATION_SERIALIZE,
}

ORGANIZATION_SERIALIZE['memberships'] = sfilter(
    # Avoid org->memberships->org loop
    MEMBERSHIP_SERIALIZE,
    blacklist=['organization']
)
ORGANIZATION_SERIALIZE['memberships']['post'] = sfilter(
    # Avoid org->memberships->post->org loop
    POST_SERIALIZE,
    blacklist=['organization']
)

PERSON_SERIALIZE['memberships'] = sfilter(
    # Avoid person->memberships->person loop
    MEMBERSHIP_SERIALIZE,
    blacklist=['person']
)

POST_SERIALIZE['memberships'] = sfilter(
    MEMBERSHIP_SERIALIZE,
    blacklist=['post']
)

LINK_BASE = dict([
    ('links', {
        'media_type': {},
        'url': {},
    }),
])

BILL_SERIALIZE = dict([
    ('id', {}),
    ('identifier', {}),
    ('legislative_session', LEGISLATIVE_SESSION_SERIALIZE),
    ('legislative_session_id', {}),

    ('title', {}),
    ("extras", lambda x: x.extras),

    ('from_organization', ORGANIZATION_SERIALIZE),
    ('from_organization_id', {}),

    ('created_at', lambda x: dout(x.created_at)),
    ('updated_at', lambda x: dout(x.updated_at)),

    ('classification', lambda x: x.classification),
    ('subject', lambda x: x.classification),

    ('abstracts', {
        "abstract": {},
        "note": {},
    }),

    ('other_titles', {
        "title": {},
        "note": {},
    }),

    ('other_identifiers', {<commit_msg>Clean up links.<commit_after>from opencivicdata.models import (Jurisdiction, Organization, Person,
                                  Bill, VoteEvent, Event)
from .helpers import PublicListEndpoint, PublicDetailEndpoint, get_field_list
import copy
import pytz


def dout(obj):
    """
    Helper function to ensure that we're always printing internal
    datetime objects in a fully qualified UTC timezone. This will
    let 'None' values pass through untouched.
    """
    if obj is None:
        return
    return pytz.UTC.localize(obj).isoformat()


def sfilter(obj, blacklist):
    """
    Helper function to deep copy a dict, and pop elements off.
    """
    ret = copy.deepcopy(obj)
    for el in blacklist:
        ret.pop(el)
    return ret


#
#
#
# OK. This is where we define *what* we allow to be public.
#
#
#

DIVISION_SERIALIZE = dict([
    ("id", {}),
    ("display_name", {}),
])

SOURCES_SERIALIZE = {"note": {}, "url": {},}

JURISDICTION_SERIALIZE = dict([
    ("id", {}),
    ("name", {}),
    ("url", {}),
    ("classification", {}),
    ("extras", lambda x: x.extras),
    ("feature_flags", lambda x: x.feature_flags),
    ("division", DIVISION_SERIALIZE),
])

LEGISLATIVE_SESSION_SERIALIZE = dict([
    ('identifier', {}),
    ('classification', {}),
    ('jurisdiction', JURISDICTION_SERIALIZE),
])


CONTACT_DETAIL_SERIALIZE = dict([
    ("type", {}),
    ("value", {}),
    ("note", {}),
    ("label", {}),
])


LINK_SERALIZE = dict([
    ("note", {}),
    ("url", {}),
])

IDENTIFIERS_SERIALIZE = {
    "identifier": {},
    "scheme": {},
}

OTHER_NAMES_SERIALIZE = {
    "name": {},
    "note": {},
    "start_date": {},
    "end_date": {},
}

ORGANIZATION_SERIALIZE = dict([
    ("id", {}),
    ("name", {}),
    ("image", {}),
    ("identifiers", IDENTIFIERS_SERIALIZE),
    ("links", LINK_SERALIZE),
    ("contact_details", CONTACT_DETAIL_SERIALIZE),
    ("other_names", OTHER_NAMES_SERIALIZE),
    ("classification", {}),
    ("founding_date", {}),
    ("dissolution_date", {}),
    ("jurisdiction", JURISDICTION_SERIALIZE),
    ("sources", SOURCES_SERIALIZE),
])

ORGANIZATION_SERIALIZE['parent'] = ORGANIZATION_SERIALIZE
ORGANIZATION_SERIALIZE['children'] = sfilter(
    ORGANIZATION_SERIALIZE,
    blacklist=['parent']
)

ORGANIZATION_SERIALIZE['identifiers'] = {
    # Don't leak 'id'
    "identifier": {},
    # Don't allow recuse into our own org.
    "scheme": {},
}

PERSON_SERIALIZE = dict([
    ("id", {}),
    ("name", {}),
    ("sort_name", {}),
    ("image", {}),
    ("gender", {}),
    ("summary", {}),
    ("national_identity", {}),
    ("biography", {}),
    ("birth_date", {}),
    ("death_date", {}),

    ("identifiers", IDENTIFIERS_SERIALIZE),
    ("other_names", OTHER_NAMES_SERIALIZE),
    ("contact_details", CONTACT_DETAIL_SERIALIZE),
    ("links", LINK_SERALIZE),
    ("sources", SOURCES_SERIALIZE),
])

POST_SERIALIZE = dict([
    ("id", {}),
    ("label", {}),
    ("role", {}),
    ("start_date", {}),
    ("end_date", {}),
    ("links", LINK_SERALIZE),
    ("contact_details", CONTACT_DETAIL_SERIALIZE),
    ("organization", ORGANIZATION_SERIALIZE),
    ("division", DIVISION_SERIALIZE),
])

ORGANIZATION_SERIALIZE['posts'] = sfilter(
    POST_SERIALIZE,
    blacklist=["organization"]
)

MEMBERSHIP_SERIALIZE = {
    # Explicit to avoid letting `id' leak out.
    "start_date": {},
    "end_date": {},
    "role": {},
    "label": {},
    "organization": ORGANIZATION_SERIALIZE,
    "person": PERSON_SERIALIZE,
    "post": POST_SERIALIZE,
    "on_behalf_of": ORGANIZATION_SERIALIZE,
}

ORGANIZATION_SERIALIZE['memberships'] = sfilter(
    # Avoid org->memberships->org loop
    MEMBERSHIP_SERIALIZE,
    blacklist=['organization']
)
ORGANIZATION_SERIALIZE['memberships']['post'] = sfilter(
    # Avoid org->memberships->post->org loop
    POST_SERIALIZE,
    blacklist=['organization']
)

PERSON_SERIALIZE['memberships'] = sfilter(
    # Avoid person->memberships->person loop
    MEMBERSHIP_SERIALIZE,
    blacklist=['person']
)

POST_SERIALIZE['memberships'] = sfilter(
    MEMBERSHIP_SERIALIZE,
    blacklist=['post']
)

LINK_BASE = {
    'media_type': {},
    'url': {},
}

BILL_SERIALIZE = dict([
    ('id', {}),
    ('identifier', {}),
    ('legislative_session', LEGISLATIVE_SESSION_SERIALIZE),
    ('legislative_session_id', {}),

    ('title', {}),
    ("extras", lambda x: x.extras),

    ('from_organization', ORGANIZATION_SERIALIZE),
    ('from_organization_id', {}),

    ('created_at', lambda x: dout(x.created_at)),
    ('updated_at', lambda x: dout(x.updated_at)),

    ('classification', lambda x: x.classification),
    ('subject', lambda x: x.classification),

    ('abstracts', {
        "abstract": {},
        "note": {},
    }),

    ('other_titles', {
        "title": {},
        "note": {},
    }),

    ('other_identifiers', {
{'fraction_non_alphanumeric': 0.18686659772492245, 'fraction_numerical': 0.0, 'mean_word_length': 3.53398968588842, 'pattern_counts': {'":': 44, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '567082', 'n_tokens_mistral': 3621, 'n_tokens_neox': 3358, 'n_words': 606}