<jupyter_start><jupyter_text>```
sender_name                    <NAME>
timestamp_ms                   1613776978603
content                     Good. Don't fit.
type                                 Generic
is_unsent                              False
users                                    NaN
source_convo         kimberlyward_wskykd6lkg
datetime          2021-02-20 10:22:58.603000
message_length                            16
platform                            Facebook
photos                                   NaN
reactions                                NaN
call_duration                            NaN
sticker                                  NaN
videos                                   NaN
share                                    NaN
ip                                       NaN
gifs                                     NaN
files                                    NaN
audio_files                              NaN
missed                                   NaN
clean_content               Good. Don't fit.```<jupyter_code>messages = []


def process_segments(segs):
    text = []
    for segment in segs:
        if segment["type"] in ["TEXT", "LINE_BREAK", "LINK"]:
            text.append(segment["text"])
        else:
            print(segs)
    return " ".join(text)


def make_payload_for_standard_chat(event, participants):
    segs = event["chat_message"]["message_content"]["segment"]
    return {
        "sender_name": participants.get(event["sender_id"]["gaia_id"], "unknown"),
        "timestamp_ms": event["timestamp"],
        "content": process_segments(segs),
        # "sender_ID": event["sender_id"]["gaia_id"],
        "platform": "Hangouts",
        "type": "Generic",
        "is_unsent": False,
    }


def make_payload_for_attachment(event, participants):
    url = event["chat_message"]["message_content"]["attachment"][0].get("url", "")
    name = participants.get(event["sender_id"]["gaia_id"], "unknown")
    return {
        "sender_name": name,
        "timestamp_ms": event["timestamp"],
        "share": {"link": url},
        "content": url,
        # "sender_ID": event["sender_id"]["gaia_id"],
        "type": "Share",
        "is_unsent": False,
        "platform": "Hangouts",
    }


def make_payload_for_hangout_event(event, participants):

    return {
        "content": np.nan,
        # "sender_ID": event["sender_id"]["gaia_id"],
        "timestamp_ms": event["timestamp"],
        "sender_name": name,
        "platform": "Hangouts",
        "type": "Call",
        "event_id": "event_id",
        "is_unsent": False,
    }


for conversation in the_json["conversations"]:

    ev = conversation["events"]
    conv = conversation["conversation"]

    participants = {}
    for p in conv["conversation"]["participant_data"]:
        name = p.get("fallback_name", "unknown")
        pid = p["id"]["gaia_id"]
        participants[pid] = name
    participants

    for event in ev:
        try:
            if event.get("chat_message") and event["chat_message"][
                "message_content"
            ].get("segment"):
                payload = make_payload_for_standard_chat(event, participants)
                messages.append(payload)
            elif event.get("chat_message") and event["chat_message"][
                "message_content"
            ].get("attachment"):
                payload = make_payload_for_attachment(event, participants)
                messages.append(payload)
            elif event.get("hangout_event"):
                payload = make_payload_for_hangout_event(event, participants)
                messages.append(payload)
        except Exception as error:
            # print("error:\n", error, event, "\n")
            pass

df = pd.DataFrame(messages)
df.sample(10)

df.sender_name.value_counts()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11389813155874072, 'fraction_numerical': 0.009982083439979524, 'mean_word_length': 1.5846560846560847, 'pattern_counts': {'":': 24, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '161135', 'license': 'MIT', 'n_tokens_mistral': 1109, 'n_tokens_neox': 1025, 'n_words': 200, 'path': 'hangouts_loader.ipynb', 'repo_name': 'notionparallax/data_project_2021', 'size': '3914'}