<commit_before># Processing thread-like documents
import pandas as pd

KEY_THREAD_ID = 'thread_id'
KEY_DATETIME = 'datetime'
KEY_SENDER_ID = 'sender_id'
KEY_RECIPIENT_IDS = 'recipient_ids'


def add_recipients(df):
    """
    Add recipients information on each thread
    as another column

    Return: pandas.DataFrame
    """
    assert KEY_THREAD_ID in df.columns
    assert KEY_DATETIME in df.columns
    assert KEY_SENDER_ID in df.columns
    all_rows = []
    for k, thread in df.groupby([KEY_THREAD_ID]):
        recipients = set()
        thread = thread.sort_values(KEY_DATETIME)
        for i, r in thread.iterrows():
            if i == 0:
                if thread.shape[0] > 1:
                    rs = [thread.iloc[1][KEY_SENDER_ID]]
                else:
                    rs = []
            else:
                rs = list(recipients)
            new_row = r.tolist() + [rs]
            all_rows.append(new_row)

            recipients.add(r[KEY_SENDER_ID])

    return pd.DataFrame(all_rows,
                        columns=df.columns.tolist() + [KEY_RECIPIENT_IDS])
            

def add_recipients_to_islamic_dataset(path):
    t = pd.read_csv(path, sep='\t',
                    error_bad_lines=False)
    t.rename(columns={
        'ThreadID': KEY_THREAD_ID,
        'MemberID': KEY_SENDER_ID,
        'P_Date': KEY_DATETIME,
        'MessageID': 'message_id',
        'Message': 'body',
        'MemberName': 'sender_name',
    },
             inplace=True)<commit_msg>ThreadName to subject in islamic data<commit_after># Processing thread-like documents
import pandas as pd

KEY_THREAD_ID = 'thread_id'
KEY_DATETIME = 'datetime'
KEY_SENDER_ID = 'sender_id'
KEY_RECIPIENT_IDS = 'recipient_ids'


def add_recipients(df):
    """
    Add recipients information on each thread
    as another column

    Return: pandas.DataFrame
    """
    assert KEY_THREAD_ID in df.columns
    assert KEY_DATETIME in df.columns
    assert KEY_SENDER_ID in df.columns
    all_rows = []
    for k, thread in df.groupby([KEY_THREAD_ID]):
        recipients = set()
        thread = thread.sort_values(KEY_DATETIME)
        for i, r in thread.iterrows():
            if i == 0:
                if thread.shape[0] > 1:
                    rs = [thread.iloc[1][KEY_SENDER_ID]]
                else:
                    rs = []
            else:
                rs = list(recipients)
            new_row = r.tolist() + [rs]
            all_rows.append(new_row)

            recipients.add(r[KEY_SENDER_ID])

    return pd.DataFrame(all_rows,
                        columns=df.columns.tolist() + [KEY_RECIPIENT_IDS])
            

def add_recipients_to_islamic_dataset(path):
    t = pd.read_csv(path, sep='\t',
                    error_bad_lines=False)
    t.rename(columns={
        'ThreadID': KEY_THREAD_ID,
        'MemberID': KEY_SENDER_ID,
        'P_Date': KEY_DATETIME,
        'MessageID': 'message_id',
        'ThreadName': 'subject',
        'Message': 'body',
        'MemberName': 'sender_name',
    },
             inplace=True)
{'fraction_non_alphanumeric': 0.1043649491302921, 'fraction_numerical': 0.002625533311453889, 'mean_word_length': 2.0602409638554215, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5124936', 'n_tokens_mistral': 1018, 'n_tokens_neox': 974, 'n_words': 202}