<jupyter_start><jupyter_text><NAME>. Master Digital Text Analysis. Student ID: 20163274 MetadataThis script contains all code used for adding metadata to data frames.<jupyter_code>import pandas as pd<jupyter_output><empty_output><jupyter_text>Add wordcounts to French dataWhile English textdata already have information on word counts from EEBO and EMMA metadata, I do not have this for the French data. Therefore, this needs to be counted. For French texts coming from Frantext as well as Google Books, all textdata was parsed into a data frame of four columns (Word, Lemma, POS, filename), containing one row per word. Therefore, it makes sense to group that data frame per filename and count the number of rows that each file contains (as one word = one row). However, it needs to be filtered, since punctuation needs to be excluded.<jupyter_code>def read(path):
    return pd.read_csv(path)
wlp_frantext_early = read("/Users/paulineclaes/Documents/dta/Thesis/Data/Dataframes/WLP/frantext_WLP_early.csv")
wlp_epub_early = read("/Users/paulineclaes/Documents/dta/Thesis/Data/Dataframes/WLP/epub_WLP_early.csv")
wlp_frantext_later = read("/Users/paulineclaes/Documents/dta/Thesis/Data/Dataframes/WLP/frantext_WLP_later.csv")
wlp_epub_later = read("/Users/paulineclaes/Documents/dta/Thesis/Data/Dataframes/WLP/epub_WLP_later.csv")
def get_separate_df_per_filename(df):
    """Function to get a separate data frame per file name. 
    Takes as input the WLP data frame, prints the number of words excluding punctuation and spaces."""
    
    for filename in df["file_name"].unique(): # for each unique filename in data frame
        new_df = df[df["file_name"] == filename] # construct a new df of only that file name
        new_df = new_df.drop(new_df.index[new_df['POS'].isin(["PUNCT", "PONCT", "SPACE"])], axis=0) # drop punctuation and spaces using their POS-tags
        print(f"filename: {filename} \tTokens: {len(new_df)}\n") # print the filename and the number of rows in that data frame ( so the number of words )
def get_dict_per_filename(df):
    """Function to get a separate data frame per file name. 
    Takes as input the WLP data frame, adds the file name and its number of words excluding punctuation and spaces to a dictionary.
    Key = filename, value = number of words.
    """
    
    author_dict = {}
    for filename in df["file_name"].unique():
        new_df = df[df["file_name"] == filename]
        new_df = new_df.drop(new_df.index[new_df['POS'].isin(["PUNCT", "PONCT", "SPACE"])], axis=0) # drop punctuation and spaces using their POS-tags
        author_dict[filename] = len(new_df)
    return author_dict
def merge_dicts(dict1, dict2):
    dict1.update(dict2)
    return dict1
## EXAMPLE 
# building the dictionary to contain the wordcounts

author_dict_frantext = get_dict_per_filename(wlp_frantext_early) # wordcounts for frantext WLP
author_dict_epub = get_dict_per_filename(wlp_epub_early) # wordcounts for EPUB WLP
author_dict = merge_dicts(author_dict_frantext, author_dict_epub) # add them to one dictionary
### EXAMPLE
# read in the data frame that we want to map the word counts to (per file name)
df = pd.read_csv("/Users/paulineclaes/Documents/dta/Thesis/Data/Dataframes/concordance/all_early_concordance.csv")
# insert a column containing the wordcounts based on the file name column
df.insert(5, "all_tokens", df["filename"].map(author_dict))<jupyter_output><empty_output><jupyter_text>Merge metadata with concordance dataframe  1. Assigning a unique ID to each author<jupyter_code>df = pd.read_excel("/Users/paulineclaes/Documents/dta/thesis/ClaesPauline_thesis_finaleversie/data/final_metadata.xlsx")
# assign unique number to authors, starting from 1 
df.insert(3, "author_id", df.groupby(["author"], sort=False).ngroup()+1) 
author_id_list = [] # instantiate empty list
for author_id, author_df in df.groupby(["author_id"]): # groupby author and iterate
    author_df = author_df.reset_index() # reset the index to the author
    author_df.insert(5, "text_id_per_author", author_df.index+1) # number of texts per author (count restarts at 1 for each new author)
    author_id_list.append(author_df) # add to list
    
new_df = pd.concat(author_id_list).reset_index(drop=True) # get it into one dataframe

# merge authorIDs with textIDs, so that each text effectively has a unique ID 
new_df.insert(6, 
              "authorId_textId", 
              [f"{row['author_id']}_{row['text_id_per_author']}" for index, row in new_df.iloc[:, 4:6].iterrows()])



# insert a new column that indicates whether a text is a translation, reference text or source text
new_df.insert(7, "transl_ref_srcTxt", 
              ["transl" if "T" in value else "srcTxt" if "FS" in value else "ref" for value in new_df["data_identifier"]])<jupyter_output><empty_output><jupyter_text>2. Mapping metadata to concordance dataframe (inserting unique identifiers per text, and other information)<jupyter_code>#function to map metadata dictionary with key=filename to the dataframe

def map_filename_dict_to_df(source_df, target_df, target_index, target_colname, col1, col2):
    
    """
    Arguments: source_df, target_df, target_index, target_colname, col1, col2
    - source_df: metadata df
    - target_df : df you want to insert metadata 
    - target_index: index you want new column to be
    - target_colname: column name you want new column to have
    - col1: column name of column you want the metadata to be based on (so a column that is shared across dataframes)
    - col2: the column containing information you want to transfer across dataframes.
    
    Actual function: 
    def map_filename_dict_to_df(source_df, target_df, target_index, target_colname, col1, col2):
        filename_dict = {filename:value for filename, value in zip(source_df[col1], source_df[col2])}
        target_df.insert(target_index, target_colname, target_df[col1].map(filename_dict))
    
        return target_df
    
    """
    
    filename_dict = {filename:value for filename, value in zip(source_df[col1], source_df[col2])}
    target_df.insert(target_index, target_colname, target_df[col1].map(filename_dict))
    
    return target_df
# EXAMPLE of doing it for 1 column


# add unique author id based on the unique data identifier
f = map_filename_dict_to_df(source_df = m, 
                            target_df = f,
                            target_index=3,
                            target_colname="author_id",
                            col1 = "data_identifier", 
                            col2 = "author_id"
)
# EXAMPLE of doing it in bulk at once


col_list = ["period", 
            "data_identifier", 
            "author_id", 
            "text_id_per_author", 
            "authorId_textId", 
            "title", 
            "USTC_subject_classification", 
            "author", 
            "textDate", "wordcount"]
print(len(col_list))
index_list = [i for i in range(0, len(col_list))]
print(index_list, len(index_list))

for col_name, col_ix in zip(col_list, index_list):
    addData = map_filename_dict_to_df(
        source_df = m, 
        target_df = addData, 
        target_index = col_ix, 
        target_colname = col_name, 
        col1 = "filename", 
        col2 = col_name
    )<jupyter_output>10
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9] 10<jupyter_text>3. Add numeric period categoryFor the English data, we have a classification into decades: 1580-1589, 1590-1599, ...We now want to turn this into a numeric variable. There are 5 decades in total (1580-1589, 1590-1599, 1600-1609, 1680-1689, 1690-1699). These will be assigned a number chronologically.<jupyter_code>en = pd.read_excel('/Users/paulineclaes/Documents/dta/thesis/finaldata/final_GoToInf.xlsx')
fr = pd.read_excel('/Users/paulineclaes/Documents/dta/thesis/finaldata/final_AllerINF.xlsx')
en.head()
en.insert(1, "period_category", en.groupby(["period"], sort=True).ngroup()+1)<jupyter_output><empty_output><jupyter_text>> However, for French data, this is less straightforward, since these were not first classified into decades, and have a wider range of text dates, since they precede the corresponding English translation.> Therefore, the range of possible textdates for French is divided into 5 equal chunks.<jupyter_code>fr.head()
import numpy as np
conditions = [
    (fr['fr_source_textDate'] >= 1502) & (fr['fr_source_textDate'] <= 1542), # 1
    (fr['fr_source_textDate'] >= 1543) & (fr['fr_source_textDate'] <= 1583), # 2
    (fr['fr_source_textDate'] >= 1584) & (fr['fr_source_textDate'] <= 1624), # 3
    (fr['fr_source_textDate'] >= 1625) & (fr['fr_source_textDate'] <= 1665), # 4
    (fr['fr_source_textDate'] >= 1666) & (fr['fr_source_textDate'] <= 1699) # 5
]

values = ['1', '2', '3', '4', '5']

fr.insert(1, 'period_category', np.select(conditions, values))
fr.head()<jupyter_output><empty_output><jupyter_text>> Inserting the same period category in the metadata dataframe.<jupyter_code>meta = pd.read_excel('/Users/paulineclaes/Documents/dta/thesis/finaldata/final_metadata.xlsx')
# english subset (which already has a classification per decade)
meta_en_subset = meta[meta['transl_ref_srcTxt'] != 'srcTxt']
# french subset (which does not yet have a classification per decade)
meta_fr_subset = meta[meta['transl_ref_srcTxt'] == 'srcTxt']
# insert period category in english data
meta_en_subset.insert(3, "period_category", meta_en_subset.groupby(["period"], sort=True).ngroup()+1) 
# insert period category in French data
meta_fr_subset.insert(3, 'period_category', np.select(conditions, values))
# concatenating two data frames row-wise
meta_new = pd.concat([meta_en_subset, meta_fr_subset])
# write to excel file 

#meta_new.to_excel('/Users/paulineclaes/Documents/dta/thesis/finaldata/final_metadata.xlsx',
#                 index=False,
#                 na_rep='NA')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10348683541730203, 'fraction_numerical': 0.018094947646640235, 'mean_word_length': 4.230196703880915, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '203362', 'license': 'BSD-4-Clause-UC', 'n_tokens_mistral': 3306, 'n_tokens_neox': 3128, 'n_words': 911, 'path': 'ClaesPauline_scripts/ClaesPauline_metadata_script.ipynb', 'repo_name': 'plinecls/MasterThesis', 'size': '9844'}