<jupyter_start><jupyter_text><NAME>. Master Digital Text Analysis. Student ID: 20163274 Calculating normalized frequencies on text level In this file, normalized frequencies of the constructions themselves as well as of certain context parameters will be calculated on **text level**. How will I go about this?In order to calculate normalized frequency of a certain observation, the raw number of observations is divided by the word count of the subcorpus in which this observation is made, and multiplied by 1,000,000. So, if we want to obtain the normalized frequency of a certain observation (e.g. _be going to_-construction), we consider the text in which it  occurs to be the 'subcorpus'. That means, that the number of observations is divided by the word count of a certain text and multiplied by 1,000,000. For example, we want to count how many times the word 'thesis' occurs in a text in our corpus, taking into account that each row in the dataframe equals 1 observation- group by file name- count number of rows for that file name and insert it in a column- divide the number of rows (so the number of observations) by the word count of that text, and multiply by 1,000,000<jupyter_code>import pandas as pd
import numpy as np
temp_en = pd.read_excel('/Users/paulineclaes/Documents/dta/thesis/ClaesPauline_thesis_finaleversie/ClaesPauline_data/ClaesPauline_final_GoToInf.xlsx')
temp_en.head()
# check the unique values
temp_en.cxn.unique()
# remove noise
en = temp_en.drop(temp_en[(temp_en.cxn == 'noise') | (temp_en.cxn == '[BE about Ving]+[GO to V]')].index)
# extracting progressive and nonProgressive

en_progr = en.drop(en[(en.aspect != 'progressive')].index)
en_nonprogr = en.drop(en[(en.aspect != 'nonProgressive')].index)
en.cxn.unique()
en_progr.cxn.unique()
en_nonprogr.cxn.unique()
# constructing a dictionary with unique authorId_textIds and their number of concordances (both go-constructions)
rows_dict_both = {key:len(value) for key, value in en.groupby('authorId_textId').groups.items()}

# constructing a dictionary with unique authorId_textIds and their number of PROGRESSIVE concordances
rows_dict_progr = {key:len(value) for key, value in en_progr.groupby('authorId_textId').groups.items()}

# constructing a dictionary with unique authorId_textIds and their number of NONPROGRESSIVE concordances
rows_dict_nonprogr = {key:len(value) for key, value in en_nonprogr.groupby('authorId_textId').groups.items()}
en.info()
# selecting only interesting columns
temp_new_df = en[['author', 'author_id', 'text_id_per_author', 'authorId_textId', 'textDate','timeframe', 'kind', 'USTC_subject_classification','period', 'period_category', 'wordcount']]
# keeping only unique authorId_textIds
new_df_en = temp_new_df.drop_duplicates(subset=['authorId_textId'])
len(new_df_en) == len(new_df_en.authorId_textId.unique())
new_df_en.info()
# inserting column with raw number of instances per authorId_textId of BOTH constructions
new_df_en.insert(11, 'both', new_df_en['authorId_textId'].map(rows_dict_both))
# inserting column with raw number of instances per authorId_textId of PROGRESSIVE construction
new_df_en.insert(12, 'progr', new_df_en['authorId_textId'].map(rows_dict_progr))
# inserting column with raw number of instances per authorId_textId of NONPROGRESSIVE construction
new_df_en.insert(13, 'nonprogr', new_df_en['authorId_textId'].map(rows_dict_nonprogr))
new_df_en = new_df_en.fillna(0)
def calculator(df, condition, grouping_object='authorId_textId'):
    new_df = df.drop(df[condition].index)
    rows_dict = {key:len(value) for key, value in new_df.groupby(grouping_object).groups.items()}
    return rows_dict
col_dict = {}
# motion
both_motion_yes = calculator(en, (en.motion != 'yes'))
col_dict['both_motion_yes'] = both_motion_yes

both_motion_no = calculator(en, (en.motion != 'no'))
col_dict['both_motion_no'] = both_motion_no

progr_motion_yes = calculator(en_progr, (en_progr.motion != 'yes'))
col_dict['progr_motion_yes'] = progr_motion_yes 

progr_motion_no = calculator(en_progr, (en_progr.motion != 'no'))
col_dict['progr_motion_no'] = progr_motion_no 

nonprogr_motion_yes = calculator(en_nonprogr, (en_nonprogr.motion != 'yes'))
col_dict['nonprogr_motion_yes'] = nonprogr_motion_yes 

nonprogr_motion_no = calculator(en_nonprogr, (en_nonprogr.motion != 'no'))
col_dict['nonprogr_motion_no'] = nonprogr_motion_no 

# voice
both_voice_active = calculator(en, (en.voice != 'active'))
col_dict['both_voice_active'] = both_voice_active

both_voice_passive = calculator(en, (en.voice != 'passive'))
col_dict['both_voice_passive'] = both_voice_passive

progr_voice_active = calculator(en_progr, (en_progr.voice != 'active'))
col_dict['progr_voice_active'] = progr_voice_active

progr_voice_passive = calculator(en_progr, (en_progr.voice != 'passive'))
col_dict['progr_voice_passive'] = progr_voice_passive

nonprogr_voice_active = calculator(en_nonprogr, (en_nonprogr.voice != 'active'))
col_dict['nonprogr_voice_active'] = nonprogr_voice_active

nonprogr_voice_passive = calculator(en_nonprogr, (en_nonprogr.voice != 'passive'))
col_dict['nonprogr_voice_passive'] = nonprogr_voice_passive


# animacy
both_anim_animate = calculator(en, (en.anim_binary != 'animate'))
col_dict['both_anim_animate'] = both_anim_animate

both_anim_inanimate = calculator(en, (en.anim_binary != 'inanimate'))
col_dict['both_anim_inanimate'] = both_anim_inanimate

progr_anim_animate = calculator(en_progr, (en_progr.anim_binary != 'animate'))
col_dict['progr_anim_animate'] = progr_anim_animate

progr_anim_inanimate = calculator(en_progr, (en_progr.anim_binary != 'inanimate'))
col_dict['progr_anim_inanimate'] = progr_anim_inanimate

nonprogr_anim_animate = calculator(en_nonprogr, (en_nonprogr.anim_binary != 'animate'))
col_dict['nonprogr_anim_animate'] = nonprogr_anim_animate

nonprogr_anim_inanimate = calculator(en_nonprogr, (en_nonprogr.anim_binary != 'inanimate'))
col_dict['nonprogr_anim_inanimate'] = nonprogr_anim_inanimate

# predictiveness 
both_pred_pred = calculator(en, (en.predictiveness_binary != 'predict.x'))
col_dict['both_pred_pred'] = both_pred_pred

both_pred_notPred = calculator(en, (en.predictiveness_binary != 'not_predict'))
col_dict['both_pred_notPred'] = both_pred_notPred



progr_pred_pred = calculator(en_progr, (en_progr.predictiveness_binary != 'predict.x'))
col_dict['progr_pred_pred'] = progr_pred_pred

progr_pred_notPred = calculator(en_progr, (en_progr.predictiveness_binary != 'not_predict'))
col_dict['progr_pred_notPred'] = progr_pred_notPred


nonprogr_pred_pred = calculator(en_nonprogr, (en_nonprogr.predictiveness_binary != 'predict.x'))
col_dict['nonprogr_pred_pred'] = nonprogr_pred_pred

nonprogr_pred_notPred = calculator(en_nonprogr, (en_nonprogr.predictiveness_binary != 'not_predict'))
col_dict['nonprogr_pred_notPred'] = nonprogr_pred_notPred

# intentionality 

both_int_int = calculator(en, (en.intentionality != 'int'))
col_dict['both_int_int'] = both_int_int
both_int_nint = calculator(en, (en.intentionality != 'nint'))
col_dict['both_int_nint'] = both_int_nint

progr_int_int = calculator(en_progr, (en_progr.intentionality != 'int'))
col_dict['progr_int_int'] = progr_int_int
progr_int_nint = calculator(en_progr, (en_progr.intentionality != 'nint'))
col_dict['progr_int_nint'] = progr_int_nint

nonprogr_int_int = calculator(en_nonprogr, (en_nonprogr.intentionality != 'int'))
col_dict['nonprogr_int_int'] = nonprogr_int_int
nonprogr_int_nint = calculator(en_nonprogr, (en_nonprogr.intentionality != 'nint'))
col_dict['nonprogr_int_nint'] = nonprogr_int_nint
for dict_name, dictionary in col_dict.items():
   # print(dict_name)
    new_df_en[dict_name] = new_df_en['authorId_textId'].map(dictionary)
new_df_en.info()
def calculate_relfreq(x, y, fixedNr=1000000):
    """
    Function to calculate relative frequency.
    
    Takes two columns as arguments, as well as a fixed number to multiply by, which defaults to 1,000,000.
    
    
    Returns: relative frequency, calculated by dividing the raw number of occurrences by the total 
    wordcount and multiplied with the fixed number.
    
    """
    
    return (x/y)*fixedNr
# for each column containing raw frequencies, insert a new column with normalized frequencies of that raw number 

for column in new_df_en.iloc[:,11:44].columns:
    new_df_en['norm_{}'.format(column)] = calculate_relfreq(new_df_en[column], new_df_en['wordcount'], 1000000)
new_df_en = new_df_en.fillna(0)
# insert binary column with 0 if early and 1 if later
new_df_en.insert(6, 'timeframe_binary', [0 if value == 'early' else 1 for value in new_df_en.timeframe])
# insert binary column with 0 if reference and 1 if translation
new_df_en.insert(8, 'kind_binary', [0 if value == 'reference' else 1 for value in new_df_en.kind])
#new_df_en.to_excel('/Users/paulineclaes/Documents/dta/thesis/finaldata/en_textStats.xlsx',
#                  index=False,
#                  na_rep=0)<jupyter_output><empty_output><jupyter_text>Doing the same for French data<jupyter_code>import pandas as pd
temp_fr = pd.read_excel('/Users/paulineclaes/Documents/dta/thesis/ClaesPauline_thesis_finaleversie/data/final_AllerINF.xlsx')
temp_fr.head()
temp_fr.info()
len(temp_fr[temp_fr['anim_simplified']=='dummy'])
temp_fr.voice.unique()
# remove noise
fr = temp_fr.drop(temp_fr[(temp_fr.voice == 'noise')].index)
fr.voice.unique()
# constructing a dictionary with unique authorId_textIds and their number of concordances 
rows_dict = {key:len(value) for key, value in fr.groupby('authorId_textId').groups.items()}
# select some columns
temp_new_df_fr = fr[['fr_source_author','author_id', 'text_id_per_author', 'authorId_textId', 'fr_source_textDate','timeframe', 'kind', 'eebo_genre', 'period_category', 'fr_source_all_tokens']]

# keeping only unique authorId_textIds

new_df_fr = temp_new_df_fr.drop_duplicates(subset=['authorId_textId'])
len(new_df_fr) == len(new_df_fr.authorId_textId.unique())
new_df_fr.info()
def calculate_relfreq(x, y, fixedNr=1000000):
    """
    Function to calculate relative frequency.
    
    Takes two columns as arguments, as well as a fixed number to multiply by, which defaults to 1,000,000.
    
    
    Returns: relative frequency, calculated by dividing the raw number of occurrences by the total 
    wordcount and multiplied with the fixed number.
    
    """
    
    return (x/y)*fixedNr
# insert column with raw number of go-constructions in French texts
new_df_fr.insert(10, 'raw_n', new_df_fr['authorId_textId'].map(rows_dict))
# insert column with normalized frequency of go-constructions in French texts
new_df_fr.insert(11, 'relfreq', calculate_relfreq(new_df_fr.raw_n, new_df_fr.fr_source_all_tokens, 1000000))
new_df_fr.info()
new_df_fr.timeframe.unique()
# insert binary column with 0 if early and 1 if later
new_df_fr.insert(6, 'timeframe_binary', [0 if value=='early' else 1 for value in new_df_fr.timeframe])
#new_df_fr.to_excel('/Users/paulineclaes/Documents/dta/thesis/finaldata/fr_textStats_new.xlsx',
#                  index=False,
#                  na_rep=0)
def calculator(df, condition, grouping_object='authorId_textId'):
    '''Function to calculate raw number of observations per unique text.'''
    # drop all rows based on a negative condition
        # so if we want to have raw count in go-constructions where motion is used, we specify the condition to drop all rows where motion is NOT 'yes'.
    new_df = df.drop(df[condition].index) 
    rows_dict = {key:len(value) for key, value in new_df.groupby(grouping_object).groups.items()}
    return rows_dict
# instantiate empty dictionary to which all counts will be added 
col_dict = {}
motion_yes = calculator(fr, (fr.motion != 'yes'))
col_dict['motion_yes'] = motion_yes
motion_no = calculator(fr, (fr.motion != 'no'))
col_dict['motion_no'] = motion_no

voice_active = calculator(fr, (fr.voice != 'active'))
col_dict['voice_active'] = voice_active
voice_passive = calculator(fr, (fr.voice != 'passive'))
col_dict['voice_passive'] = voice_passive

anim_animate = calculator(fr, (fr.anim_simplified != 'animate'))
col_dict['anim_animate'] = anim_animate
anim_dummy = calculator(fr, (fr.anim_simplified != 'dummy'))
col_dict['anim_dummy'] = anim_dummy
anim_inanimate = calculator(fr, (fr.anim_simplified != 'inanimate'))
col_dict['anim_inanimate'] = anim_inanimate

pred_pred = calculator(fr, (fr.predictiveness_binary != 'predict.x'))
col_dict['pred_pred'] = pred_pred
pred_notPred = calculator(fr, (fr.predictiveness_binary != 'not_predict'))
col_dict['pred_notPred'] = pred_notPred

int_int = calculator(fr, (fr.intentionality != 'int'))
col_dict['int_int'] = int_int
int_nint = calculator(fr, (fr.intentionality != 'nint'))
col_dict['int_nint'] = int_nint
# check to see if it worked
print(list(col_dict.items())[1])<jupyter_output>('motion_no', {'35_1': 1, '36_1': 70, '37_1': 3, '38_1': 4, '39_1': 15, '39_2': 15, '40_1': 59, '42_1': 21, '43_1': 11, '44_1': 27, '45_1': 34, '46_1': 4, '47_1': 17, '48_1': 7, '49_1': 1, '50_1': 3, '51_1': 3, '53_1': 15, '54_1': 4, '71_1': 9, '71_3': 2, '72_1': 26, '73_1': 14, '74_1': 3, '75_1': 18, '76_1': 11, '76_2': 25, '77_1': 2, '78_1': 3, '79_1': 17, '80_1': 9, '81_1': 25, '82_1': 3, '82_2': 23, '84_1': 5, '85_1': 3})<jupyter_text>> Result: dictionary with values for each authorId_textId<jupyter_code>for dict_name, dictionary in col_dict.items():
   # print(dict_name)
    new_df_fr[dict_name] = new_df_fr['authorId_textId'].map(dictionary)
new_df_fr.info()
new_df_fr.rename(columns={'fr_source_all_tokens':'wordcount'}, inplace=True)
new_df_fr.info()
def calculate_relfreq(x, y, fixedNr=1000000):
    """
    Function to calculate relative frequency.
    
    Takes two columns as arguments, as well as a fixed number to multiply by, which defaults to 1,000,000.
    
    
    Returns: relative frequency, calculated by dividing the raw number of occurrences by the total 
    wordcount and multiplied with the fixed number.
    
    """
    
    return (x/y)*fixedNr
for column in new_df_fr.iloc[:,13:24].columns:
    new_df_fr['norm_{}'.format(column)] = calculate_relfreq(new_df_fr[column], new_df_fr['wordcount'], 1000000)
    
    
new_df_fr.head()
new_df_fr = new_df_fr.fillna(0)
#new_df_fr.to_excel('/Users/paulineclaes/Documents/dta/thesis/finaldata/fr_textStats.xlsx',
#                  index=False,
#                  na_rep=0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11909078231766838, 'fraction_numerical': 0.020568958304281133, 'mean_word_length': 6.262278481012658, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '614862', 'license': 'BSD-4-Clause-UC', 'n_tokens_mistral': 5310, 'n_tokens_neox': 4938, 'n_words': 1161, 'path': 'ClaesPauline_scripts/ClaesPauline_calculating_normalizedFrequencies_textLevel.ipynb', 'repo_name': 'plinecls/MasterThesis', 'size': '14349'}