<jupyter_start><jupyter_text>Make topic data at book levelHere we simply aggregate the chunk-level data, at book (title) level, and combine it with metadata that may be useful later.<jupyter_code>import pandas as pd
import numpy as np
def get_doctopics(filename, parsemeta, docs2get):
    chunks = dict()
    weights = dict()
    with open(filename, encoding = 'utf-8') as f:
        for line in f:
            fields = line.strip().split()
            chunkid = fields[1]
            docid = fields[1].split('_')[0]
            
            if docid not in docs2get:
                continue
        
            if docid not in chunks:
                chunks[docid] = []
                weights[docid] = []
                
            vector = np.array([float(x) for x in fields[2: ]])
            chunks[docid].append(vector)
            weights[docid].append(parsemeta.at[chunkid, 'tokens'])
            
    docs = dict()
   
    for docid, value in chunks.items():
        avgvector = np.average(value, axis = 0, weights = weights[docid])
        docs[docid] = avgvector
          
    return docs
parsemeta = pd.read_csv('../getEF/parsing_metadata3.tsv', sep = '\t')
liwc = pd.read_csv('../liwc/liwc_w_meta.tsv', sep = '\t', low_memory = False)
liwc.shape
parsemeta.set_index('id', inplace = True)
docs = get_doctopics('../modelselection/final/k200doctopics.txt', parsemeta, liwc.docid.tolist())
docdf = pd.DataFrame.from_dict(docs, orient = 'index', columns = ['t' + str(i) for i in range(200)])
meta = pd.read_csv('../metadata/corpus4.tsv', sep = '\t', low_memory = False)
demograph = meta.loc[meta.docid.isin(docdf.index), ['docid', 'birthyear', 'firstpub', 'hathi_author', 'hathi_title', 'us_national', 'authof3ormore']]
demograph = demograph.assign(age = demograph.firstpub - demograph.birthyear)
demograph.shape
docdf.shape
bookdata = docdf.merge(demograph, right_on = 'docid', left_index = True)
bookdata.shape
bookdata.head()
bookdata.to_csv('bookleveltopicdata.tsv', sep = '\t', index = False)
sum(bookdata.us_national & (bookdata.firstpub > 1889) & (bookdata.firstpub < 1990))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1388888888888889, 'fraction_numerical': 0.011770244821092278, 'mean_word_length': 3.0018832391713746, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '631525', 'license': 'MIT', 'n_tokens_mistral': 730, 'n_tokens_neox': 704, 'n_words': 164, 'path': 'topicdata/MakeBookLevelData.ipynb', 'repo_name': 'tedunderwood/period-cohort', 'size': '2124'}