<jupyter_start><jupyter_text>Interactive Lab:<jupyter_code>notebook = is_notebook()

if notebook == True:
    keywords_list = pd.read_csv('propertywords_cleaned_for_collocates.csv')
    kw_col_name = keywords_list.columns[0]
    keywords_list = keywords_list[kw_col_name].tolist()

    # can't use usecols here bc text is str andn year is int -- dumb 
    data = collocate_analysis.import_data('/users/sbuongiorno/rerun_csv_chunk.csv', ',', text_col='text', year_col='year', sub='/users/sbuongiorno/preprocess_propertywords.csv')
    
    data = dict_keyword_lookup(data, keywords_list)

    data = collocate_analysis.extract_grammatical_collocates(data, keywords_list) # add a cli option for return type 
    
    data = sentiment_score_df(data, 'grammatical_collocates')
    
    export_folder = 'collocates_sentiment'
    
    if not os.path.exists(export_folder):
        os.mkdir(export_folder)
    
    data.to_csv(export_folder + '/' + 'collocates_sentiment_scores.csv', index=False)<jupyter_output><empty_output><jupyter_text>sbatch:<jupyter_code>keywords_list = ['absentee', 'adscription', 'agist', 'allod', 'allotment', 'almoign', 'amortized', 'apanage', 'atar', 'attorn', 
                 'blettro', 'bond-land', 'bookland', 'bordage', 'bordar', 'boscage', 'burgage', 'burgery', 'cablicum', 
                 'cammandery', 'cartbote', 'charter-hold', 'charter-land', 'chattel', 'co-feof', 'co-tenancy', 'co-tenant', 
                 'coedcae', 'commonage', 'commonties', 'commonty', 'croft', 'curiality', 'demesn', 'depopulation', 'dispossess',
                 'domanial', 'domesday', 'dreng', 'eject', 'emin', 'enclosure','escuage', 'esplees', 'estover', 'ethel', 'evict',
                 'feu', 'ffridd', 'fiar', 'fief', 'firebote', 'fogg', 'folkland', 'frank-ferm', 'frank-marriage', 'frankalmoign',
                 'franklin', 'frith', 'gavelkind', 'gaveller', 'grasanese', 'gwely', 'haybote', 'herbage', 'holdership', 
                 'homager', 'householdership', 'hypothec', 'inclosure', 'intercommoning', 'joint-tenancy', 'joint-tenant', 
                 'knight-service', 'laen', 'laetic', 'land', 'lease', 'lessee', 'lifehold', 'liferent', 'livier', 'lotment', 
                 'mail-payer', 'majorat', 'manorialize', 'manurance', 'mese', 'mesn', 'metayage', 'metayer', 'mivvy', 'occupance',
                 'occupancy', 'outland', 'pannage', 'parage', 'patrony', 'pendicle', 'perpetual', 'piscary', 'ploughbote', 
                 'poffle', 'pollam', 'pre-emptive', 'property', 'rack-rent', 'radknight', 'radman', 'rent', 'rere-fief', 'roture',
                 'roturier', 'rundale', 'runrig', 'ryoti', 'ryotwar', 'scattald', 'seisin', 'severalty', 'socage', 'sokeman', 
                 'solidate', 'sorning', 'sple', 'squat', 'steelbow', 'sub-fief', 'subaltern', 'subfeu', 'sublessee', 'subsman', 
                 'subtenancy', 'subtenant', 'subvassal', 'suit-hold', 'swinamote', 'tanistic', 'tanistry', 'tariot', 'tenancy',
                 'tenant', 'tenement', 'tenurial', 'termon', 'termor', 'terre-tenant', 'thanage', 'thaneland', 'three-life',
                 'thring', 'turbary', 'udal', 'under-tenancy', 'under-tenant', 'underlessee', 'undertenant', 'undervassal',
                 'unfeued', 'unleased', 'unlet', 'urbarial', 'vassal', 'venville', 'vidame', 'villan', 'villar', 'villein', 
                 'woodmote', 'zemindar']
arg1 = sys.argv[1]

data = collocate_analysis.import_data(arg1, ',', text_col='text', year_col='year', sub='/users/sbuongiorno/preprocess_propertywords.csv')

data = data_contains(data, keywords_list)
    
data = collocate_analysis.extract_grammatical_collocates(data, keywords_list) # add a cli option for return type 

data = sentiment_score(data, 'grammatical_collocates')

if not data.empty:
    handle = open(arg1 + '.pickle', 'wb') 
    pickle.dump(data, handle)
else:
    exit()
!jupyter nbconvert --to script collocates_sentiment.ipynb<jupyter_output>[NbConvertApp] Converting notebook collocates_sentiment.ipynb to script
[NbConvertApp] Writing 11853 bytes to collocates_sentiment.py
{'fraction_non_alphanumeric': 0.17728055077452667, 'fraction_numerical': 0.0024588148512417014, 'mean_word_length': 4.2153846153846155, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '500800', 'license': 'MIT', 'n_tokens_mistral': 1473, 'n_tokens_neox': 1436, 'n_words': 274, 'path': 'collocates/collocates-sentiment/collocates_sentiment.ipynb', 'repo_name': 'stephbuon/democracy-lab', 'size': '4067'}