<jupyter_start><jupyter_text>Word2vec Alternative to LDA<jupyter_code>def create_w2v(col1,col2,max_epochs,vec_size,window,min_count,resume=False):
    now  = datetime.now()
    start_time_str = now.strftime('%H:%M:%S')
    print('started at',start_time_str)
        
    start_time = time.time()
    word_list = [[words for words in line] for line in col1]
    unpacked_word_list = [word for line in word_list for word in line]
    
    filename = 'model.sav'
    
    if resume == False:
        model = Word2Vec(col1,vector_size=vec_size,window=window,min_count=min_count,workers=-1)
        model.train(col1,total_examples=len(col1),epochs=max_epochs)
        print('word2vec model has been trained.')
        pickle.dump(model,open(filename,'wb'))
    else:
        file = open(filename,'rb')
        model = pickle.load(file)
        
    averaged_vectors = [] if resume == False else pd.read_csv('intermediate_results.csv',index_col=0)
    
    if resume == True:
        averaged_vectors = [list(averaged_vectors.loc[val,:]) for val in averaged_vectors.index]
    
    j = start_index= len(averaged_vectors)
    if j != 0:
        print(f'{round(j/len(col1),3) * 100}% complete.')
    start_checkpoint_time = time.time()
    
    progress_indicators = []
    
    
    for row in range(j,len(col1)):
        list_of_words = col1[row]
        list_of_vectors = [model.wv[word] for word in list_of_words if unpacked_word_list.count(word) >= min_count]
        avg_vector = np.mean(list_of_vectors,axis=0) if list_of_vectors else np.zeros(vec_size)
        averaged_vectors.append(avg_vector)
#        print(avg_vector)
        
        j += 1
#        print(f'{j} done')
        if j % 1000 == 0:
            intermediate_results = pd.DataFrame(averaged_vectors)
            intermediate_results.to_csv('intermediate_results.csv')
            
        percent_complete = round(j/len(col1) * 100)
        if (percent_complete % 1 == 0) & (percent_complete not in progress_indicators):
            cum_time_raw = (time.time() - start_time)/60
            cum_time = round((time.time() - start_time)/60,1)
            print('{}% complete. Time elapsed: {} min'.format(percent_complete,cum_time))
            progress_indicators.append(percent_complete)
            
            remaining_time = cum_time_raw * (len(col1) - j) / (j - start_index)
            
            est_finish_time = datetime.now() + timedelta(minutes=remaining_time)
            est_finish_time = est_finish_time.strftime('%D:%H:%M:%S')
            print(f'Estimated finish time: {est_finish_time}')
            print('-' * 15)
            
    df_of_vectors = pd.DataFrame(averaged_vectors)
        
    intermediate_table = pd.concat([col2,df_of_vectors],axis=1)
        
    rows_to_remove = [ind for ind in intermediate_table.index if intermediate_table.iloc[ind,2] == 0]
        
    removed_comments = pd.DataFrame(col2[rows_to_remove])
    final_table = intermediate_table.drop(rows_to_remove,axis=0)
        
    interval = round((time.time() - start_time)/60,1)
    print('-' * 25)
    print(f'Finished. That took {interval} min.')
    now = datetime.now()
    end_time_str = now.strftime('%H:%M:%S')
    print(f'Actual finish time: {end_time_str}')
        
    return final_table, removed_comments
final_table, removed_comments = create_w2v(tab['article_words'],tab['tweet'],5,100,5,5)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12261099676565716, 'fraction_numerical': 0.017347838870920316, 'mean_word_length': 2.9742990654205608, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '458247', 'license': 'MIT', 'n_tokens_mistral': 1189, 'n_tokens_neox': 1142, 'n_words': 194, 'path': 'notebooks/0.1-maf-lda-model-training-word2vec-exploration.ipynb', 'repo_name': 'fourthbrain-students/glg-mas', 'size': '3401'}