<jupyter_start><jupyter_text>Create Embeddings<jupyter_code>#To load from pickle file
documents = []
count = 0
with open(path.format('preclean/documents2.pickle'), 'rb') as fr:
    try:
        while True:
            documents.append(pickle.load(fr))
            if count%5000 == 0:
                print(count, end=" ")
            count += 1
    except EOFError:
        pass

print(len(documents))
#print(documents[0])
#Alternative
documents = []
for ID in tqdm_notebook(IDs):
    doc = text[ID]['full']
    tokens = word_tokenize(doc)
    documents.append(tokens)
print(len(documents))<jupyter_output><empty_output><jupyter_text>We use these hyper-parameter values for WIKI (APNEWS): vector size = 300 (300), window size = 15 (15), min count = 20 (10), sub-sampling threshold = 10−5(10−5),negative sample = 5, epoch = 20 (30).<jupyter_code>documents = [TaggedDocument(doc, [i]) 
             for i, doc in tqdm_notebook(enumerate(documents))]
# define training data
for s in [100,200,300]:
    # train model
    model = Doc2Vec(documents, size=s, window=5, min_count=10, workers=4,epoch = 20)
    name = 'embeddings/doc2vec2_{}'.format(s)
    model.save(path.format(name))
    print(s)<jupyter_output>/Users/conorosully/miniconda3/lib/python3.6/site-packages/gensim/models/doc2vec.py:366: UserWarning: The parameter `size` is deprecated, will be removed in 4.0.0, use `vector_size` instead.
  warnings.warn("The parameter `size` is deprecated, will be removed in 4.0.0, use `vector_size` instead.")
/Users/conorosully/miniconda3/lib/python3.6/site-packages/smart_open/smart_open_lib.py:398: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function
  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL
{'fraction_non_alphanumeric': 0.13059701492537312, 'fraction_numerical': 0.037846481876332626, 'mean_word_length': 4.257703081232493, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '593292', 'license': 'MIT', 'n_tokens_mistral': 690, 'n_tokens_neox': 632, 'n_words': 147, 'path': 'src/Archive/create_doc2vec_embeddings.ipynb', 'repo_name': 'conorosully/legal-case-prediction', 'size': '1876'}