<jupyter_start><jupyter_text>To avoide data leakage:- Exclude the law and verdict section of any judgement used in train or test set. (3840)- Exclude the sumamry of any judgemtn used in train or test set. All the other documents are used to create word2vec and doc2vec embeddingsm  word2vec<jupyter_code># - take case
# - if ID remove law
# - 3 stopword removal
# - dump each to a pickel file
path = "/Users/conorosully/Documents/Legal-Case-Prediction/data/preclean/{}"

with open(path.format('sentence_none.pickle'), 'wb') as f:
    for ID in tqdm_notebook(doc_ids):
        doc = clean.getCase(ID)[ID]

        if ID in law_split:
            doc = splitLaw(doc)

        if ID not in exclude:
            parsed_sentences = sentence.feed(doc)
            for sent in parsed_sentences:
                tokens = word_tokenize(sent)

                tokens = [word for word in tokens if word.isalpha()]
                tokens = [w.lower() for w in tokens]
                if len(tokens) > 2:
                    pickle.dump(tokens,f)
                
    f.close()
#To load from pickle file
sentences = []
with open(path.format('sentence_none.pickle'), 'rb') as fr:
    try:
        while True:
            sentences.append(pickle.load(fr))
    except EOFError:
        pass

print(len(sentences))
#print(documents[0])
"""stopwords = stopwords.words('english')

path = "/Users/conorosully/Documents/Legal-Case-Prediction/data/preclean/{}"

with open(path.format('sentence_english.pickle'), 'wb') as f:
    tokens = [word for word in tokens if word not in stopwords]"""
count = 0
for s in tqdm_notebook(sentences):
    count += len(s)
print(count) #Number of unique tokens<jupyter_output><empty_output><jupyter_text>Create embeddings<jupyter_code>path = "/Users/conorosully/Documents/Legal-Case-Prediction/data/embeddings/{}"

for s in [100,200]:
    # train model
    model = Word2Vec(sentences, min_count=10,size=s)

    words = list(model.wv.vocab)
    #print(model['case'])
    name = 'echt2vec_{}.txt'.format(s)
    model.save(path.format(name))
    print(s)
new_model = Word2Vec.load(path.format('echt2vec_100.txt'))
print(new_model)
print(new_model['the'])<jupyter_output><empty_output><jupyter_text>doc2vec<jupyter_code>#GOOGLE BEST PRACTISES
# Create documents from all judgements with 3 versions: none, english and legal
# - For each text json
# - If ID: exclude else full
# - dump to file
#create docuemnts
with open(path.format('documents_none.pickle'), 'wb') as f:
    
    for ID in tqdm_notebook(doc_ids):
        
        doc = clean.getCase(ID)[ID]
        
        if ID in law_split:
            doc = splitLaw(doc)

        if ID not in exclude:
            parsed_sentences = sentence.feed(doc)

            document = []
            for sent in parsed_sentences:

                tokens = word_tokenize(sent)
                tokens = [word for word in tokens if word.isalpha()]
                tokens = [w.lower() for w in tokens]
                document += tokens
        
            pickle.dump(document,f)
                
f.close()
#To load from pickle file
documents = []
count = 0
with open(path.format('documents_none.pickle'), 'rb') as fr:
    try:
        while True:
            documents.append(pickle.load(fr))
            if count%5000 == 0:
                print(count, end=" ")
            count += 1
    except EOFError:
        pass

print(len(documents))
#print(documents[0])
documents = [TaggedDocument(doc, [i]) 
             for i, doc in tqdm_notebook(enumerate(documents))]<jupyter_output><empty_output><jupyter_text>We use these hyper-parameter values for WIKI (APNEWS): vector size = 300 (300), window size = 15 (15),min count = 20 (10), sub-sampling threshold = 10−5(10−5),negative sample = 5, epoch = 20 (30). After removing lowfrequency words, the vocabulary size is approximately 670Kfor WIKI and 300K for AP-NEWS.10That is, test data is held out and<jupyter_code># define training data
path = "/Users/conorosully/Documents/Legal-Case-Prediction/data/embeddings/{}"

for s in [100,200]:
    # train model
    model = Doc2Vec(documents, 
                    size=s, 
                    window=15, 
                    min_count=10, 
                    workers = 4,
                    epoch = 20,
                    negative = 5)
    name = 'doc2vec2_{}'.format(s)
    model.save(path.format(name))
    print(s)<jupyter_output>/Users/conorosully/miniconda3/lib/python3.6/site-packages/gensim/models/doc2vec.py:366: UserWarning: The parameter `size` is deprecated, will be removed in 4.0.0, use `vector_size` instead.
  warnings.warn("The parameter `size` is deprecated, will be removed in 4.0.0, use `vector_size` instead.")
{'fraction_non_alphanumeric': 0.11308251389482685, 'fraction_numerical': 0.021162890123984607, 'mean_word_length': 2.6900630914826498, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '338865', 'license': 'MIT', 'n_tokens_mistral': 1588, 'n_tokens_neox': 1467, 'n_words': 401, 'path': 'src/create_embeddings.ipynb', 'repo_name': 'conorosully/legal-case-prediction', 'size': '4678'}