<commit_before>####################################################################
#./mkPageRepresentations v0.1 is a script which takes lemmatised web
# pages and  outputs one distribution per sentence -- the addition 
# of all word vectors in that sentence.
# ./mkPageRepresentations is called by ./getDomainPages
####################################################################

from composes.semantic_space.space import Space
from composes.utils import io_utils
from composes.composition.weighted_additive import WeightedAdditive
from composes.similarity.cos import CosSimilarity
from numpy import *
import sys
import os
import re


path_to_PeARS = "/home/aurelie/PeARS/"
pear=""	#Initialise variable for the pear we're adding page representations to<commit_msg>Update runDistSem.py<commit_after>####################################################################
#./mkPageRepresentations v0.1 is a script which takes lemmatised web
# pages and  outputs one distribution per sentence -- the addition 
# of all word vectors in that sentence.
# ./mkPageRepresentations is called by ./getDomainPages
####################################################################

from numpy import *
import sys
import os
import re


path_to_PeARS = "/home/aurelie/PeARS/"
pear=""	#Initialise variable for the pear we're adding page representations to
{'fraction_non_alphanumeric': 0.2524127691165553, 'fraction_numerical': 0.002969561989606533, 'mean_word_length': 6.406593406593407, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '614982', 'n_tokens_mistral': 334, 'n_tokens_neox': 309, 'n_words': 122}