<commit_before>import os
from urllib import urlopen
from bs4 import BeautifulSoup

import nltk

from _exp.dataset import db

__author__ = '<NAME>'

collection = db.graph

source = 'snap'
snap_index = 'https://snap.stanford.edu/data/egonets-Facebook.html'


# print int('88,234'.replace(',', ''))
# print '{:,}'.format(1234567890)
# '1,234,567,890'

# tmp = os.path.expanduser('~/Desktop/tmp/')
tmp = 'D:/data/tmp/'
if not os.path.exists(tmp):
    os.makedirs(tmp)


raw = nltk.clean_html(urlopen(snap_index).read())
print raw
# open(tmp + "raw", 'w+').write(raw)



# soup = BeautifulSoup(urlopen(snap_index).read())

# with open("example.txt", "w") as f:
#     f.write(soup.get_text())
# print soup.get_text()<commit_msg>start to unify datasets from snap<commit_after>import os
from urllib import urlopen

import nltk

from _exp.dataset import db


__author__ = '<NAME>'

collection = db.graph

source = 'snap'

# print int('88,234'.replace(',', ''))
# print '{:,}'.format(1234567890)
# '1,234,567,890'

# tmp = os.path.expanduser('~/Desktop/tmp/')
tmp = 'D:/data/tmp/'
if not os.path.exists(tmp):
    os.makedirs(tmp)

raw = nltk.clean_html(urlopen(snap_index).read())
print raw
{'fraction_non_alphanumeric': 0.18389830508474575, 'fraction_numerical': 0.043220338983050846, 'mean_word_length': 4.733009708737864, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2929777', 'n_tokens_mistral': 499, 'n_tokens_neox': 453, 'n_words': 93}