<commit_before>import cProfile as profile
import random
import time
import pickle

from deltas import segment_matcher, sequence_matcher
from deltas.segmenters import ParagraphsSentencesAndWhitespace
from deltas.tokenizers import wikitext_split, text_split
from mw import api

segmenter = ParagraphsSentencesAndWhitespace()

session = api.Session("https://en.wikipedia.org/w/api.php")
common1 = session.revisions.get(638029546, properties={"content"})['*']
common2 = session.revisions.get(638077284, properties={"content"})['*']

common1_tokens = list(wikitext_split.tokenize(common1))
common2_tokens = list(wikitext_split.tokenize(common2))

words = [l.strip() for l in open('/usr/share/dict/words')]
random1 = ''.join(random.choice(words) if t.type == "word" else str(t)
                  for t in common1_tokens)
random2 = ''.join(random.choice(words) if t.type == "word" else str(t)
                  for t in common1_tokens)

random2_tokens = list(wikitext_split.tokenize(random2))
random1_tokens = list(wikitext_split.tokenize(random1))

print("Tokenizing (text_split):")
def tokenize_common():
    start = time.time()
    for _ in range(50):
        tokens = list(text_split.tokenize(common1))
    print("\ttext_split: {0}".format((time.time() - start)/50))

    start = time.time()
    for _ in range(50):
        tokens = list(wikitext_split.tokenize(common1))
    print("\twikitext_split: {0}".format((time.time() - start)/50))
tokenize_common()
#profile.run('segment_common()', sort="cumulative")

print("Pickling segments:")
def segments_pickle():
    segments = segmenter.segment(common1_tokens)
    pickled_segments = pickle.dumps(segments)
    start = time.time()
    for _ in range(25):
        pickled_segments = pickle.dumps(segments)
    print("\tpickling: {0}".format((time.time() - start)/25))
    for _ in range(25):
        unpickled_segments = pickle.loads(pickled_segments)
    print("\tunpickling: {0}".format((time.time() - start)/25))
segments_pickle()<commit_msg>Minor fix to speed demo script.<commit_after>import cProfile as profile
import random
import time
import pickle

from deltas import segment_matcher, sequence_matcher
from deltas.segmenters import ParagraphsSentencesAndWhitespace
from deltas.tokenizers import wikitext_split, text_split
from mw import api

segmenter = ParagraphsSentencesAndWhitespace()

session = api.Session("https://en.wikipedia.org/w/api.php")
common1 = session.revisions.get(638029546, properties={"content"})['*']
common2 = session.revisions.get(638077284, properties={"content"})['*']

common1_tokens = list(wikitext_split.tokenize(common1))
common2_tokens = list(wikitext_split.tokenize(common2))

words = [l.strip() for l in open('/usr/share/dict/words')]
random1 = ''.join(random.choice(words) if t.type == "word" else str(t)
                  for t in common1_tokens)
random2 = ''.join(random.choice(words) if t.type == "word" else str(t)
                  for t in common1_tokens)

random2_tokens = list(wikitext_split.tokenize(random2))
random1_tokens = list(wikitext_split.tokenize(random1))

print("Tokenizing:")
def tokenize_common():
    start = time.time()
    for _ in range(50):
        tokens = list(text_split.tokenize(common1))
    print("\ttext_split: {0}".format((time.time() - start)/50))

    start = time.time()
    for _ in range(50):
        tokens = list(wikitext_split.tokenize(common1))
    print("\twikitext_split: {0}".format((time.time() - start)/50))
tokenize_common()
#profile.run('segment_common()', sort="cumulative")

print("Pickling segments:")
def segments_pickle():
    segments = segmenter.segment(common1_tokens)
    pickled_segments = pickle.dumps(segments)
    start = time.time()
    for _ in range(25):
        pickled_segments = pickle.dumps(segments)
    print("\tpickling: {0}".format((time.time() - start)/25))
    for _ in range(25):
        unpickled_segments = pickle.loads(pickled_segments)
    print("\tunpickling: {0}".format((time.time() - start)/25))
segments_pickle()
{'fraction_non_alphanumeric': 0.14221218961625282, 'fraction_numerical': 0.02758966641585152, 'mean_word_length': 4.7298850574712645, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5175729', 'n_tokens_mistral': 1395, 'n_tokens_neox': 1341, 'n_words': 259}