<commit_before>#!/usr/bin/env python

import requests
import time
import sys
from pymongo.mongo_client import MongoClient


MONGO_DB = 'ws_test'
SHOCK_HOST = 'http://localhost:7044'

def main():
    token = sys.argv[1]
    mcli = MongoClient()
    db = mcli[MONGO_DB]
    ws = -1
    md5s = []
    md5start = time.clock()
    for ver in db.workspaceObjVersions.find().sort([('ws', 1), ('id', 1)]):
        if ws == -1:
            ws = ver['ws']
        else:
            if ws != ver['ws']:
                raise ValueError('more than one workspace found')
        md5s.append(ver['chksum'])
    md5end = time.clock()
    nodes = []
    for md5 in md5s:
        rec = db.shock_nodeMap.find_one({'chksum': md5})
        nodes.append(rec['node'])
    nodesend = time.clock()
    headers = {'Authorization': 'OAuth ' + token}
    times = []
    count = 1
    for node in nodes:
        now = time.clock()
        ret = requests.get(SHOCK_HOST + '/node/' + node + '/?download', headers=headers)
        ret.text
        times.append(time.clock() - now)
        if ret.status_code != 200:
            raise ValueError('Non 200 return code')<commit_msg>check for token in py timing script<commit_after>#!/usr/bin/env python

import requests
import time
import sys
from pymongo.mongo_client import MongoClient


MONGO_DB = 'ws_test'
SHOCK_HOST = 'http://localhost:7044'

def main():
    token = sys.argv[1]
    if not token:
        raise ValueError('no token')
    mcli = MongoClient()
    db = mcli[MONGO_DB]
    ws = -1
    md5s = []
    md5start = time.clock()
    for ver in db.workspaceObjVersions.find().sort([('ws', 1), ('id', 1)]):
        if ws == -1:
            ws = ver['ws']
        else:
            if ws != ver['ws']:
                raise ValueError('more than one workspace found')
        md5s.append(ver['chksum'])
    md5end = time.clock()
    nodes = []
    for md5 in md5s:
        rec = db.shock_nodeMap.find_one({'chksum': md5})
        nodes.append(rec['node'])
    nodesend = time.clock()
    headers = {'Authorization': 'OAuth ' + token}
    times = []
    count = 1
    for node in nodes:
        now = time.clock()
        ret = requests.get(SHOCK_HOST + '/node/' + node + '/?download', headers=headers)
        ret.text
        times.append(time.clock() - now)
        if ret.status_code != 200:
            raise ValueError('Non 200 return code')
{'fraction_non_alphanumeric': 0.14538558786346398, 'fraction_numerical': 0.01938474504846186, 'mean_word_length': 2.2835408022130013, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3229335', 'n_tokens_mistral': 823, 'n_tokens_neox': 783, 'n_words': 201}