<commit_before>import os, requests, json, subprocess, time, pymongo, signal, sys
from parse import parse_files
from threading import Thread, Lock
from Queue import Queue
from datetime import timedelta, date

script_dir = os.path.dirname(os.path.realpath(__file__))

client = pymongo.MongoClient()
db = client.pyscrape

queue = Queue(100)
searchers = 1
parsers = 1
page = 1
page_lock = Lock()
done = False
done_lock = Lock()

total_count = 1000
req_count = 0
req_lock = Lock()
prev_time = 0
missed = []

date_range = ''
size_range = ''

class Searcher(Thread):
    def run(self):
        global queue
        global page
	global done
        while True:
	    with done_lock:
		    if done:
			return

            with page_lock:
                my_page = page
                page += 1

	    if total_count < (my_page+1)*100:
		with done_lock:
		    done = True
		return

            try:
                results = search(my_page)
            except:
                with done_lock:
		    done = True
                return

	    if len(results) == 0:
		time.sleep(0.2)
            for result in results:
                while queue.full():
                    time.sleep(0.01)

                queue.put(result)

class Parser(Thread):
    def run(self):
        global queue
        global data
        while True:
	    pkg = None
	    while not pkg:
		try:
		    pkg = queue.get(timeout=1)
		except:
		    with done_lock:
			if done:
			    return

            pkg = format_pkg(pkg)
	    if db.packages.find({'_id': pkg['_id']}).count() > 0:
		#print('package data already in db')
		continue

            # want to use tmpfs for this but ran out of memory
            path = '%s/clone/%s' % (script_dir, pkg['_id'])

            start = time.time()
	    try:
                subprocess.check_output(['git', 'clone', pkg['clone_url'], path], stderr=subprocess.STDOUT)
	    except Exception as e:
	        print('clone failed due to: %s' % e.output)
		subprocess.check_output(['rm', '-rf', path])<commit_msg>add thread print stmts<commit_after>import os, requests, json, subprocess, time, pymongo, signal, sys
from parse import parse_files
from threading import Thread, Lock, current_thread
from Queue import Queue
from datetime import timedelta, date

script_dir = os.path.dirname(os.path.realpath(__file__))

client = pymongo.MongoClient()
db = client.pyscrape

queue = Queue(100)
searchers = 2
parsers = 2
page = 1
page_lock = Lock()
done = False
done_lock = Lock()

total_count = 1000
req_count = 0
req_lock = Lock()
prev_time = 0
missed = []

date_range = ''
size_range = ''

class Searcher(Thread):
    def run(self):
        global queue
        global page
	global done
        while True:
	    with done_lock:
		    if done:
			return

            with page_lock:
                my_page = page
                print('%s got page %s' % (current_thread(), my_page))
                page += 1

	    if total_count < (my_page+1)*100:
		with done_lock:
		    done = True
		return

            try:
                results = search(my_page)
            except:
                with done_lock:
		    done = True
                return

	    if len(results) == 0:
		time.sleep(0.2)
            for result in results:
                while queue.full():
                    time.sleep(0.01)

                _id = result['id']
                if db.packages.find({'_id': _id}).count() == 0:
                    print('%s putting %s into queue' % (current_thread(), _id))
                    queue.put(result)

class Parser(Thread):
    def run(self):
        global queue
        global data
        while True:
	    pkg = None
	    while not pkg:
		try:
		    pkg = queue.get(timeout=1)
		except:
		    with done_lock:
			if done:
			    return

            pkg = format_pkg(pkg)
	    if db.packages.find({'_id': pkg['_id']}).count() > 0:
		print('package data already in db') # SHOULDNT EVER HAPPEN
		continue

            print('%s got %s from queue' % (current_thread(), pkg['_id']))
            # want to use tmpfs for this but ran out of memory
            path = '%s/clone/%s' % (script_dir, pkg['_id'])

            start = time.time()
	    try:
                subprocess.check_output(['git', 'clone', pkg['clone_url'], path], stderr=subprocess.STDOUT)
	    except Exception as e:
	        print('clone failed due to: %s' % e.output)
		subprocess.check_output(['rm', '-rf', path])
{'fraction_non_alphanumeric': 0.10913647755753018, 'fraction_numerical': 0.01161995898838004, 'mean_word_length': 2.013040494166095, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1288883', 'n_tokens_mistral': 1490, 'n_tokens_neox': 1329, 'n_words': 396}