<commit_before>#!/usr/bin/env python
import gzip, sys, threading
import Crawle.core

class SaveURLHandler(Crawle.core.Handler):
    """This handler simply saves all pages. into a gziped file. Any reponses
    with status other than 200 is placed back on the queue.

    This example also demonstrates the importance of synchronization as
    multiple threads can attempt to write to the file conncurrently.
    """

    def __init__(self, output):
        self.output = gzip.open(output,'ab')
	self.lock = threading.Lock()
	self.exit = False

    def process(self, info, rmi):
        if info['status'] != 200:
            print "%d - putting %s back on queue" % (info['status'],
                                                     info['url'])
            rmi.put(info['url'])
        else:
            self.lock.acquire()
            if self.exit:
                self.lock.release()
                return
            self.output.write(info['body'])
            self.output.write("===*===\n")
            self.lock.release()
				

    def stop(self):
        self.exit = True
        self.output.close()

if __name__ == '__main__':
    Crawle.core.runCrawle(sys.argv, handler=SaveURLHandler('output.gz'))<commit_msg>Corrections to example to work with refactored code<commit_after>#!/usr/bin/env python
import gzip, sys, threading
import Crawle.core

class SaveURLHandler(Crawle.core.Handler):
    """This handler simply saves all pages. into a gziped file. Any reponses
    with status other than 200 is placed back on the queue.

    This example also demonstrates the importance of synchronization as
    multiple threads can attempt to write to the file conncurrently.
    """

    def __init__(self, output):
        self.output = gzip.open(output,'ab')
	self.lock = threading.Lock()
	self.exit = False

    def process(self, reqRes, queue):
        if reqRes.responseStatus != 200:
            print "%d - putting %s back on queue" % (reqRes.responseStatus,
                                                     reqRes.responseURL)
            queue.put(reqRes.responseURL)
        else:
            self.lock.acquire()
            if self.exit:
                self.lock.release()
                return
            self.output.write(reqRes.responseBody)
            self.output.write("===*===\n")
            self.lock.release()
				

    def stop(self):
        self.exit = True
        self.output.close()

if __name__ == '__main__':
    Crawle.core.runCrawle(sys.argv, handler=SaveURLHandler('output.gz'))
{'fraction_non_alphanumeric': 0.10837633981738785, 'fraction_numerical': 0.004763795156808257, 'mean_word_length': 2.1072749691738593, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1515046', 'n_tokens_mistral': 730, 'n_tokens_neox': 690, 'n_words': 200}