<commit_before># -*- coding: utf-8 -*-

import optparse
import os.path
import sys

from crawlcrunch.companies import CompaniesList
from crawlcrunch.crawler import Crawler

def main(argv=sys.argv, quiet=False):
    command = CrawlCrunchCommand(argv, quiet)
    return command.run()

class CrawlCrunchCommand(object):
    
    description = "Crawl the companies information from crunchbase.com"
    usage = "usage: %prog [options] dest_dir"
    parser = optparse.OptionParser(usage, description=description)

    def __init__(self, argv, quiet=False):
        self.quiet = quiet
        self.options, self.args = self.parser.parse_args(argv[1:])

    def run(self):
        if len(self.args) == 0:
            self.out('You must provide a destination directory')
            return 2
        elif len(self.args) > 1:
            self.out(
                'You must provide one destination directory, not {0}.'.format(len(self.args)))
            return 2
        elif not os.path.isdir(self.args[0]):
            self.out("The directory '{0}' does not exist!".format(
                    self.args[0]))
            self.out('Please, create it first.')
            return 2
        basedir = os.path.expanduser(self.args[0])
        basedir = os.path.abspath(basedir)
        basedir = os.path.normpath(basedir)
        cl = CompaniesList(basedir)
        cl.create_list()
        crawler = Crawler(cl)
        crawler.run()
        return 0

    def out(self, msg): # pragma: no cover
        if not self.quiet:
            print msg
<commit_msg>better naming<commit_after># -*- coding: utf-8 -*-

import optparse
import os.path
import sys

from crawlcrunch.companies import CompaniesList
from crawlcrunch.crawler import Crawler

def main(argv=sys.argv, quiet=False):
    command = CrawlCrunchCommand(argv, quiet)
    return command.run()

class CrawlCrunchCommand(object):
    
    description = "Crawl the companies information from crunchbase.com"
    usage = "usage: %prog [options] dest_dir"
    parser = optparse.OptionParser(usage, description=description)

    def __init__(self, argv, quiet=False):
        self.quiet = quiet
        self.options, self.args = self.parser.parse_args(argv[1:])

    def run(self):
        if len(self.args) == 0:
            self.out('You must provide a destination directory')
            return 2
        elif len(self.args) > 1:
            self.out(
                'You must provide one destination directory, not {0}.'.format(len(self.args)))
            return 2
        elif not os.path.isdir(self.args[0]):
            self.out("The directory '{0}' does not exist!".format(
                    self.args[0]))
            self.out('Please, create it first.')
            return 2
        basedir = os.path.expanduser(self.args[0])
        basedir = os.path.abspath(basedir)
        basedir = os.path.normpath(basedir)
        cl = CompaniesList(basedir)
        cl.create_list()
        crawler = Crawler(cl)
        crawler.crawl()
        return 0

    def out(self, msg): # pragma: no cover
        if not self.quiet:
            print msg

{'fraction_non_alphanumeric': 0.11067961165048544, 'fraction_numerical': 0.008414239482200648, 'mean_word_length': 2.3165236051502145, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5440250', 'n_tokens_mistral': 950, 'n_tokens_neox': 910, 'n_words': 239}