<commit_before>import subprocess
import os

from .config import SEED_FILES, MODEL_FILES, CONFIG_FILES, CRAWLS_PATH, LANG_DETECT_PATH


class AcheCrawl(object):

    def __init__(self, crawl_name, seeds_file, model_name, conf_name):
        self.crawl_name = crawl_name
        self.config = os.path.join(CONFIG_FILES, conf_name)
        self.seeds_file = os.path.join(SEED_FILES, seeds_file)
        self.model_dir = os.path.join(MODEL_FILES, model_name)
        self.crawl_dir = os.path.join(CRAWLS_PATH, crawl_name)
        self.proc = None


    def start(self):
        with open(os.path.join(self.crawl_dir, 'stdout.txt'), 'w') as stdout:
            with open(os.path.join(self.crawl_dir,'stderr.txt'), 'w') as stderr:
                self.proc = subprocess.Popen(['ache', 'startCrawl', self.crawl_dir, self.config, self.seeds_file,
                                          self.model_dir, LANG_DETECT_PATH], stdout=stdout, stderr=stderr)
        return self.proc.pid

    def stop(self):
        if self.proc is not None:
            print("Killing %s" % str(self.proc.pid))
            self.proc.kill()

    def status(self):
        if self.proc is None:
            return "No process exists"
        elif self.proc.returncode is None:
            return "Running"
        elif self.proc.returncode < 0:
            return "Stopped (Unused)"
        else:
            return "An error occurred"


class NutchCrawl(object):

    def __init__(self, seed_dir, crawl_dir):
        self.seed_dir =  SEED_FILES + "/" + seed_dir
        self.crawl_dir = CRAWLS_PATH + "/" + crawl_dir
        #TODO Switch from "2" to parameter.
        # For now let's set up number_of_rounds to 2.
        self.number_of_rounds = "2"
        #self.number_of_rounds = numberOfRounds
        self.proc = None

    def start(self):
        subprocess.Popen(['mkdir', self.crawl_dir]).wait()
        self.proc = subprocess.Popen(['crawl', self.seed_dir, self.crawl_dir, self.number_of_rounds])
        return self.proc.pid

    def stop(self):
        if self.proc is not None:
            print("Killing %s" % str(self.proc.pid))
            self.proc.kill()

    def get_status(self):
        self.proc.poll()
        if self.proc is None:
            self.status = "No process exists"
        elif self.proc.returncode is None:
            self.status = "Running crawl"
        elif self.proc.returncode < 0:
            self.status = "Crawl process was terminated by signal %s" % self.proc.returncode
        else:
            self.status = "Crawl process ended"
        return self.status

    def dump_images(self):
        subprocess.Popen(['mkdir', self.img_dir]).wait()
        with open(os.path.join(self.crawl_dir, 'img_stdout.txt'), 'w') as stdout:
            with open(os.path.join(self.crawl_dir,'img_stderr.txt'), 'w') as stderr:<commit_msg>clean nutch instance init and change number of rounds parameter to one<commit_after>import subprocess
import os

from .config import SEED_FILES, MODEL_FILES, CONFIG_FILES, CRAWLS_PATH, LANG_DETECT_PATH, IMAGE_SPACE_PATH


class AcheCrawl(object):

    def __init__(self, crawl_name, seeds_file, model_name, conf_name):
        self.crawl_name = crawl_name
        self.config = os.path.join(CONFIG_FILES, conf_name)
        self.seeds_file = os.path.join(SEED_FILES, seeds_file)
        self.model_dir = os.path.join(MODEL_FILES, model_name)
        self.crawl_dir = os.path.join(CRAWLS_PATH, crawl_name)
        self.proc = None


    def start(self):
        with open(os.path.join(self.crawl_dir, 'stdout.txt'), 'w') as stdout:
            with open(os.path.join(self.crawl_dir,'stderr.txt'), 'w') as stderr:
                self.proc = subprocess.Popen(['ache', 'startCrawl', self.crawl_dir, self.config, self.seeds_file,
                                          self.model_dir, LANG_DETECT_PATH], stdout=stdout, stderr=stderr)
        return self.proc.pid

    def stop(self):
        if self.proc is not None:
            print("Killing %s" % str(self.proc.pid))
            self.proc.kill()

    def status(self):
        if self.proc is None:
            return "No process exists"
        elif self.proc.returncode is None:
            return "Running"
        elif self.proc.returncode < 0:
            return "Stopped (Unused)"
        else:
            return "An error occurred"


class NutchCrawl(object):

    def __init__(self, seed_dir, crawl_dir):
        self.seed_dir =  os.path.join(SEED_FILES, seed_dir)
        self.crawl_dir = os.path.join(CRAWLS_PATH, crawl_dir)
        self.img_dir = os.path.join(IMAGE_SPACE_PATH, crawl_dir)
        #TODO Switch from "1" to parameter.
        # For now let's set up number_of_rounds to 1.
        self.number_of_rounds = "1"
        #self.number_of_rounds = numberOfRounds
        self.status = ""
        self.proc = None

    def start(self):
        subprocess.Popen(['mkdir', self.crawl_dir]).wait()
        self.proc = subprocess.Popen(['crawl', self.seed_dir, self.crawl_dir, self.number_of_rounds])
        return self.proc.pid

    def stop(self):
        if self.proc is not None:
            print("Killing %s" % str(self.proc.pid))
            self.proc.kill()

    def get_status(self):
        self.proc.poll()
        if self.proc is None:
            self.status = "No process exists"
        elif self.proc.returncode is None:
            self.status = "Running crawl"
        elif self.proc.returncode < 0:
            self.status = "Crawl process was terminated by signal %s" % self.proc.returncode
        else:
            self.status = "Crawl process ended"
        return self.status

    def dump_images(self):
        subprocess.Popen(['mkdir', self.img_dir]).wait()
        with open(os.path.join(self.crawl_dir, 'img_stdout.txt'), 'w') as stdout:
            with open(os.path.join(self.crawl_dir,'img_stderr.txt'), 'w') as stderr:
{'fraction_non_alphanumeric': 0.11260876983449923, 'fraction_numerical': 0.0017061934823408974, 'mean_word_length': 2.3497142857142856, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4958255', 'n_tokens_mistral': 1976, 'n_tokens_neox': 1858, 'n_words': 425}