<commit_before>import concurrent.futures
import pickle
import os

from .project import Project

import logging
l = logging.getLogger('project.Orgy')

import ana

def _run_one(p, analyses):
    for a in analyses:
        p.analyses.__dict__[a]()
    return p

class Orgy(object):
    def __init__(self, directory, save_dir='pickles'):
        self._save_dir = save_dir
        self._dir = directory
        self.projects = { }
        self._full_list = os.listdir(self._dir)
        ana.set_dl(self._save_dir + '/orgy')

        self.load()

    def run(self, analyses, background=False):
        if background:
            with concurrent.futures.ProcessPoolExecutor() as executor:
                futures = { }
                for f,p in self.projects.items():
                    completed = set(a for a,_,_ in p._analysis_results)
                    if len(set(analyses) - completed) > 0:
                        futures[executor.submit(_run_one, p, analyses)] = f

                for future in concurrent.futures.as_completed(futures):
                    self.projects[futures[future]] = future.result()
        else:
            for p in self.projects.values():
                _run_one(p, analyses)

    def _load_one(self, e):
        try:
            return pickle.load(open(self._save_dir + '/' + e + '/project.p'))
        except IOError: #pylint:disable=broad-except
            l.warning("Unable to load fucker %s (IOError)", e)

        try:
            return Project(self._dir + '/' + e)
        except (AngrError, cle.CLException):
            l.warning("Unable to create fucker %s", e, exc_info=True)

    def load(self):
        for e in self._full_list:
            self.projects[e] = self._load_one(e)

    def _save_one(self, e):
        try:
            pickle.dump(self.projects[e], open(self._save_dir + '/' + e, 'w'), pickle.HIGHEST_PROTOCOL)
        except Exception: #pylint:disable=broad-except
            l.warning("Unable to save fucker %s", e, exc_info=True)

    def save(self):
        for e in self._full_list:
            self._save_one(e)

from .errors import AngrError
import cle
<commit_msg>Orgy: only get actual files in self._full_list<commit_after>import concurrent.futures
import pickle
import os

from .project import Project

import logging
l = logging.getLogger('project.Orgy')

import ana

def _run_one(p, analyses):
    for a in analyses:
        p.analyses.__dict__[a]()
    return p

class Orgy(object):
    def __init__(self, directory, save_dir='pickles'):
        self._save_dir = save_dir
        self._dir = directory
        self.projects = { }
        self._full_list = (f for f in os.listdir(self._dir) if os.path.isfile(os.path.join(self._dir, f)))
        ana.set_dl(self._save_dir + '/orgy')

        self.load()

    def run(self, analyses, background=False):
        if background:
            with concurrent.futures.ProcessPoolExecutor() as executor:
                futures = { }
                for f,p in self.projects.items():
                    completed = set(a for a,_,_ in p._analysis_results)
                    if len(set(analyses) - completed) > 0:
                        futures[executor.submit(_run_one, p, analyses)] = f

                for future in concurrent.futures.as_completed(futures):
                    self.projects[futures[future]] = future.result()
        else:
            for p in self.projects.values():
                _run_one(p, analyses)

    def _load_one(self, e):
        try:
            return pickle.load(open(self._save_dir + '/' + e + '/project.p'))
        except IOError: #pylint:disable=broad-except
            l.warning("Unable to load fucker %s (IOError)", e)

        try:
            return Project(self._dir + '/' + e)
        except (AngrError, cle.CLException):
            l.warning("Unable to create fucker %s", e, exc_info=True)

    def load(self):
        for e in self._full_list:
            self.projects[e] = self._load_one(e)

    def _save_one(self, e):
        try:
            pickle.dump(self.projects[e], open(self._save_dir + '/' + e, 'w'), pickle.HIGHEST_PROTOCOL)
        except Exception: #pylint:disable=broad-except
            l.warning("Unable to save fucker %s", e, exc_info=True)

    def save(self):
        for e in self._full_list:
            self._save_one(e)

from .errors import AngrError
import cle

{'fraction_non_alphanumeric': 0.11427260211106012, 'fraction_numerical': 0.0004589261128958238, 'mean_word_length': 2.1314655172413794, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1487441', 'n_tokens_mistral': 1388, 'n_tokens_neox': 1335, 'n_words': 325}