<commit_before># Copyright (C) 2012 Codethink Limited


import cliapp
import os


class InvalidStateError(cliapp.AppException):
    
    def __init__(self, ref):
        cliapp.AppException.__init__(
                self, 'Failed to resolve state "%s"' % ref)


class State(object):

    def __init__(self, app, cache, repository):
        self.app = app
        self.cache = cache
        self.repository = repository
        self.title = None
        self.body = None
        self.author = None
        self.author_email = None
        self.left = None
        self.right = None
        self.diff = None
        self.date = None

    def history(self):
        if self.sha1 == 'UNCOMMITTED':
            sha1s = ['UNCOMMITTED'] + self.repository.history(None)
        else:
            sha1s = self.repository.history(self.sha1)
        return [self.cache.get(x) for x in sha1s]

    def diff_against(self, other):
        return self.repository.diff(other.sha1, self.sha1)


class UncommittedState(State):

    def __init__(self, app, cache, repository):
        State.__init__(self, app, cache, repository)

        self.identifier = 'UNCOMMITTED'
        self.sha1 = 'UNCOMMITTED'
        self.title = 'UNCOMMITTED'
        self.diff = self.repository.diff(None)

    def filenames(self):
        return self._list_files()

    def _list_files(self):
        # collect all elements from the project dir
        for root, dirs, files in os.walk(self.repository.dirname):
            # do not recurse into hidden subdirectories
            dirs[:] = [x for x in dirs if not x.startswith('.')]

            # remove the /files directory from the list as we don't
            # want to include YAML files in that directory
            if os.path.samefile(root, self.repository.dirname):
                dirs.remove('files')

            # skip all non-YAML and hidden files
            files[:] = [x for x in files
                        if x.endswith('.yaml') and not x.startswith('.')]

            # load all YAML files into the element tree
            for filename in files:
                yield os.path.relpath(os.path.join(root, filename),
                                      self.repository.dirname)
    
    def read(self, filename):<commit_msg>Fix an error if /files does not exist in UNCOMMITTED<commit_after># Copyright (C) 2012 Codethink Limited


import cliapp
import os


class InvalidStateError(cliapp.AppException):
    
    def __init__(self, ref):
        cliapp.AppException.__init__(
                self, 'Failed to resolve state "%s"' % ref)


class State(object):

    def __init__(self, app, cache, repository):
        self.app = app
        self.cache = cache
        self.repository = repository
        self.title = None
        self.body = None
        self.author = None
        self.author_email = None
        self.left = None
        self.right = None
        self.diff = None
        self.date = None

    def history(self):
        if self.sha1 == 'UNCOMMITTED':
            sha1s = ['UNCOMMITTED'] + self.repository.history(None)
        else:
            sha1s = self.repository.history(self.sha1)
        return [self.cache.get(x) for x in sha1s]

    def diff_against(self, other):
        return self.repository.diff(other.sha1, self.sha1)


class UncommittedState(State):

    def __init__(self, app, cache, repository):
        State.__init__(self, app, cache, repository)

        self.identifier = 'UNCOMMITTED'
        self.sha1 = 'UNCOMMITTED'
        self.title = 'UNCOMMITTED'
        self.diff = self.repository.diff(None)

    def filenames(self):
        return self._list_files()

    def _list_files(self):
        # collect all elements from the project dir
        for root, dirs, files in os.walk(self.repository.dirname):
            # do not recurse into hidden subdirectories
            dirs[:] = [x for x in dirs if not x.startswith('.')]

            # remove the /files directory from the list as we don't
            # want to include YAML files in that directory
            if os.path.samefile(root, self.repository.dirname):
                if 'files' in dirs:
                    dirs.remove('files')

            # skip all non-YAML and hidden files
            files[:] = [x for x in files
                        if x.endswith('.yaml') and not x.startswith('.')]

            # load all YAML files into the element tree
            for filename in files:
                yield os.path.relpath(os.path.join(root, filename),
                                      self.repository.dirname)
    
    def read(self, filename):
{'fraction_non_alphanumeric': 0.09571459647596259, 'fraction_numerical': 0.005220796171416141, 'mean_word_length': 1.9455477258167841, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '867584', 'n_tokens_mistral': 1358, 'n_tokens_neox': 1285, 'n_words': 384}