<commit_before>import os
from alembic import util
import shutil
import re
import inspect
import datetime

_rev_file = re.compile(r'([a-z0-9]+)\.py$')
_mod_def_re = re.compile(r'(upgrade|downgrade)_([a-z0-9]+)')

class ScriptDirectory(object):
    def __init__(self, dir):
        self.dir = dir
        self.versions = os.path.join(self.dir, 'versions')

        if not os.access(dir, os.F_OK):
            raise util.CommandError("Path doesn't exist: %r.  Please use "
                        "the 'init' command to create a new "
                        "scripts folder." % dir)

    @classmethod
    def from_config(cls, config):
        return ScriptDirectory(
                    config.get_main_option('script_location'))

    def walk_revisions(self):
        """Iterate through all revisions.

        This is actually a breadth-first tree traversal,
        with leaf nodes being heads.

        """
        heads = set(self._get_heads())
        base = self._get_rev("base")
        while heads:
            todo = set(heads)
            heads = set()
            for head in todo:
                if head in heads:
                    break
                for sc in self._revs(head, base):
                    if sc.is_branch_point and sc.revision not in todo:
                        heads.add(sc.revision)
                        break
                    else:
                        yield sc

    def _get_rev(self, id_):
        if id_ == 'head':
            id_ = self._current_head()
        elif id_ == 'base':
            id_ = None
        try:
            return self._revision_map[id_]
        except KeyError:
            raise util.CommandError("No such revision %s" % id_)

    def _revs(self, upper, lower):
        lower = self._get_rev(lower)
        upper = self._get_rev(upper)
        script = upper
        while script != lower:
            yield script
            script = self._revision_map[script.down_revision]

    def upgrade_from(self, destination, current_rev):
        return [
            (script.module.upgrade, script.revision) for script in 
            reversed(list(self._revs(destination, current_rev)))
            ]

    def downgrade_to(self, destination, current_rev):
        return [
            (script.module.downgrade, script.down_revision) for script in 
            self._revs(current_rev, destination)
            ]

    def run_env(self):
        util.load_python_file(self.dir, 'env.py')

    @util.memoized_property
    def _revision_map(self):
        map_ = {}
        for file_ in os.listdir(self.versions):
            script = Script.from_filename(self.versions, file_)
            if script is None:
                continue
            if script.revision in map_:
                util.warn("Revision %s is present more than once" %
                                script.revision)
            map_[script.revision] = script
        for rev in map_.values():
            if rev.down_revision is None:
                continue
            if rev.down_revision not in map_:
                util.warn("Revision %s referenced from %s is not present"
                            % (rev.down_revision, rev))
                rev.down_revision = None
            else:
                map_[rev.down_revision].add_nextrev(rev.revision)
        map_[None] = None
        return map_

    def _rev_path(self, rev_id):
        filename = "%s.py" % rev_id
        return os.path.join(self.versions, filename)

    def write(self, rev_id, content):
        path = self._rev_path(rev_id)
        file(path, 'w').write(content)<commit_msg>another file call<commit_after>import os
from alembic import util
import shutil
import re
import inspect
import datetime

_rev_file = re.compile(r'([a-z0-9]+)\.py$')
_mod_def_re = re.compile(r'(upgrade|downgrade)_([a-z0-9]+)')

class ScriptDirectory(object):
    def __init__(self, dir):
        self.dir = dir
        self.versions = os.path.join(self.dir, 'versions')

        if not os.access(dir, os.F_OK):
            raise util.CommandError("Path doesn't exist: %r.  Please use "
                        "the 'init' command to create a new "
                        "scripts folder." % dir)

    @classmethod
    def from_config(cls, config):
        return ScriptDirectory(
                    config.get_main_option('script_location'))

    def walk_revisions(self):
        """Iterate through all revisions.

        This is actually a breadth-first tree traversal,
        with leaf nodes being heads.

        """
        heads = set(self._get_heads())
        base = self._get_rev("base")
        while heads:
            todo = set(heads)
            heads = set()
            for head in todo:
                if head in heads:
                    break
                for sc in self._revs(head, base):
                    if sc.is_branch_point and sc.revision not in todo:
                        heads.add(sc.revision)
                        break
                    else:
                        yield sc

    def _get_rev(self, id_):
        if id_ == 'head':
            id_ = self._current_head()
        elif id_ == 'base':
            id_ = None
        try:
            return self._revision_map[id_]
        except KeyError:
            raise util.CommandError("No such revision %s" % id_)

    def _revs(self, upper, lower):
        lower = self._get_rev(lower)
        upper = self._get_rev(upper)
        script = upper
        while script != lower:
            yield script
            script = self._revision_map[script.down_revision]

    def upgrade_from(self, destination, current_rev):
        return [
            (script.module.upgrade, script.revision) for script in 
            reversed(list(self._revs(destination, current_rev)))
            ]

    def downgrade_to(self, destination, current_rev):
        return [
            (script.module.downgrade, script.down_revision) for script in 
            self._revs(current_rev, destination)
            ]

    def run_env(self):
        util.load_python_file(self.dir, 'env.py')

    @util.memoized_property
    def _revision_map(self):
        map_ = {}
        for file_ in os.listdir(self.versions):
            script = Script.from_filename(self.versions, file_)
            if script is None:
                continue
            if script.revision in map_:
                util.warn("Revision %s is present more than once" %
                                script.revision)
            map_[script.revision] = script
        for rev in map_.values():
            if rev.down_revision is None:
                continue
            if rev.down_revision not in map_:
                util.warn("Revision %s referenced from %s is not present"
                            % (rev.down_revision, rev))
                rev.down_revision = None
            else:
                map_[rev.down_revision].add_nextrev(rev.revision)
        map_[None] = None
        return map_

    def _rev_path(self, rev_id):
        filename = "%s.py" % rev_id
        return os.path.join(self.versions, filename)

    def write(self, rev_id, content):
        path = self._rev_path(rev_id)
        open(path, 'w').write(content)
{'fraction_non_alphanumeric': 0.09391497638232843, 'fraction_numerical': 0.0011114198388441233, 'mean_word_length': 1.6692621431219874, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1153387', 'n_tokens_mistral': 2070, 'n_tokens_neox': 1987, 'n_words': 517}