<commit_before>import os
import stat
import yaml
from checksumdir import dirhash

from dvc.system import System
from dvc.utils import file_md5
from dvc.exceptions import DvcException


class OutputError(DvcException):
    pass


class CmdOutputError(DvcException):
    def __init__(self, path, msg):
        super(CmdOutputError, self).__init__('Output file \'{}\' error: {}'.format(path, msg))


class CmdOutputNoCacheError(CmdOutputError):
    def __init__(self, path):
        super(CmdOutputNoCacheError, self).__init__(path, 'no cache')


class CmdOutputOutsideOfRepoError(CmdOutputError):
    def __init__(self, path):
        super(CmdOutputOutsideOfRepoError, self).__init__(path, 'outside of repository')


class CmdOutputDoesNotExistError(CmdOutputError):
    def __init__(self, path):
        super(CmdOutputDoesNotExistError, self).__init__(path, 'does not exist')


class CmdOutputIsNotFileOrDirError(CmdOutputError):
    def __init__(self, path):
        super(CmdOutputIsNotFileOrDirError, self).__init__(path, 'not a file or directory')


class CmdOutputAlreadyTrackedError(CmdOutputError):
    def __init__(self, path):
        super(CmdOutputAlreadyTrackedError, self).__init__(path, 'already tracked by scm(e.g. git)')


class Dependency(object):
    PARAM_PATH = 'path'
    PARAM_MD5 = 'md5'

    def __init__(self, project, path, md5=None):
        self.project = project
        self.path = os.path.abspath(os.path.realpath(path))

        if not self.path.startswith(self.project.root_dir):
            raise CmdOutputOutsideOfRepoError(self.rel_path)

        self.md5 = md5

    @property
    def dvc_path(self):
        return os.path.relpath(self.path, self.project.root_dir)

    @property
    def rel_path(self):
        return os.path.relpath(self.path, '.')

    def _changed_md5(self):
        if not os.path.exists(self.path):
            return True

        state = self.project.state.get(self.path)
        if state and state.mtime == self.mtime() and state.inode == self.inode():
            md5 = state.md5
        else:
            md5 = self.compute_md5()

        return self.md5 != md5

    def changed(self):
        return self._changed_md5()

    def compute_md5(self):
        if os.path.isdir(self.path):
            return dirhash(self.path, hashfunc='md5')
        else:
            return file_md5(self.path)[0]

    def mtime(self):
        return os.path.getmtime(self.path)

    def inode(self):
        return os.stat(self.path).st_ino

    def save(self):
        if not os.path.exists(self.path):
            raise CmdOutputDoesNotExistError(self.rel_path)

        if not os.path.isfile(self.path) and not os.path.isdir(self.path):
            raise CmdOutputIsNotFileOrDirError(self.rel_path)

        state = self.project.state.get(self.path)
        if state and state.mtime == self.mtime() and state.inode == self.inode():
            md5 = state.md5
            msg = '{} using md5 {} from state file'
            self.project.logger.debug(msg.format(self.path, md5))
            self.md5 = md5
        else:
            self.md5 = self.compute_md5()
            self.project.state.update(self.path, self.md5, self.mtime(), self.inode())

    def dumpd(self, cwd):
        return {
            Output.PARAM_PATH: os.path.relpath(self.path, cwd),
            Output.PARAM_MD5: self.md5,
        }

    @classmethod
    def loadd(cls, project, d, cwd=os.curdir):
        path = os.path.join(cwd, d[Output.PARAM_PATH])
        md5 = d.get(Output.PARAM_MD5, None)
        return cls(project, path, md5=md5)

    @classmethod
    def loadd_from(cls, project, d_list, cwd=os.curdir):
        return [cls.loadd(project, x, cwd=cwd) for x in d_list]

    @classmethod
    def loads(cls, project, s, cwd=os.curdir):
        return cls(project, os.path.join(cwd, s), md5=None)

    @classmethod
    def loads_from(cls, project, s_list, cwd=os.curdir):
        return [cls.loads(project, x, cwd=cwd) for x in s_list]

    def stage(self):
        for stage in self.project.stages():
            for out in stage.outs:
                if self.path == out.path:
                    return stage
        return None


class Output(Dependency):
    PARAM_CACHE = 'cache'

    def __init__(self, project, path, md5=None, use_cache=True):
        super(Output, self).__init__(project, path, md5=md5)
        self.use_cache = use_cache

    @property
    def cache(self):
        return self.project.cache.get(self.md5)

    def dumpd(self, cwd):
        ret = super(Output, self).dumpd(cwd)
        ret[Output.PARAM_CACHE] = self.use_cache
        return ret

    @classmethod
    def loadd(cls, project, d, cwd=os.curdir):
        ret = super(Output, cls).loadd(project, d, cwd=cwd)
        ret.use_cache = d.get(Output.PARAM_CACHE, True)
        return ret

    @classmethod
    def loads(cls, project, s, use_cache=True, cwd=os.curdir):
        ret = super(Output, cls).loads(project, s, cwd=cwd)
        ret.use_cache = use_cache
        return ret

    @classmethod
    def loads_from(cls, project, s_list, use_cache=False, cwd=os.curdir):
        return [cls.loads(project, x, use_cache=use_cache, cwd=cwd) for x in s_list]

    def changed(self):
        ret = True

        if not self.use_cache:
            ret = super(Output, self).changed()
        elif os.path.exists(self.path) and \
           os.path.exists(self.cache) and \
           System.samefile(self.path, self.cache) and \
           os.stat(self.cache).st_mode & stat.S_IREAD:
            ret = False

        msg = "Data {} with cache {} "
        if ret:
            msg += "changed"
        else:
            msg += "didn't change"
        self.project.logger.debug(msg.format(self.path, self.cache))

        return ret

    def hardlink(self, src, link):
        self.project.logger.debug("creating hardlink {} -> {}".format(src, link))
        System.hardlink(src, link)
        os.chmod(src, stat.S_IREAD)

    def dir_cache(self):
        res = {}
        for root, dirs, files in os.walk(self.cache):
            for fname in files:
                path = os.path.join(root, fname)
                relpath = os.path.relpath(path, self.cache)
                with open(path, 'r') as fd:
                    d = yaml.safe_load(fd)
                md5 = d[Output.PARAM_MD5]
                res[relpath] = self.project.cache.get(md5)
        return res

    def checkout(self):
        if not self.use_cache:
            return
<commit_msg>dvc: properly check dir<commit_after>import os
import stat
import yaml
from checksumdir import dirhash

from dvc.system import System
from dvc.utils import file_md5
from dvc.exceptions import DvcException


class OutputError(DvcException):
    pass


class CmdOutputError(DvcException):
    def __init__(self, path, msg):
        super(CmdOutputError, self).__init__('Output file \'{}\' error: {}'.format(path, msg))


class CmdOutputNoCacheError(CmdOutputError):
    def __init__(self, path):
        super(CmdOutputNoCacheError, self).__init__(path, 'no cache')


class CmdOutputOutsideOfRepoError(CmdOutputError):
    def __init__(self, path):
        super(CmdOutputOutsideOfRepoError, self).__init__(path, 'outside of repository')


class CmdOutputDoesNotExistError(CmdOutputError):
    def __init__(self, path):
        super(CmdOutputDoesNotExistError, self).__init__(path, 'does not exist')


class CmdOutputIsNotFileOrDirError(CmdOutputError):
    def __init__(self, path):
        super(CmdOutputIsNotFileOrDirError, self).__init__(path, 'not a file or directory')


class CmdOutputAlreadyTrackedError(CmdOutputError):
    def __init__(self, path):
        super(CmdOutputAlreadyTrackedError, self).__init__(path, 'already tracked by scm(e.g. git)')


class Dependency(object):
    PARAM_PATH = 'path'
    PARAM_MD5 = 'md5'

    def __init__(self, project, path, md5=None):
        self.project = project
        self.path = os.path.abspath(os.path.realpath(path))

        if not self.path.startswith(self.project.root_dir):
            raise CmdOutputOutsideOfRepoError(self.rel_path)

        self.md5 = md5

    @property
    def dvc_path(self):
        return os.path.relpath(self.path, self.project.root_dir)

    @property
    def rel_path(self):
        return os.path.relpath(self.path, '.')

    def _changed_md5(self):
        if not os.path.exists(self.path):
            return True

        state = self.project.state.get(self.path)
        if state and state.mtime == self.mtime() and state.inode == self.inode():
            md5 = state.md5
        else:
            md5 = self.compute_md5()

        return self.md5 != md5

    def changed(self):
        return self._changed_md5()

    def compute_md5(self):
        if os.path.isdir(self.path):
            return dirhash(self.path, hashfunc='md5')
        else:
            return file_md5(self.path)[0]

    def mtime(self):
        return os.path.getmtime(self.path)

    def inode(self):
        return os.stat(self.path).st_ino

    def save(self):
        if not os.path.exists(self.path):
            raise CmdOutputDoesNotExistError(self.rel_path)

        if not os.path.isfile(self.path) and not os.path.isdir(self.path):
            raise CmdOutputIsNotFileOrDirError(self.rel_path)

        state = self.project.state.get(self.path)
        if state and state.mtime == self.mtime() and state.inode == self.inode():
            md5 = state.md5
            msg = '{} using md5 {} from state file'
            self.project.logger.debug(msg.format(self.path, md5))
            self.md5 = md5
        else:
            self.md5 = self.compute_md5()
            self.project.state.update(self.path, self.md5, self.mtime(), self.inode())

    def dumpd(self, cwd):
        return {
            Output.PARAM_PATH: os.path.relpath(self.path, cwd),
            Output.PARAM_MD5: self.md5,
        }

    @classmethod
    def loadd(cls, project, d, cwd=os.curdir):
        path = os.path.join(cwd, d[Output.PARAM_PATH])
        md5 = d.get(Output.PARAM_MD5, None)
        return cls(project, path, md5=md5)

    @classmethod
    def loadd_from(cls, project, d_list, cwd=os.curdir):
        return [cls.loadd(project, x, cwd=cwd) for x in d_list]

    @classmethod
    def loads(cls, project, s, cwd=os.curdir):
        return cls(project, os.path.join(cwd, s), md5=None)

    @classmethod
    def loads_from(cls, project, s_list, cwd=os.curdir):
        return [cls.loads(project, x, cwd=cwd) for x in s_list]

    def stage(self):
        for stage in self.project.stages():
            for out in stage.outs:
                if self.path == out.path:
                    return stage
        return None


class Output(Dependency):
    PARAM_CACHE = 'cache'

    def __init__(self, project, path, md5=None, use_cache=True):
        super(Output, self).__init__(project, path, md5=md5)
        self.use_cache = use_cache

    @property
    def cache(self):
        return self.project.cache.get(self.md5)

    def dumpd(self, cwd):
        ret = super(Output, self).dumpd(cwd)
        ret[Output.PARAM_CACHE] = self.use_cache
        return ret

    @classmethod
    def loadd(cls, project, d, cwd=os.curdir):
        ret = super(Output, cls).loadd(project, d, cwd=cwd)
        ret.use_cache = d.get(Output.PARAM_CACHE, True)
        return ret

    @classmethod
    def loads(cls, project, s, use_cache=True, cwd=os.curdir):
        ret = super(Output, cls).loads(project, s, cwd=cwd)
        ret.use_cache = use_cache
        return ret

    @classmethod
    def loads_from(cls, project, s_list, use_cache=False, cwd=os.curdir):
        return [cls.loads(project, x, use_cache=use_cache, cwd=cwd) for x in s_list]

    def _changed_file(self, path, cache):
        if os.path.isfile(path) and \
           os.path.isfile(cache) and \
           System.samefile(path, cache) and \
           os.stat(cache).st_mode & stat.S_IREAD:
            return False

        return True

    def _changed_dir(self):
        if not os.path.isdir(self.path) or not os.path.isdir(self.cache):
            return True

        for root, dirs, files in os.walk(self.path):
            for fname in files:
                path = os.path.join(root, fname)
                mtime = os.path.getmtime(path)
                inode = os.stat(path).st_ino

                state = self.project.state.get(path)
                if state and state.mtime == mtime and state.inode == inode:
                    md5 = state.md5
                else:
                    md5 = file_md5(path)[0]

                cache = self.project.cache.get(md5)
                if self._changed_file(path, cache):
                    return True

        return False

    def changed(self):
        ret = True

        if not self.use_cache:
            ret = super(Output, self).changed()
        elif os.path.isfile(self.path) and \
             os.path.isfile(self.cache):
            ret = self._changed_file(self.path, self.cache)
        elif os.path.isdir(self.path) and \
             os.path.isdir(self.cache):
            ret = self._changed_dir()

        msg = "Data {} with cache {} "
        if ret:
            msg += "changed"
        else:
            msg += "didn't change"
        self.project.logger.debug(msg.format(self.path, self.cache))

        return ret

    def hardlink(self, src, link):
        self.project.logger.debug("creating hardlink {} -> {}".format(src, link))
        System.hardlink(src, link)
        os.chmod(src, stat.S_IREAD)

    def dir_cache(self):
        res = {}
        for root, dirs, files in os.walk(self.cache):
            for fname in files:
                path = os.path.join(root, fname)
                relpath = os.path.relpath(path, self.cache)
                with open(path, 'r') as fd:
                    d = yaml.safe_load(fd)
                md5 = d[Output.PARAM_MD5]
                res[relpath] = self.project.cache.get(md5)
        return res

    def checkout(self):
        if not self.use_cache:
            return

{'fraction_non_alphanumeric': 0.11498606446080183, 'fraction_numerical': 0.006288858715071822, 'mean_word_length': 2.292705882352941, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '291928', 'n_tokens_mistral': 4630, 'n_tokens_neox': 4493, 'n_words': 997}