<commit_before>import os
from shutil import copyfile
import re
import requests
from multiprocessing.pool import ThreadPool

from dvc.command.base import CmdBase, DvcLock
from dvc.data_cloud import sizeof_fmt
from dvc.logger import Logger
from dvc.exceptions import DvcException
from dvc.runtime import Runtime
from dvc.state_file import StateFile
from dvc.system import System
from dvc.command.data_sync import POOL_SIZE
from dvc.progress import Progress


class ImportFileError(DvcException):
    def __init__(self, msg):
        DvcException.__init__(self, 'Import file: {}'.format(msg))


class CmdImportFile(CmdBase):
    def __init__(self, settings):
        super(CmdImportFile, self).__init__(settings)

    def define_args(self, parser):
        self.set_no_git_actions(parser)
        self.set_lock_action(parser)

        parser.add_argument('input',
                            help='Input file/files.',
                            nargs='+')
        parser.add_argument('output',
                            help='Output file/directory.')

    def run(self):
        targets = []
        with DvcLock(self.is_locker, self.git):
            output = self.get_output()
            if not output:
                return 1

            for input in self.parsed_args.input:
                if not os.path.isdir(input):
                    targets.append((input, output))
                else:
                    input_dir = os.path.basename(input)
                    for root, dirs, files in os.walk(input):
                        for file in files:
                            filename = os.path.join(root, file)

                            rel = os.path.relpath(filename, input)
                            out = os.path.join(output, input_dir, rel)

                            out_dir = os.path.dirname(out)
                            if not os.path.exists(out_dir):
                                os.mkdir(out_dir)

                            targets.append((filename, out))
                pass
            self.import_files(targets, self.parsed_args.lock)
            message = 'DVC import files: {} -> {}'.format(str(self.parsed_args.input), output)
            self.commit_if_needed(message)
        pass

    def get_output(self):
        output = self.parsed_args.output

        if len(output) > 0 and output[-1] == os.path.sep and not os.path.isdir(output):
            Logger.error(u'Import error: output directory {} does not exist'.format(output))
            return None

        if type(self.parsed_args.input) == list and len(self.parsed_args.input) > 1 and not os.path.isdir(output):
            Logger.error(u'Import error: output {} has to be directory for multiple file import'.format(output))
            return None

        return output

    def import_and_commit_if_needed(self, input, output, lock=False, check_if_ready=True):
        if check_if_ready and not self.no_git_actions and not self.git.is_ready_to_go():
            return 1

        self.import_file(input, output, lock)

        message = 'DVC import file: {} {}'.format(input, output)
        return self.commit_if_needed(message)

    def import_file(self, input, output, lock):
        return self.import_files([(input, output)], lock)

    def collect_targets(self, targets):
        """
        Collect input, output and data_item's into tuples
        to be able to download them simulteniously later.
        """
        data_targets = []

        for target in targets:
            input = target[0]
            output = target[1]
            if not CmdImportFile.is_url(input):
                if not os.path.exists(input):
                    raise ImportFileError('Input file "{}" does not exist'.format(input))<commit_msg>Simplify output validation logic<commit_after>import os
from shutil import copyfile
import re
import requests
from multiprocessing.pool import ThreadPool

from dvc.command.base import CmdBase, DvcLock
from dvc.data_cloud import sizeof_fmt
from dvc.logger import Logger
from dvc.exceptions import DvcException
from dvc.runtime import Runtime
from dvc.state_file import StateFile
from dvc.system import System
from dvc.command.data_sync import POOL_SIZE
from dvc.progress import Progress


class ImportFileError(DvcException):
    def __init__(self, msg):
        DvcException.__init__(self, 'Import file: {}'.format(msg))


class CmdImportFile(CmdBase):
    def __init__(self, settings):
        super(CmdImportFile, self).__init__(settings)

    def define_args(self, parser):
        self.set_no_git_actions(parser)
        self.set_lock_action(parser)

        parser.add_argument('input',
                            help='Input file/files.',
                            nargs='+')
        parser.add_argument('output',
                            help='Output file/directory.')

    def run(self):
        targets = []
        with DvcLock(self.is_locker, self.git):
            output = self.parsed_args.output
            self.verify_output(output)

            for input in self.parsed_args.input:
                if not os.path.isdir(input):
                    targets.append((input, output))
                else:
                    input_dir = os.path.basename(input)
                    for root, dirs, files in os.walk(input):
                        for file in files:
                            filename = os.path.join(root, file)

                            rel = os.path.relpath(filename, input)
                            out = os.path.join(output, input_dir, rel)

                            out_dir = os.path.dirname(out)
                            if not os.path.exists(out_dir):
                                os.mkdir(out_dir)

                            targets.append((filename, out))
                pass
            self.import_files(targets, self.parsed_args.lock)
            message = 'DVC import files: {} -> {}'.format(str(self.parsed_args.input), output)
            self.commit_if_needed(message)
        pass

    def verify_output(self, output):
        if len(output) > 0 and output[-1] == os.path.sep and not os.path.isdir(output):
            raise ImportFileError(u'Import error: output directory {} does not exist'.format(output))

        if type(self.parsed_args.input) == list and len(self.parsed_args.input) > 1 and not os.path.isdir(output):
            msg = u'Import error: output {} has to be directory for multiple file import'
            raise ImportFileError(msg.format(output))
        pass

    def import_and_commit_if_needed(self, input, output, lock=False, check_if_ready=True):
        if check_if_ready and not self.no_git_actions and not self.git.is_ready_to_go():
            return 1

        self.import_file(input, output, lock)

        message = 'DVC import file: {} {}'.format(input, output)
        return self.commit_if_needed(message)

    def import_file(self, input, output, lock):
        return self.import_files([(input, output)], lock)

    def collect_targets(self, targets):
        """
        Collect input, output and data_item's into tuples
        to be able to download them simulteniously later.
        """
        data_targets = []

        for target in targets:
            input = target[0]
            output = target[1]
            if not CmdImportFile.is_url(input):
                if not os.path.exists(input):
                    raise ImportFileError('Input file "{}" does not exist'.format(input))
{'fraction_non_alphanumeric': 0.09267965077233042, 'fraction_numerical': 0.0017461383478844861, 'mean_word_length': 2.0170178282009723, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2102505', 'n_tokens_mistral': 2087, 'n_tokens_neox': 2010, 'n_words': 524}