<commit_before>import os
import re
import getpass
import posixpath
from subprocess import Popen, PIPE

from dvc.config import Config
from dvc.remote.base import RemoteBase
from dvc.remote.local import RemoteLOCAL
from dvc.exceptions import DvcException
from dvc.logger import Logger
from dvc.utils import fix_env


class RemoteHDFS(RemoteBase):
    scheme = 'hdfs'
    REGEX = r'^hdfs://((?P<user>.*)@)?.*$'
    PARAM_CHECKSUM = 'checksum'

    def __init__(self, project, config):
        self.project = project
        self.url = config.get(Config.SECTION_REMOTE_URL, '/')
        self.user = self.group('user')
        if not self.user:
            self.user = config.get(Config.SECTION_REMOTE_USER,
                                   getpass.getuser())

    def hadoop_fs(self, cmd, user=None):
        cmd = 'hadoop fs -' + cmd
        if user:
            cmd = 'HADOOP_USER_NAME={} '.format(user) + cmd
        p = Popen(cmd,
                  shell=True,
                  close_fds=True,
                  executable=os.getenv('SHELL'),
                  env=fix_env(os.environ),
                  stdin=PIPE,
                  stdout=PIPE,
                  stderr=PIPE)
        out, err = p.communicate()
        if p.returncode != 0:
            raise DvcException('HDFS command failed: {}: {}'.format(cmd, err))
        return out.decode('utf-8')

    @staticmethod
    def _group(regex, s, gname):
        match = re.match(regex, s)
        assert match is not None
        return match.group(gname)

    def checksum(self, path_info):
        regex = r'.*\t.*\t(?P<checksum>.*)'
        stdout = self.hadoop_fs('checksum {}'.format(path_info['url']),
                                user=path_info['user'])
        return self._group(regex, stdout, 'checksum')
<commit_msg>hdfs: don't use close_fds on Windows<commit_after>import os
import re
import getpass
import posixpath
from subprocess import Popen, PIPE

from dvc.config import Config
from dvc.remote.base import RemoteBase
from dvc.remote.local import RemoteLOCAL
from dvc.exceptions import DvcException
from dvc.logger import Logger
from dvc.utils import fix_env


class RemoteHDFS(RemoteBase):
    scheme = 'hdfs'
    REGEX = r'^hdfs://((?P<user>.*)@)?.*$'
    PARAM_CHECKSUM = 'checksum'

    def __init__(self, project, config):
        self.project = project
        self.url = config.get(Config.SECTION_REMOTE_URL, '/')
        self.user = self.group('user')
        if not self.user:
            self.user = config.get(Config.SECTION_REMOTE_USER,
                                   getpass.getuser())

    def hadoop_fs(self, cmd, user=None):
        cmd = 'hadoop fs -' + cmd
        if user:
            cmd = 'HADOOP_USER_NAME={} '.format(user) + cmd

        # NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
        # See https://github.com/iterative/dvc/issues/1197.
        close_fds = (os.name != 'nt')

        p = Popen(cmd,
                  shell=True,
                  close_fds=close_fds,
                  executable=os.getenv('SHELL'),
                  env=fix_env(os.environ),
                  stdin=PIPE,
                  stdout=PIPE,
                  stderr=PIPE)
        out, err = p.communicate()
        if p.returncode != 0:
            raise DvcException('HDFS command failed: {}: {}'.format(cmd, err))
        return out.decode('utf-8')

    @staticmethod
    def _group(regex, s, gname):
        match = re.match(regex, s)
        assert match is not None
        return match.group(gname)

    def checksum(self, path_info):
        regex = r'.*\t.*\t(?P<checksum>.*)'
        stdout = self.hadoop_fs('checksum {}'.format(path_info['url']),
                                user=path_info['user'])
        return self._group(regex, stdout, 'checksum')

{'fraction_non_alphanumeric': 0.12116402116402117, 'fraction_numerical': 0.0021164021164021165, 'mean_word_length': 2.0739837398373986, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1310514', 'n_tokens_mistral': 1221, 'n_tokens_neox': 1160, 'n_words': 269}