<commit_before>"""
gridftp interface
"""

from __future__ import absolute_import, division, print_function

import os
import logging
from threading import Thread, Event
from functools import partial
from collections import namedtuple
from datetime import datetime
import time
import tempfile
import shutil
import subprocess

logger = logging.getLogger('gridftp')

def _cmd(cmd, timeout=1200):
    subprocess.run(cmd, timeout=timeout, check=True)

def _cmd_output(cmd, timeout=1200):
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    try:
        output = p.communicate(timeout=timeout)[0].decode('utf-8')
        return (p.returncode, output)
    except subprocess.TimeoutExpired:
        p.kill()
        raise Exception('Request timed out')


def listify(lines,details=False,dotfiles=False):
    """Turn ls output into a list of NamedTuples"""
    out = []
    if details:
        File = namedtuple('File', ['directory','perms','subfiles',
                                   'owner','group','size','date',
                                   'name'])
        months = {'jan':1,'feb':2,'mar':3,'apr':4,'may':5,'jun':6,
                  'jul':7,'aug':8,'sep':9,'oct':10,'nov':11,'dec':12}
        for x in lines.split('\n'):
            if not x.strip():
                continue
            pieces = x.split()
            name = os.path.basename(pieces[-1])
            if name.startswith('.') and not dotfiles:
                continue
            d = x[0] == 'd'
            perms = pieces[0][1:]
            subfiles = 0
            if pieces[1].isdigit():
                subfiles = int(pieces[1])
                pieces = pieces[1:]
            else:
                subfiles = int(pieces[3])
            year = datetime.now().year
            month = months[pieces[4].lower()]
            day = int(pieces[5])
            if ':' in pieces[6]:
                hour,minute = pieces[6].split(':')
                dt = datetime(year,month,day,int(hour),int(minute))
            else:
                year = int(pieces[6])
                dt = datetime(year,month,day)
            out.append(File(d,perms,subfiles,pieces[1],pieces[2],
                            int(pieces[3]),dt,name))
    else:
        for x in lines.split('\n'):
            if not x.strip():
                continue
            f = x.split()[-1]
            if not f.startswith('.') or dotfiles:
                out.append(f)
    return out

class GridFTP(object):
    """GridFTP interface to command line client.

       Example:
           GridFTP.get('gsiftp://data.icecube.wisc.edu/file',
                       filename='/path/to/file')
    """

    _timeout = 1200 # 20 min default timeout

    @classmethod
    def supported_address(cls,address):
        """Return False for address types that are not supported"""
        if '://' not in address:
            return False
        addr_type = address.split(':')[0]
        if addr_type not in ('gsiftp','ftp'):
            return False
        return True

    @classmethod
    def address_split(cls,address):
        """Split an address into server/path parts"""
        pieces = address.split('://',1)
        if '/' in pieces[1]:
            pieces2 = pieces[1].split('/',1)
            return (pieces[0]+'://'+pieces2[0],'/'+pieces2[1])
        else:
            return (address,'/')

    @classmethod<commit_msg>increase gridftp timeout to handle large file sizes<commit_after>"""
gridftp interface
"""

from __future__ import absolute_import, division, print_function

import os
import logging
from threading import Thread, Event
from functools import partial
from collections import namedtuple
from datetime import datetime
import time
import tempfile
import shutil
import subprocess

logger = logging.getLogger('gridftp')

def _cmd(cmd, timeout=1200):
    subprocess.run(cmd, timeout=timeout, check=True)

def _cmd_output(cmd, timeout=1200):
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    try:
        output = p.communicate(timeout=timeout)[0].decode('utf-8')
        return (p.returncode, output)
    except subprocess.TimeoutExpired:
        p.kill()
        raise Exception('Request timed out')


def listify(lines,details=False,dotfiles=False):
    """Turn ls output into a list of NamedTuples"""
    out = []
    if details:
        File = namedtuple('File', ['directory','perms','subfiles',
                                   'owner','group','size','date',
                                   'name'])
        months = {'jan':1,'feb':2,'mar':3,'apr':4,'may':5,'jun':6,
                  'jul':7,'aug':8,'sep':9,'oct':10,'nov':11,'dec':12}
        for x in lines.split('\n'):
            if not x.strip():
                continue
            pieces = x.split()
            name = os.path.basename(pieces[-1])
            if name.startswith('.') and not dotfiles:
                continue
            d = x[0] == 'd'
            perms = pieces[0][1:]
            subfiles = 0
            if pieces[1].isdigit():
                subfiles = int(pieces[1])
                pieces = pieces[1:]
            else:
                subfiles = int(pieces[3])
            year = datetime.now().year
            month = months[pieces[4].lower()]
            day = int(pieces[5])
            if ':' in pieces[6]:
                hour,minute = pieces[6].split(':')
                dt = datetime(year,month,day,int(hour),int(minute))
            else:
                year = int(pieces[6])
                dt = datetime(year,month,day)
            out.append(File(d,perms,subfiles,pieces[1],pieces[2],
                            int(pieces[3]),dt,name))
    else:
        for x in lines.split('\n'):
            if not x.strip():
                continue
            f = x.split()[-1]
            if not f.startswith('.') or dotfiles:
                out.append(f)
    return out

class GridFTP(object):
    """GridFTP interface to command line client.

       Example:
           GridFTP.get('gsiftp://data.icecube.wisc.edu/file',
                       filename='/path/to/file')
    """

    _timeout = 3600 # 1 hour default timeout

    @classmethod
    def supported_address(cls,address):
        """Return False for address types that are not supported"""
        if '://' not in address:
            return False
        addr_type = address.split(':')[0]
        if addr_type not in ('gsiftp','ftp'):
            return False
        return True

    @classmethod
    def address_split(cls,address):
        """Split an address into server/path parts"""
        pieces = address.split('://',1)
        if '/' in pieces[1]:
            pieces2 = pieces[1].split('/',1)
            return (pieces[0]+'://'+pieces2[0],'/'+pieces2[1])
        else:
            return (address,'/')

    @classmethod
{'fraction_non_alphanumeric': 0.1387419165196943, 'fraction_numerical': 0.01748971193415638, 'mean_word_length': 1.958695652173913, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2607347', 'n_tokens_mistral': 2053, 'n_tokens_neox': 1940, 'n_words': 462}