<commit_before>"""
FTP-based retrieval of files.
"""
from __future__ import absolute_import

import ftplib
import logging
import os
import re

from ._core import DataSource, fetch_file, RemoteFetchException

_log = logging.getLogger(__name__)
DEFAULT_SOCKET_TIMEOUT_SECS = 60 * 5.0


def _fetch_files(hostname,
                 target_dir,
                 reporter,
                 get_filepaths_fn,
                 override_existing=False,
                 filename_transform=None,
                 retries=3):
    """
    Fetch fetch files on the given FTP server.

    The get_filepaths_fn callback is used to get a list of files to download.

    It it passed an instance of the connection so that it can query the server if needed.

    :type get_filepaths_fn: (ftplib.FTP) -> list of str
    :type hostname: str
    :type target_dir: str
    :type reporter: ResultHandler
    """

    try:
        ftp = ftplib.FTP(hostname, timeout=DEFAULT_SOCKET_TIMEOUT_SECS)
    except BaseException:
        _log.exception('Error connecting to FTP')
        raise RemoteFetchException(
            'Error connecting to FTP',
            'host: {}, timeout: {}'.format(hostname, DEFAULT_SOCKET_TIMEOUT_SECS)
        )

    try:
        ftp.login()

        files_itr = iter(get_filepaths_fn(ftp))
        filename = next(files_itr)
        retry_count = 0

        while True:
            try:
                retry_count += 1
                _log.debug('Next filename: %r', filename)

                def ftp_fetch(t):
                    """Fetch data to filename t"""
                    # https://bitbucket.org/logilab/pylint/issue/271/spurious-warning-w0640-issued-for-loop
                    # pylint: disable=cell-var-from-loop
                    _log.debug('Retrieving %r to %r', filename, t)
                    with open(t, 'wb') as f:
                        ftp.retrbinary('RETR ' + filename, f.write)
                    return True

                fetch_file(
                    'ftp://%s%s' % (hostname, filename),
                    ftp_fetch,
                    reporter,
                    os.path.basename(filename),
                    target_dir,
                    filename_transform=filename_transform,
                    override_existing=override_existing
                )
                filename = next(files_itr)
                retry_count = 0

            except EOFError:
                if retry_count >= retries:
                    _log.debug('Error fetching file. Reconnecting to ftp server...')
                    raise
                _log.debug('Error fetching file. Reconnecting to ftp server...')

                # Connection was closed; try to re-connect
                try:
                    ftp = ftplib.FTP(hostname, timeout=DEFAULT_SOCKET_TIMEOUT_SECS)<commit_msg>Dev/ftplib retry on 400 (#3)<commit_after>"""
FTP-based retrieval of files.
"""
from __future__ import absolute_import

import ftplib
import logging
import os
import re
import time

from ._core import DataSource, fetch_file, RemoteFetchException

_log = logging.getLogger(__name__)
DEFAULT_SOCKET_TIMEOUT_SECS = 60 * 5.0


def _fetch_files(hostname,
                 target_dir,
                 reporter,
                 get_filepaths_fn,
                 override_existing=False,
                 filename_transform=None,
                 retries=3,
                 retry_delay=5):
    """
    Fetch fetch files on the given FTP server.

    The get_filepaths_fn callback is used to get a list of files to download.

    It it passed an instance of the connection so that it can query the server if needed.

    :type get_filepaths_fn: (ftplib.FTP) -> list of str
    :type hostname: str
    :type target_dir: str
    :type reporter: ResultHandler
    :type retries: int
    :type retry_delay: int
    """

    try:
        ftp = ftplib.FTP(hostname, timeout=DEFAULT_SOCKET_TIMEOUT_SECS)
    except BaseException:
        _log.exception('Error connecting to FTP')
        raise RemoteFetchException(
            'Error connecting to FTP',
            'host: {}, timeout: {}'.format(hostname, DEFAULT_SOCKET_TIMEOUT_SECS)
        )

    try:
        ftp.login()

        files_itr = iter(get_filepaths_fn(ftp))
        filename = next(files_itr)
        retry_count = 0

        while True:
            try:
                retry_count += 1
                _log.debug('Next filename: %r', filename)

                def ftp_fetch(t):
                    """Fetch data to filename t"""
                    # https://bitbucket.org/logilab/pylint/issue/271/spurious-warning-w0640-issued-for-loop
                    # pylint: disable=cell-var-from-loop
                    _log.debug('Retrieving %r to %r', filename, t)
                    with open(t, 'wb') as f:
                        ftp.retrbinary('RETR ' + filename, f.write)
                    return True

                fetch_file(
                    'ftp://%s%s' % (hostname, filename),
                    ftp_fetch,
                    reporter,
                    os.path.basename(filename),
                    target_dir,
                    filename_transform=filename_transform,
                    override_existing=override_existing
                )
                filename = next(files_itr)
                retry_count = 0

            except (EOFError, ftplib.error_temp):
                # ftplib.error_temp represents a 4XX error by the server

                if retry_count >= retries:
                    _log.debug('Error fetching file. Reconnecting to ftp server...')
                    raise
                _log.debug('Error fetching file. Reconnecting to ftp server...')

                # Connection was closed; try to re-connect
                time.sleep(retry_delay)
                try:
                    ftp = ftplib.FTP(hostname, timeout=DEFAULT_SOCKET_TIMEOUT_SECS)
{'fraction_non_alphanumeric': 0.07970768184908225, 'fraction_numerical': 0.006118286879673691, 'mean_word_length': 1.6284055381866904, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1487965', 'n_tokens_mistral': 1663, 'n_tokens_neox': 1568, 'n_words': 414}