<commit_before># -*- coding: utf-8 -*-
#
# The internetarchive module is a Python/CLI interface to Archive.org.
#
# Copyright (C) 2012-2019 Internet Archive
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
internetarchive.item
~~~~~~~~~~~~~~~~~~~~

:copyright: (C) 2012-2019 by Internet Archive.
:license: AGPL 3, see LICENSE for more details.
"""
from __future__ import absolute_import, unicode_literals, print_function

import os
import sys
from fnmatch import fnmatch
from logging import getLogger
from time import sleep

try:
    from functools import total_ordering
except ImportError:
    from total_ordering import total_ordering
import json
from copy import deepcopy

from six import string_types
from six.moves import urllib
from requests import Response
from clint.textui import progress
from requests.exceptions import HTTPError

from internetarchive.utils import IdentifierListAsItems, get_md5, chunk_generator, \
    IterableToFileAdapter, iter_directory, recursive_file_count, norm_filepath
from internetarchive.files import File
from internetarchive.iarequest import MetadataRequest, S3Request
from internetarchive.utils import get_s3_xml_text, get_file_size, is_dir

log = getLogger(__name__)


@total_ordering
class BaseItem(object):
    EXCLUDED_ITEM_METADATA_KEYS = (u'workable_servers', u'server')

    def __init__(self, identifier=None, item_metadata=None):
        # Default attributes.
        self.identifier = identifier
        self.item_metadata = {} if not item_metadata else item_metadata
        self.exists = None

        # Archive.org metadata attributes.
        self.metadata = {}
        self.files = []
        self.created = None
        self.d1 = None
        self.d2 = None
        self.dir = None
        self.files_count = None
        self.item_size = None
        self.reviews = []
        self.server = None
        self.uniq = None
        self.updated = None
        self.tasks = None
        self.is_dark = None

        # Load item.
        self.load()

    def __repr__(self):
        return ('{0.__class__.__name__}(identifier={0.identifier!r}{notloaded})'.format(
            self, notloaded=', item_metadata={}' if not self.exists else ''))

    def load(self, item_metadata=None):
        if item_metadata:
            self.item_metadata = item_metadata

        self.exists = True if self.item_metadata else False

        for key in self.item_metadata:
            setattr(self, key, self.item_metadata[key])

        if not self.identifier:
            self.identifier = self.metadata.get('identifier')

        mc = self.metadata.get('collection', [])
        self.collection = IdentifierListAsItems(mc, self.session)

    def __eq__(self, other):
        return self.item_metadata == other.item_metadata or \
            (self.item_metadata.keys() == other.item_metadata.keys() and
             all(self.item_metadata[x] == other.item_metadata[x]
                 for x in self.item_metadata
                 if x not in self.EXCLUDED_ITEM_METADATA_KEYS))

    def __le__(self, other):
        return self.identifier <= other.identifier

    def __hash__(self):
        without_excluded_keys = dict(
            (k, v) for (k, v) in self.item_metadata.items()
            if k not in self.EXCLUDED_ITEM_METADATA_KEYS)
        return hash(json.dumps(without_excluded_keys,
                               sort_keys=True, check_circular=False))


class Item(BaseItem):
    """This class represents an archive.org item. Generally this class
    should not be used directly, but rather via the
    ``internetarchive.get_item()`` function::

        >>> from internetarchive import get_item
        >>> item = get_item('stairs')
        >>> print(item.metadata)

    Or to modify the metadata for an item::

        >>> metadata = dict(title='The Stairs')
        >>> item.modify_metadata(metadata)
        >>> print(item.metadata['title'])
        'The Stairs'

    This class also uses IA's S3-like interface to upload files to an
    item. You need to supply your IAS3 credentials in environment
    variables in order to upload::

        >>> item.upload('myfile.tar', access_key='Y6oUrAcCEs4sK8ey',
        ...                           secret_key='youRSECRETKEYzZzZ')
        True

    You can retrieve S3 keys here: `https://archive.org/account/s3.php
    <https://archive.org/account/s3.php>`__
    """

    def __init__(self, archive_session, identifier, item_metadata=None):
        """
        :type archive_session: :class:`ArchiveSession <ArchiveSession>`

        :type identifier: str
        :param identifier: The globally unique Archive.org identifier for this item.

                           An identifier is composed of any unique combination of
                           alphanumeric characters, underscore ( _ ) and dash ( - ). While
                           there are no official limits it is strongly suggested that they
                           be between 5 and 80 characters in length. Identifiers must be
                           unique across the entirety of Internet Archive, not simply
                           unique within a single collection.

                           Once defined an identifier can not be changed. It will travel
                           with the item or object and is involved in every manner of
                           accessing or referring to the item.

        :type item_metadata: dict
        :param item_metadata: (optional) The Archive.org item metadata used to initialize
                              this item.  If no item metadata is provided, it will be
                              retrieved from Archive.org using the provided identifier.
        """
        self.session = archive_session
        super(Item, self).__init__(identifier, item_metadata)

        self.urls = Item.URLs(self)

        if self.metadata.get('title'):
            # A copyable link to the item, in MediaWiki format
            self.wikilink = '* [{0.urls.details} {0.identifier}] ' \
                            '-- {0.metadata[title]}'.format(self)

    class URLs:
        def __init__(self, itm_obj):
            self._itm_obj = itm_obj
            self._paths = []
            self._make_URL('details')
            self._make_URL('metadata')
            self._make_URL('download')
            self._make_URL('history')
            self._make_URL('edit')
            self._make_URL('editxml')
            self._make_URL('manage')
            if self._itm_obj.metadata.get('mediatype') == 'collection':
                self._make_tab_URL('about')
                self._make_tab_URL('collection')

        def _make_tab_URL(self, tab):
            """Make URLs for the separate tabs of Collections details page."""
            self._make_URL(tab, self.details + "&tab={tab}".format(tab=tab))

        DEFAULT_URL_FORMAT = '{0.session.protocol}//archive.org/{path}/{0.identifier}'

        def _make_URL(self, path, url_format=DEFAULT_URL_FORMAT):
            setattr(self, path, url_format.format(self._itm_obj, path=path))
            self._paths.append(path)

        def __str__(self):
            return "URLs ({1}) for {0.identifier}" \
                   .format(self._itm_obj, ', '.join(self._paths))

    def refresh(self, item_metadata=None, **kwargs):
        if not item_metadata:
            item_metadata = self.session.get_metadata(self.identifier, **kwargs)
        self.load(item_metadata)

    def get_file(self, file_name, file_metadata=None):
        """Get a :class:`File <File>` object for the named file.

        :rtype: :class:`internetarchive.File <File>`
        :returns: An :class:`internetarchive.File <File>` object.

        :type file_metadata: dict
        :param file_metadata: (optional) a dict of metadata for the
                              given fille.
        """
        return File(self, file_name, file_metadata)

    def get_files(self, files=None, formats=None, glob_pattern=None, on_the_fly=None):
        files = [] if not files else files
        formats = [] if not formats else formats
        on_the_fly = False if not on_the_fly else True

        if not isinstance(files, (list, tuple, set)):
            files = [files]
        if not isinstance(formats, (list, tuple, set)):
            formats = [formats]

        item_files = deepcopy(self.files)
        # Add support for on-the-fly files (e.g. EPUB).
        if on_the_fly:
            otf_files = [
                '{0}.epub'.format(self.identifier),
                '{0}.mobi'.format(self.identifier),
                '{0}_daisy.zip'.format(self.identifier),
                '{0}_archive_marc.xml'.format(self.identifier),
            ]
            for f in otf_files:
                item_files.append(dict(name=f, otf=True))

        if not any(k for k in [files, formats, glob_pattern]):
            for f in item_files:
                yield self.get_file(f.get('name'), file_metadata=f)

        for f in item_files:
            if f.get('name') in files:
                yield self.get_file(f.get('name'))
            elif f.get('format') in formats:
                yield self.get_file(f.get('name'))
            elif glob_pattern:
                if not isinstance(glob_pattern, list):
                    patterns = glob_pattern.split('|')
                else:
                    patterns = glob_pattern
                for p in patterns:
                    if fnmatch(f.get('name', ''), p):
                        yield self.get_file(f.get('name'))

    def download(self,
                 files=None,
                 formats=None,
                 glob_pattern=None,
                 dry_run=None,
                 verbose=None,
                 silent=None,
                 ignore_existing=None,
                 checksum=None,
                 destdir=None,
                 no_directory=None,
                 retries=None,
                 item_index=None,
                 ignore_errors=None,
                 on_the_fly=None,
                 return_responses=None,
                 no_change_timestamp=None,
                 params=None):
        """Download files from an item.

        :param files: (optional) Only download files matching given file names.

        :type formats: str
        :param formats: (optional) Only download files matching the given
                        Formats.

        :type glob_pattern: str
        :param glob_pattern: (optional) Only download files matching the given
                             glob pattern.

        :type dry_run: bool
        :param dry_run: (optional) Output download URLs to stdout, don't
                        download anything.

        :type verbose: bool
        :param verbose: (optional) Turn on verbose output.

        :type silent: bool
        :param silent: (optional) Suppress all output.

        :type ignore_existing: bool
        :param ignore_existing: (optional) Skip files that already exist
                                locally.

        :type checksum: bool
        :param checksum: (optional) Skip downloading file based on checksum.

        :type destdir: str
        :param destdir: (optional) The directory to download files to.

        :type no_directory: bool
        :param no_directory: (optional) Download files to current working
                             directory rather than creating an item directory.

        :type retries: int
        :param retries: (optional) The number of times to retry on failed
                        requests.

        :type item_index: int
        :param item_index: (optional) The index of the item for displaying
                           progress in bulk downloads.

        :type ignore_errors: bool
        :param ignore_errors: (optional) Don't fail if a single file fails to
                              download, continue to download other files.

        :type on_the_fly: bool
        :param on_the_fly: (optional) Download on-the-fly files (i.e. derivative EPUB,
                           MOBI, DAISY files).

        :type return_responses: bool
        :param return_responses: (optional) Rather than downloading files to disk, return
                                 a list of response objects.

        :type no_change_timestamp: bool
        :param no_change_timestamp: (optional) If True, leave the time stamp as the
                                    current time instead of changing it to that given in
                                    the original archive.

        :type params: dict
        :param params: (optional) URL parameters to send with
                       download request (e.g. `cnt=0`).

        :rtype: bool
        :returns: True if if all files have been downloaded successfully.
        """
        dry_run = False if dry_run is None else dry_run
        verbose = False if verbose is None else verbose
        silent = False if silent is None else silent
        ignore_existing = False if ignore_existing is None else ignore_existing
        ignore_errors = False if not ignore_errors else ignore_errors
        checksum = False if checksum is None else checksum
        no_directory = False if no_directory is None else no_directory
        return_responses = False if not return_responses else True
        no_change_timestamp = False if not no_change_timestamp else no_change_timestamp
        params = None if not params else params

        if not dry_run:
            if item_index and verbose is True:
                print('{0} ({1}):'.format(self.identifier, item_index))
            elif item_index and silent is False:
                print('{0} ({1}): '.format(self.identifier, item_index), end='')
            elif item_index is None and verbose is True:
                print('{0}:'.format(self.identifier))
            elif item_index is None and silent is False:
                print(self.identifier, end=': ')
            sys.stdout.flush()

        if self.is_dark is True:
            msg = 'skipping {0}, item is dark'.format(self.identifier)
            log.warning(msg)
            if verbose:
                print(' ' + msg)
            elif silent is False:
                print(msg)
            return
        elif self.metadata == {}:
            msg = 'skipping {0}, item does not exist.'.format(self.identifier)
            log.warning(msg)
            if verbose:
                print(' ' + msg)
            elif silent is False:
                print(msg)
            return

        if files:
            files = self.get_files(files, on_the_fly=on_the_fly)
        else:
            files = self.get_files(on_the_fly=on_the_fly)
        if formats:
            files = self.get_files(formats=formats, on_the_fly=on_the_fly)
        if glob_pattern:
            files = self.get_files(glob_pattern=glob_pattern, on_the_fly=on_the_fly)

        if not files:
            msg = 'skipping {0}, no matching files found.'.format(self.identifier)
            log.info(msg)
            if verbose:
                print(' ' + msg)
            elif silent is False:
                print(msg, end='')

        errors = list()
        responses = list()
        for f in files:
            if no_directory:
                path = f.name
            else:
                path = os.path.join(self.identifier, f.name)
            if dry_run:
                print(f.url)
                continue
            r = f.download(path, verbose, silent, ignore_existing, checksum, destdir,
                           retries, ignore_errors, None, return_responses,
                           no_change_timestamp, params)
            if return_responses:
                responses.append(r)
            if r is False:
                errors.append(f.name)
        if silent is False and verbose is False and dry_run is False:
            if errors:
                print(' - errors')
            else:
                print(' - success')
        if return_responses:
            return responses
        else:
            return errors

    def modify_metadata(self, metadata,<commit_msg>Display 'download failed' instead of 'success' when download fails<commit_after># -*- coding: utf-8 -*-
#
# The internetarchive module is a Python/CLI interface to Archive.org.
#
# Copyright (C) 2012-2019 Internet Archive
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
internetarchive.item
~~~~~~~~~~~~~~~~~~~~

:copyright: (C) 2012-2019 by Internet Archive.
:license: AGPL 3, see LICENSE for more details.
"""
from __future__ import absolute_import, unicode_literals, print_function

import os
import sys
from fnmatch import fnmatch
from logging import getLogger
from time import sleep

try:
    from functools import total_ordering
except ImportError:
    from total_ordering import total_ordering
import json
from copy import deepcopy

from six import string_types
from six.moves import urllib
from requests import Response
from clint.textui import progress
from requests.exceptions import HTTPError

from internetarchive.utils import IdentifierListAsItems, get_md5, chunk_generator, \
    IterableToFileAdapter, iter_directory, recursive_file_count, norm_filepath
from internetarchive.files import File
from internetarchive.iarequest import MetadataRequest, S3Request
from internetarchive.utils import get_s3_xml_text, get_file_size, is_dir

log = getLogger(__name__)


@total_ordering
class BaseItem(object):
    EXCLUDED_ITEM_METADATA_KEYS = (u'workable_servers', u'server')

    def __init__(self, identifier=None, item_metadata=None):
        # Default attributes.
        self.identifier = identifier
        self.item_metadata = {} if not item_metadata else item_metadata
        self.exists = None

        # Archive.org metadata attributes.
        self.metadata = {}
        self.files = []
        self.created = None
        self.d1 = None
        self.d2 = None
        self.dir = None
        self.files_count = None
        self.item_size = None
        self.reviews = []
        self.server = None
        self.uniq = None
        self.updated = None
        self.tasks = None
        self.is_dark = None

        # Load item.
        self.load()

    def __repr__(self):
        return ('{0.__class__.__name__}(identifier={0.identifier!r}{notloaded})'.format(
            self, notloaded=', item_metadata={}' if not self.exists else ''))

    def load(self, item_metadata=None):
        if item_metadata:
            self.item_metadata = item_metadata

        self.exists = True if self.item_metadata else False

        for key in self.item_metadata:
            setattr(self, key, self.item_metadata[key])

        if not self.identifier:
            self.identifier = self.metadata.get('identifier')

        mc = self.metadata.get('collection', [])
        self.collection = IdentifierListAsItems(mc, self.session)

    def __eq__(self, other):
        return self.item_metadata == other.item_metadata or \
            (self.item_metadata.keys() == other.item_metadata.keys() and
             all(self.item_metadata[x] == other.item_metadata[x]
                 for x in self.item_metadata
                 if x not in self.EXCLUDED_ITEM_METADATA_KEYS))

    def __le__(self, other):
        return self.identifier <= other.identifier

    def __hash__(self):
        without_excluded_keys = dict(
            (k, v) for (k, v) in self.item_metadata.items()
            if k not in self.EXCLUDED_ITEM_METADATA_KEYS)
        return hash(json.dumps(without_excluded_keys,
                               sort_keys=True, check_circular=False))


class Item(BaseItem):
    """This class represents an archive.org item. Generally this class
    should not be used directly, but rather via the
    ``internetarchive.get_item()`` function::

        >>> from internetarchive import get_item
        >>> item = get_item('stairs')
        >>> print(item.metadata)

    Or to modify the metadata for an item::

        >>> metadata = dict(title='The Stairs')
        >>> item.modify_metadata(metadata)
        >>> print(item.metadata['title'])
        'The Stairs'

    This class also uses IA's S3-like interface to upload files to an
    item. You need to supply your IAS3 credentials in environment
    variables in order to upload::

        >>> item.upload('myfile.tar', access_key='Y6oUrAcCEs4sK8ey',
        ...                           secret_key='youRSECRETKEYzZzZ')
        True

    You can retrieve S3 keys here: `https://archive.org/account/s3.php
    <https://archive.org/account/s3.php>`__
    """

    def __init__(self, archive_session, identifier, item_metadata=None):
        """
        :type archive_session: :class:`ArchiveSession <ArchiveSession>`

        :type identifier: str
        :param identifier: The globally unique Archive.org identifier for this item.

                           An identifier is composed of any unique combination of
                           alphanumeric characters, underscore ( _ ) and dash ( - ). While
                           there are no official limits it is strongly suggested that they
                           be between 5 and 80 characters in length. Identifiers must be
                           unique across the entirety of Internet Archive, not simply
                           unique within a single collection.

                           Once defined an identifier can not be changed. It will travel
                           with the item or object and is involved in every manner of
                           accessing or referring to the item.

        :type item_metadata: dict
        :param item_metadata: (optional) The Archive.org item metadata used to initialize
                              this item.  If no item metadata is provided, it will be
                              retrieved from Archive.org using the provided identifier.
        """
        self.session = archive_session
        super(Item, self).__init__(identifier, item_metadata)

        self.urls = Item.URLs(self)

        if self.metadata.get('title'):
            # A copyable link to the item, in MediaWiki format
            self.wikilink = '* [{0.urls.details} {0.identifier}] ' \
                            '-- {0.metadata[title]}'.format(self)

    class URLs:
        def __init__(self, itm_obj):
            self._itm_obj = itm_obj
            self._paths = []
            self._make_URL('details')
            self._make_URL('metadata')
            self._make_URL('download')
            self._make_URL('history')
            self._make_URL('edit')
            self._make_URL('editxml')
            self._make_URL('manage')
            if self._itm_obj.metadata.get('mediatype') == 'collection':
                self._make_tab_URL('about')
                self._make_tab_URL('collection')

        def _make_tab_URL(self, tab):
            """Make URLs for the separate tabs of Collections details page."""
            self._make_URL(tab, self.details + "&tab={tab}".format(tab=tab))

        DEFAULT_URL_FORMAT = '{0.session.protocol}//archive.org/{path}/{0.identifier}'

        def _make_URL(self, path, url_format=DEFAULT_URL_FORMAT):
            setattr(self, path, url_format.format(self._itm_obj, path=path))
            self._paths.append(path)

        def __str__(self):
            return "URLs ({1}) for {0.identifier}" \
                   .format(self._itm_obj, ', '.join(self._paths))

    def refresh(self, item_metadata=None, **kwargs):
        if not item_metadata:
            item_metadata = self.session.get_metadata(self.identifier, **kwargs)
        self.load(item_metadata)

    def get_file(self, file_name, file_metadata=None):
        """Get a :class:`File <File>` object for the named file.

        :rtype: :class:`internetarchive.File <File>`
        :returns: An :class:`internetarchive.File <File>` object.

        :type file_metadata: dict
        :param file_metadata: (optional) a dict of metadata for the
                              given fille.
        """
        return File(self, file_name, file_metadata)

    def get_files(self, files=None, formats=None, glob_pattern=None, on_the_fly=None):
        files = [] if not files else files
        formats = [] if not formats else formats
        on_the_fly = False if not on_the_fly else True

        if not isinstance(files, (list, tuple, set)):
            files = [files]
        if not isinstance(formats, (list, tuple, set)):
            formats = [formats]

        item_files = deepcopy(self.files)
        # Add support for on-the-fly files (e.g. EPUB).
        if on_the_fly:
            otf_files = [
                '{0}.epub'.format(self.identifier),
                '{0}.mobi'.format(self.identifier),
                '{0}_daisy.zip'.format(self.identifier),
                '{0}_archive_marc.xml'.format(self.identifier),
            ]
            for f in otf_files:
                item_files.append(dict(name=f, otf=True))

        if not any(k for k in [files, formats, glob_pattern]):
            for f in item_files:
                yield self.get_file(f.get('name'), file_metadata=f)

        for f in item_files:
            if f.get('name') in files:
                yield self.get_file(f.get('name'))
            elif f.get('format') in formats:
                yield self.get_file(f.get('name'))
            elif glob_pattern:
                if not isinstance(glob_pattern, list):
                    patterns = glob_pattern.split('|')
                else:
                    patterns = glob_pattern
                for p in patterns:
                    if fnmatch(f.get('name', ''), p):
                        yield self.get_file(f.get('name'))

    def download(self,
                 files=None,
                 formats=None,
                 glob_pattern=None,
                 dry_run=None,
                 verbose=None,
                 silent=None,
                 ignore_existing=None,
                 checksum=None,
                 destdir=None,
                 no_directory=None,
                 retries=None,
                 item_index=None,
                 ignore_errors=None,
                 on_the_fly=None,
                 return_responses=None,
                 no_change_timestamp=None,
                 params=None):
        """Download files from an item.

        :param files: (optional) Only download files matching given file names.

        :type formats: str
        :param formats: (optional) Only download files matching the given
                        Formats.

        :type glob_pattern: str
        :param glob_pattern: (optional) Only download files matching the given
                             glob pattern.

        :type dry_run: bool
        :param dry_run: (optional) Output download URLs to stdout, don't
                        download anything.

        :type verbose: bool
        :param verbose: (optional) Turn on verbose output.

        :type silent: bool
        :param silent: (optional) Suppress all output.

        :type ignore_existing: bool
        :param ignore_existing: (optional) Skip files that already exist
                                locally.

        :type checksum: bool
        :param checksum: (optional) Skip downloading file based on checksum.

        :type destdir: str
        :param destdir: (optional) The directory to download files to.

        :type no_directory: bool
        :param no_directory: (optional) Download files to current working
                             directory rather than creating an item directory.

        :type retries: int
        :param retries: (optional) The number of times to retry on failed
                        requests.

        :type item_index: int
        :param item_index: (optional) The index of the item for displaying
                           progress in bulk downloads.

        :type ignore_errors: bool
        :param ignore_errors: (optional) Don't fail if a single file fails to
                              download, continue to download other files.

        :type on_the_fly: bool
        :param on_the_fly: (optional) Download on-the-fly files (i.e. derivative EPUB,
                           MOBI, DAISY files).

        :type return_responses: bool
        :param return_responses: (optional) Rather than downloading files to disk, return
                                 a list of response objects.

        :type no_change_timestamp: bool
        :param no_change_timestamp: (optional) If True, leave the time stamp as the
                                    current time instead of changing it to that given in
                                    the original archive.

        :type params: dict
        :param params: (optional) URL parameters to send with
                       download request (e.g. `cnt=0`).

        :rtype: bool
        :returns: True if if all files have been downloaded successfully.
        """
        dry_run = False if dry_run is None else dry_run
        verbose = False if verbose is None else verbose
        silent = False if silent is None else silent
        ignore_existing = False if ignore_existing is None else ignore_existing
        ignore_errors = False if not ignore_errors else ignore_errors
        checksum = False if checksum is None else checksum
        no_directory = False if no_directory is None else no_directory
        return_responses = False if not return_responses else True
        no_change_timestamp = False if not no_change_timestamp else no_change_timestamp
        params = None if not params else params

        if not dry_run:
            if item_index and verbose is True:
                print('{0} ({1}):'.format(self.identifier, item_index))
            elif item_index and silent is False:
                print('{0} ({1}): '.format(self.identifier, item_index), end='')
            elif item_index is None and verbose is True:
                print('{0}:'.format(self.identifier))
            elif item_index is None and silent is False:
                print(self.identifier, end=': ')
            sys.stdout.flush()

        if self.is_dark is True:
            msg = 'skipping {0}, item is dark'.format(self.identifier)
            log.warning(msg)
            if verbose:
                print(' ' + msg)
            elif silent is False:
                print(msg)
            return
        elif self.metadata == {}:
            msg = 'skipping {0}, item does not exist.'.format(self.identifier)
            log.warning(msg)
            if verbose:
                print(' ' + msg)
            elif silent is False:
                print(msg)
            return

        if files:
            files = self.get_files(files, on_the_fly=on_the_fly)
        else:
            files = self.get_files(on_the_fly=on_the_fly)
        if formats:
            files = self.get_files(formats=formats, on_the_fly=on_the_fly)
        if glob_pattern:
            files = self.get_files(glob_pattern=glob_pattern, on_the_fly=on_the_fly)

        if not files:
            msg = 'skipping {0}, no matching files found.'.format(self.identifier)
            log.info(msg)
            if verbose:
                print(' ' + msg)
            elif silent is False:
                print(msg, end='')

        errors = list()
        downloaded = 0
        responses = list()

        for f in files:
            if no_directory:
                path = f.name
            else:
                path = os.path.join(self.identifier, f.name)
            if dry_run:
                print(f.url)
                continue
            r = f.download(path, verbose, silent, ignore_existing, checksum, destdir,
                           retries, ignore_errors, None, return_responses,
                           no_change_timestamp, params)
            if return_responses:
                responses.append(r)

            if r is False:
                errors.append(f.name)
            else:
                downloaded += 1

        if silent is False and verbose is False and dry_run is False:
            if errors:
                print(' - errors')
            elif not downloaded:
                print(' - download failed')
            else:
                print(' - success')

        if return_responses:
            return responses
        else:
            return errors

    def modify_metadata(self, metadata,
{'fraction_non_alphanumeric': 0.08382804808925748, 'fraction_numerical': 0.0034605172876704155, 'mean_word_length': 1.9276855895196507, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 57, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 52, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1515225', 'n_tokens_mistral': 8950, 'n_tokens_neox': 8600, 'n_words': 2770}