<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
This bot downloads dump from dumps.wikimedia.org.

This script supports the following command line parameters:

    -filename:#     The name of the file (e.g. abstract.xml)

    -storepath:#    The stored file's path.

    -revision:#     The revision date of the dump (default to `latest`)
                    formatted as YYYYMMDD.

"""
#
# (C) Pywikibot team, 2017
# (C) <NAME>, 2017
#
# Distributed under the terms of the MIT license.
#
from __future__ import absolute_import, division, unicode_literals

import binascii

import os.path
import sys

from os import remove, symlink, urandom

try:
    from os import replace
except ImportError:   # py2
    if sys.platform == 'win32':
        import os

        def replace(src, dst):
            try:
                os.rename(src, dst)
            except OSError:
                remove(dst)
                os.rename(src, dst)
    else:
        from os import rename as replace

import pywikibot

from pywikibot import Bot

from pywikibot.comms.http import fetch


class DownloadDumpBot(Bot):

    """Download dump bot."""

    availableOptions = {
        'wikiname': '',
        'filename': '',
        'storepath': './',
        'revision': 'latest',
    }

    def __init__(self, **kwargs):
        """Constructor."""
        super(DownloadDumpBot, self).__init__(**kwargs)

    def get_dump_name(self, db_name, typ):
        """Check if dump file exists locally in a Toolforge server."""
        db_path = '/public/dumps/public/{0}/'.format(db_name)
        if os.path.isdir(db_path):
            dates = map(int, os.listdir(db_path))
            dates = sorted(dates, reverse=True)
            for date in dates:
                dump_filepath = ('/public/dumps/public/{0}/{1}/{2}-{3}-{4}'
                                 .format(db_name, date, db_name, date, typ))
                if os.path.isfile(dump_filepath):
                    return dump_filepath
        return None

    def run(self):
        """Run bot."""
        pywikibot.output('Downloading dump from ' + self.getOption('wikiname'))

        download_filename = '{wiki_name}-{revision}-{filename}'.format(
            wiki_name=self.getOption('wikiname'),
            revision=self.getOption('revision'),
            filename=self.getOption('filename')
        )
        temp_filename = download_filename + '-' + \
            binascii.b2a_hex(urandom(8)).decode('ascii') + '.part'

        file_final_storepath = os.path.join(
            self.getOption('storepath'), download_filename)
        file_current_storepath = os.path.join(
            self.getOption('storepath'), temp_filename)

        # https://wikitech.wikimedia.org/wiki/Help:Toolforge#Dumps
        toolforge_dump_filepath = self.get_dump_name(
            self.getOption('wikiname'), self.getOption('filename'))

        # First iteration for atomic download with temporary file
        # Second iteration for fallback non-atomic download
        for non_atomic in range(2):
            try:
                if toolforge_dump_filepath:
                    pywikibot.output('Symlinking file from ' +
                                     toolforge_dump_filepath)
                    if non_atomic:
                        if os.path.exists(file_final_storepath):
                            remove(file_final_storepath)
                    symlink(toolforge_dump_filepath, file_current_storepath)
                else:
                    url = 'https://dumps.wikimedia.org/{0}/{1}/{2}'.format(
                        self.getOption('wikiname'),
                        self.getOption('revision'),
                        download_filename)
                    pywikibot.output('Downloading file from ' + url)
                    response = fetch(url, stream=True)
                    if response.status == 200:
                        with open(file_current_storepath, 'wb') as result_file:
                            for data in response.data.iter_content(100 * 1024):
                                result_file.write(data)
                    elif response.status == 404:
                        pywikibot.output(
                            'File with name "{filename}", '
                            'from revision "{revision}", '
                            'and wiki "{wikiname}" ({url}) isn\'t '
                            'available in the Wikimedia Dumps'.format(
                                filename=self.getOption('filename'),<commit_msg>download_dump: Add a progress bar<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
This bot downloads dump from dumps.wikimedia.org.

This script supports the following command line parameters:

    -filename:#     The name of the file (e.g. abstract.xml)

    -storepath:#    The stored file's path.

    -revision:#     The revision date of the dump (default to `latest`)
                    formatted as YYYYMMDD.

"""
#
# (C) Pywikibot team, 2017
# (C) <NAME>, 2017
#
# Distributed under the terms of the MIT license.
#
from __future__ import absolute_import, division, unicode_literals

import binascii

import os.path
import sys

from os import remove, symlink, urandom

try:
    from os import replace
except ImportError:   # py2
    if sys.platform == 'win32':
        import os

        def replace(src, dst):
            try:
                os.rename(src, dst)
            except OSError:
                remove(dst)
                os.rename(src, dst)
    else:
        from os import rename as replace

import pywikibot

from pywikibot import Bot

from pywikibot.comms.http import fetch


class DownloadDumpBot(Bot):

    """Download dump bot."""

    availableOptions = {
        'wikiname': '',
        'filename': '',
        'storepath': './',
        'revision': 'latest',
    }

    def __init__(self, **kwargs):
        """Constructor."""
        super(DownloadDumpBot, self).__init__(**kwargs)

    def get_dump_name(self, db_name, typ):
        """Check if dump file exists locally in a Toolforge server."""
        db_path = '/public/dumps/public/{0}/'.format(db_name)
        if os.path.isdir(db_path):
            dates = map(int, os.listdir(db_path))
            dates = sorted(dates, reverse=True)
            for date in dates:
                dump_filepath = ('/public/dumps/public/{0}/{1}/{2}-{3}-{4}'
                                 .format(db_name, date, db_name, date, typ))
                if os.path.isfile(dump_filepath):
                    return dump_filepath
        return None

    def run(self):
        """Run bot."""
        def convert_from_bytes(bytes):
            for unit in ['B', 'K', 'M', 'G', 'T']:
                if abs(bytes) < 1024:
                    return str(bytes) + unit
                bytes = float(format(
                    bytes / 1024.0, '.2f'))
            return str(bytes) + 'P'

        pywikibot.output('Downloading dump from ' + self.getOption('wikiname'))

        download_filename = '{wiki_name}-{revision}-{filename}'.format(
            wiki_name=self.getOption('wikiname'),
            revision=self.getOption('revision'),
            filename=self.getOption('filename')
        )
        temp_filename = download_filename + '-' + \
            binascii.b2a_hex(urandom(8)).decode('ascii') + '.part'

        file_final_storepath = os.path.join(
            self.getOption('storepath'), download_filename)
        file_current_storepath = os.path.join(
            self.getOption('storepath'), temp_filename)

        # https://wikitech.wikimedia.org/wiki/Help:Toolforge#Dumps
        toolforge_dump_filepath = self.get_dump_name(
            self.getOption('wikiname'), self.getOption('filename'))

        # First iteration for atomic download with temporary file
        # Second iteration for fallback non-atomic download
        for non_atomic in range(2):
            try:
                if toolforge_dump_filepath:
                    pywikibot.output('Symlinking file from ' +
                                     toolforge_dump_filepath)
                    if non_atomic:
                        if os.path.exists(file_final_storepath):
                            remove(file_final_storepath)
                    symlink(toolforge_dump_filepath, file_current_storepath)
                else:
                    url = 'https://dumps.wikimedia.org/{0}/{1}/{2}'.format(
                        self.getOption('wikiname'),
                        self.getOption('revision'),
                        download_filename)
                    pywikibot.output('Downloading file from ' + url)
                    response = fetch(url, stream=True)
                    if response.status == 200:
                        with open(file_current_storepath, 'wb') as result_file:
                            try:
                                total = int(response.response_headers[
                                    'content-length'])
                            except KeyError:
                                pywikibot.exception()
                                total = -1
                            downloaded = 0
                            parts = 50
                            display_string = ''

                            pywikibot.output('')
                            for data in response.data.iter_content(100 * 1024):
                                result_file.write(data)

                                if total > 0:
                                    downloaded += len(data)
                                    done = int(parts * downloaded / total)
                                    display = map(convert_from_bytes,
                                                  (downloaded, total))
                                    prior_display = display_string
                                    display_string = ('\r|{0}{1}|' +
                                                      ' ' * 5 +
                                                      '{2}/{3}').format(
                                        '=' * done,
                                        '-' * (parts - done),
                                        *display)
                                    # Add whitespace to cover up prior bar
                                    display_string += ' ' * (
                                        len(prior_display.rstrip()) -
                                        len(display_string.rstrip()))

                                    pywikibot.output(display_string,
                                                     newline=False)
                            pywikibot.output('')
                    elif response.status == 404:
                        pywikibot.output(
                            'File with name "{filename}", '
                            'from revision "{revision}", '
                            'and wiki "{wikiname}" ({url}) isn\'t '
                            'available in the Wikimedia Dumps'.format(
                                filename=self.getOption('filename'),
{'fraction_non_alphanumeric': 0.10278004165534728, 'fraction_numerical': 0.008512179661323916, 'mean_word_length': 1.4103011785246617, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 59, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3206892', 'n_tokens_mistral': 2969, 'n_tokens_neox': 2772, 'n_words': 650}