# Path: wikidat/utils/misc.py
# SUFFIXES = {1000: ['KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'],
#             1024: ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']}
# def hfile_size(size, kb_1024_bytes=True):


from bs4 import BeautifulSoup
from wikidat.utils import misc
import multiprocessing as mp
import itertools
import requests
import re
import os
import sys
import hashlib
import logging
        print("Paths in download: ", str(self.dump_paths))
        # Verify integrity of downloaded dumps
        try:
            self._verify(self.target_url)
        except DumpIntegrityError as e:
            print(e.msg)

        print("File integrity checked, no errors found.")
        # Return list of paths to dumpfiles for data extraction
        return self.dump_paths, dump_date

    def _get_file(self, dump_url, path_file):
        """
        Retrieve individual dump file from dump_url and save it in dump_dir
        Progress bar taken from:
        http://stackoverflow.com/questions/15644964/
        python-progress-bar-and-downloads
        """
        local_dir = os.path.split(path_file)[0]
        file_name = os.path.split(path_file)[1]
        file_url = "".join([self.mirror, dump_url])
        print("File URL is: %s" % (file_url))

        # Setup log file
        log_file = os.path.join(local_dir, "logs", file_name + ".log")
        logging.basicConfig(filename=log_file, level=logging.INFO)

        resp_file = requests.get(file_url, stream=True)
        meta_file_size = float(resp_file.headers.get('content-length'))
        log_size_msg = "Downloading: {0} - [Size: {1}]"

# Path: wikidat/utils/misc.py
# SUFFIXES = {1000: ['KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'],
#             1024: ['KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB']}
# def hfile_size(size, kb_1024_bytes=True):


# Path: wikidat/tasks/download.py
from bs4 import BeautifulSoup
from wikidat.utils import misc
import multiprocessing as mp
import itertools
import requests
import re
import os
import sys
import hashlib
import logging

        print("Paths in download: ", str(self.dump_paths))
        # Verify integrity of downloaded dumps
        try:
            self._verify(self.target_url)
        except DumpIntegrityError as e:
            print(e.msg)

        print("File integrity checked, no errors found.")
        # Return list of paths to dumpfiles for data extraction
        return self.dump_paths, dump_date

    def _get_file(self, dump_url, path_file):
        """
        Retrieve individual dump file from dump_url and save it in dump_dir
        Progress bar taken from:
        http://stackoverflow.com/questions/15644964/
        python-progress-bar-and-downloads
        """
        local_dir = os.path.split(path_file)[0]
        file_name = os.path.split(path_file)[1]
        file_url = "".join([self.mirror, dump_url])
        print("File URL is: %s" % (file_url))

        # Setup log file
        log_file = os.path.join(local_dir, "logs", file_name + ".log")
        logging.basicConfig(filename=log_file, level=logging.INFO)

        resp_file = requests.get(file_url, stream=True)
        meta_file_size = float(resp_file.headers.get('content-length'))
        log_size_msg = "Downloading: {0} - [Size: {1}]"

        print(log_size_msg.format(file_name, misc.hfile_size(meta_file_size)))