<commit_before>import tarfile
import shutil

import requests
from datetime import timedelta, datetime
import os
from logging import Logger
import pandas as pd
import numpy as np

import pytz

from catalyst.data.bundles import from_bundle_ingest_dirname
from catalyst.data.bundles.core import download_without_progress
from catalyst.exchange.exchange_errors import ApiCandlesError
from catalyst.exchange.exchange_utils import get_exchange_bundles_folder
from catalyst.utils.deprecate import deprecated
from catalyst.utils.paths import data_path

log = Logger('test_exchange_bundle')

EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
API_URL = 'http://data.enigma.co/api/v1'


def get_date_from_ms(ms):
    return datetime.fromtimestamp(ms / 1000.0)


def get_seconds_from_date(date):
    epoch = datetime.utcfromtimestamp(0)
    epoch = epoch.replace(tzinfo=pytz.UTC)

    return int((date - epoch).total_seconds())


def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
    """
    Download and extract a bcolz bundle.

    :param exchange_name:
    :param symbol:
    :param data_frequency:
    :param period:
    :return:

    Note:
        Filename: bitfinex-daily-neo_eth-2017-10.tar.gz
    """

    root = get_exchange_bundles_folder(exchange_name)
    name = '{exchange}-{frequency}-{symbol}-{period}'.format(
        exchange=exchange_name,
        frequency=data_frequency,
        symbol=symbol,
        period=period
    )
    path = os.path.join(root, name)

    if not os.path.isdir(path):
        url = 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/' \
              'exchange-{exchange}/{name}.tar.gz'.format(
            exchange=exchange_name,
            name=name
        )

        bytes = download_without_progress(url)
        with tarfile.open('r', fileobj=bytes) as tar:
            tar.extractall(path)

    return path


def get_delta(periods, data_frequency):
    return timedelta(minutes=periods) \
        if data_frequency == 'minute' else timedelta(days=periods)


def get_periods(start_dt, end_dt, data_frequency):
    delta = end_dt - start_dt

    if data_frequency == 'minute':
        delta_periods = delta.total_seconds() / 60

    elif data_frequency == 'daily':
        delta_periods = delta.total_seconds() / 60 / 60 / 24

    else:
        raise ValueError('frequency not supported')

    return int(delta_periods)


def get_start_dt(end_dt, bar_count, data_frequency):
    periods = bar_count - 1
    if periods > 1:
        delta = get_delta(periods, data_frequency)
        start_dt = end_dt - delta
    else:
        start_dt = end_dt

    return start_dt


def get_ffill_candles(candles, bar_count, end_dt, data_frequency,
                      previous_candle=None):
    """
    Create candles for each period of the specified range, forward-filling
    missing candles with the previous value.

    :param candles:
    :param bar_count:
    :param end_dt:
    :param data_frequency:
    :param previous_candle:

    :return:<commit_msg>Fixes a start date issue<commit_after>import tarfile
import shutil

import requests
from datetime import timedelta, datetime
import os
from logging import Logger
import pandas as pd
import numpy as np

import pytz

from catalyst.data.bundles import from_bundle_ingest_dirname
from catalyst.data.bundles.core import download_without_progress
from catalyst.exchange.exchange_errors import ApiCandlesError
from catalyst.exchange.exchange_utils import get_exchange_bundles_folder
from catalyst.utils.deprecate import deprecated
from catalyst.utils.paths import data_path

log = Logger('test_exchange_bundle')

EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
API_URL = 'http://data.enigma.co/api/v1'


def get_date_from_ms(ms):
    return datetime.fromtimestamp(ms / 1000.0)


def get_seconds_from_date(date):
    epoch = datetime.utcfromtimestamp(0)
    epoch = epoch.replace(tzinfo=pytz.UTC)

    return int((date - epoch).total_seconds())


def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
    """
    Download and extract a bcolz bundle.

    :param exchange_name:
    :param symbol:
    :param data_frequency:
    :param period:
    :return:

    Note:
        Filename: bitfinex-daily-neo_eth-2017-10.tar.gz
    """

    root = get_exchange_bundles_folder(exchange_name)
    name = '{exchange}-{frequency}-{symbol}-{period}'.format(
        exchange=exchange_name,
        frequency=data_frequency,
        symbol=symbol,
        period=period
    )
    path = os.path.join(root, name)

    if not os.path.isdir(path):
        url = 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/' \
              'exchange-{exchange}/{name}.tar.gz'.format(
            exchange=exchange_name,
            name=name
        )

        bytes = download_without_progress(url)
        with tarfile.open('r', fileobj=bytes) as tar:
            tar.extractall(path)

    return path


def get_delta(periods, data_frequency):
    return timedelta(minutes=periods) \
        if data_frequency == 'minute' else timedelta(days=periods)


def get_periods(start_dt, end_dt, data_frequency):
    delta = end_dt - start_dt

    if data_frequency == 'minute':
        delta_periods = delta.total_seconds() / 60

    elif data_frequency == 'daily':
        delta_periods = delta.total_seconds() / 60 / 60 / 24

    else:
        raise ValueError('frequency not supported')

    return int(delta_periods)


def get_start_dt(end_dt, bar_count, data_frequency):
    periods = bar_count
    if periods > 1:
        delta = get_delta(periods, data_frequency)
        start_dt = end_dt - delta
    else:
        start_dt = end_dt

    return start_dt


def get_ffill_candles(candles, bar_count, end_dt, data_frequency,
                      previous_candle=None):
    """
    Create candles for each period of the specified range, forward-filling
    missing candles with the previous value.

    :param candles:
    :param bar_count:
    :param end_dt:
    :param data_frequency:
    :param previous_candle:

    :return:
{'fraction_non_alphanumeric': 0.09214677071226964, 'fraction_numerical': 0.007803420222480492, 'mean_word_length': 3.2482369534555713, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4491585', 'n_tokens_mistral': 1987, 'n_tokens_neox': 1845, 'n_words': 436}