<commit_before>import os
import tarfile
import importlib
import pandas as pd

from catalyst import get_calendar

from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader
from catalyst.data.minute_bars import BcolzMinuteBarMetadata
from catalyst.exchange.bundle_utils import get_df_from_arrays, get_bcolz_chunk

import matplotlib
import matplotlib.pyplot as plt
from matplotlib.finance import candlestick2_ohlc
from matplotlib.finance import volume_overlay
import matplotlib.ticker as ticker

from catalyst.exchange.factory import get_exchange

EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
exchanges = dict((e, getattr(importlib.import_module(
    'catalyst.exchange.{0}.{0}'.format(e)), e.capitalize()))
                 for e in EXCHANGE_NAMES)


class ValidateChunks(object):
    def __init__(self):
        self.columns = ['open', 'high', 'low', 'close', 'volume']

    def chunk_to_df(self, exchange_name, symbol, data_frequency, period):

        exchange = get_exchange(exchange_name)
        asset = exchange.get_asset(symbol)

        filename = get_bcolz_chunk(
            exchange_name=exchange_name,
            symbol=symbol,
            data_frequency=data_frequency,
            period=period
        )

        reader = BcolzExchangeBarReader(rootdir=filename,
                                        data_frequency=data_frequency)

        # metadata = BcolzMinuteBarMetadata.read(filename)

        start = reader.first_trading_day
        end = reader.last_available_dt

        if data_frequency == 'daily':
            end = end - pd.Timedelta(hours=23, minutes=59)

        print start, end, data_frequency

        arrays = reader.load_raw_arrays(self.columns, start, end,
                                        [asset.sid, ])

        bundle = ExchangeBundle(exchange_name)

        periods = bundle.get_calendar_periods_range(
            start, end, data_frequency<commit_msg>Python version 3 compatible<commit_after>import os
import tarfile
import importlib
import pandas as pd

from catalyst import get_calendar

from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader
from catalyst.data.minute_bars import BcolzMinuteBarMetadata
from catalyst.exchange.bundle_utils import get_df_from_arrays, get_bcolz_chunk

import matplotlib
import matplotlib.pyplot as plt
from matplotlib.finance import candlestick2_ohlc
from matplotlib.finance import volume_overlay
import matplotlib.ticker as ticker

from catalyst.exchange.factory import get_exchange

EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
exchanges = dict((e, getattr(importlib.import_module(
    'catalyst.exchange.{0}.{0}'.format(e)), e.capitalize()))
                 for e in EXCHANGE_NAMES)


class ValidateChunks(object):
    def __init__(self):
        self.columns = ['open', 'high', 'low', 'close', 'volume']

    def chunk_to_df(self, exchange_name, symbol, data_frequency, period):

        exchange = get_exchange(exchange_name)
        asset = exchange.get_asset(symbol)

        filename = get_bcolz_chunk(
            exchange_name=exchange_name,
            symbol=symbol,
            data_frequency=data_frequency,
            period=period
        )

        reader = BcolzExchangeBarReader(rootdir=filename,
                                        data_frequency=data_frequency)

        # metadata = BcolzMinuteBarMetadata.read(filename)

        start = reader.first_trading_day
        end = reader.last_available_dt

        if data_frequency == 'daily':
            end = end - pd.Timedelta(hours=23, minutes=59)

        print(start, end, data_frequency)

        arrays = reader.load_raw_arrays(self.columns, start, end,
                                        [asset.sid, ])

        bundle = ExchangeBundle(exchange_name)

        periods = bundle.get_calendar_periods_range(
            start, end, data_frequency
{'fraction_non_alphanumeric': 0.07786988193921125, 'fraction_numerical': 0.0037678975131876413, 'mean_word_length': 3.100926879505664, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2999203', 'n_tokens_mistral': 1188, 'n_tokens_neox': 1109, 'n_words': 249}