<commit_before>from __future__ import division, print_function
from multiprocessing import Pool
import argparse
import sys

import numpy as np
from cooler.io import CoolerAggregator
import cooler.ice
import cooler
import h5py


FACTOR = 2
TILESIZE = 256
N_CPU = 8


def set_postmortem_hook():
    import sys, traceback, ipdb
    def _excepthook(exc_type, value, tb):
        traceback.print_exception(exc_type, value, tb)
        print()
        ipdb.pm()
    sys.excepthook = _excepthook

set_postmortem_hook()


def main(infile, outfile, chunksize):
    c = cooler.Cooler(infile)
    binsize = c.info['bin-size']
    chromtable = c.chroms()[:]
    chromsizes = chromtable.set_index('name')['length']
    chroms = chromtable['name'].values
    lengths = chromtable['length'].values
    total_length = np.sum(chromsizes.values)
    n_tiles = total_length / binsize / TILESIZE
    n_zooms = int(np.ceil(np.log2(n_tiles)))

    print(
        "Copying base matrix to level {0} and producing {0} zoom levels starting from 0...".format(n_zooms),
        file=sys.stderr
    )

    # transfer base matrix
    with h5py.File(outfile, 'w') as dest, \
         h5py.File(infile, 'r') as src:

        zoomLevel = str(n_zooms)
        src.copy('/', dest, zoomLevel)

        print(zoomLevel, file=sys.stderr)


    # produce aggregations
    with h5py.File(outfile, 'r+') as f:
        grp = f[str(n_zooms)]
        c = cooler.Cooler(grp)
        binsize = cooler.info(grp)['bin-size']

        for i in range(n_zooms - 1, -1, -1):
            zoomLevel = str(i)

            # aggregate
            new_binsize = binsize * FACTOR
            new_bins = cooler.util.binnify(chromsizes, new_binsize)
     
            reader = CoolerAggregator(c, new_bins, chunksize)
            
            grp = f.create_group(zoomLevel)<commit_msg>Make note of the number of zoom levels<commit_after>from __future__ import division, print_function
from multiprocessing import Pool
import argparse
import sys

import numpy as np
from cooler.io import CoolerAggregator
import cooler.ice
import cooler
import h5py


FACTOR = 2
TILESIZE = 256
N_CPU = 8


def set_postmortem_hook():
    import sys, traceback, ipdb
    def _excepthook(exc_type, value, tb):
        traceback.print_exception(exc_type, value, tb)
        print()
        ipdb.pm()
    sys.excepthook = _excepthook

set_postmortem_hook()


def main(infile, outfile, chunksize):
    c = cooler.Cooler(infile)
    binsize = c.info['bin-size']
    chromtable = c.chroms()[:]
    chromsizes = chromtable.set_index('name')['length']
    chroms = chromtable['name'].values
    lengths = chromtable['length'].values
    total_length = np.sum(chromsizes.values)
    n_tiles = total_length / binsize / TILESIZE
    n_zooms = int(np.ceil(np.log2(n_tiles)))

    print("binsize:", binsize)
    print("total_length:", total_length)

    print(
        "Copying base matrix to level {0} and producing {0} zoom levels starting from 0...".format(n_zooms),
        file=sys.stderr
    )

    # transfer base matrix
    with h5py.File(outfile, 'w') as dest, \
         h5py.File(infile, 'r') as src:

        zoomLevel = str(n_zooms)
        src.copy('/', dest, zoomLevel)

        print(zoomLevel, file=sys.stderr)


    # produce aggregations
    with h5py.File(outfile, 'r+') as f:
        grp = f[str(n_zooms)]

        f.attrs[str(n_zooms)] = binsize
        f.attrs['max-zoom'] = n_zooms

        c = cooler.Cooler(grp)
        binsize = cooler.info(grp)['bin-size']
        print("n_zooms:", n_zooms)

        for i in range(n_zooms - 1, -1, -1):
            zoomLevel = str(i)

            # aggregate
            new_binsize = binsize * FACTOR
            new_bins = cooler.util.binnify(chromsizes, new_binsize)
     
            reader = CoolerAggregator(c, new_bins, chunksize)
            
            grp = f.create_group(zoomLevel)
{'fraction_non_alphanumeric': 0.11151358344113842, 'fraction_numerical': 0.008279430789133247, 'mean_word_length': 2.667931688804554, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2741486', 'n_tokens_mistral': 1383, 'n_tokens_neox': 1303, 'n_words': 306}