<commit_before>import pandas as pd
import numpy as np
import fastparquet
import itertools
import re
import schwimmbad

def perturb_catalog(filename, ra_offset=2, dec_offset=2, tag='fake'):
    # if ra_offset==0 and dec_offset==0:
    #     pass

    cat = pd.read_hdf(filename)
    new = cat.copy()
    for i, c in enumerate(cat.columns):
        if c=='coord_ra':
            new.loc[:, c] = cat[c] + np.deg2rad(ra_offset)
        elif c=='coord_dec':
            new.loc[:, c] = cat[c] + np.deg2rad(dec_offset)
        elif re.search('_flux$', c):
            new.loc[:, c] *= (1 + np.random.randn(len(new))*0.01)

    ra_sign = 'm' if ra_offset < 0 else 'p'
    dec_sign = 'm' if dec_offset < 0 else 'p'

    new_filename = filename[:-3] + '_{}_{}{}_{}{}.parq'.format(tag, ra_sign, abs(ra_offset),
                                                                 dec_sign, abs(dec_offset))

    fastparquet.write(new_filename, new)


class write_worker(object):
    def __init__(self, filename, tag='fake'):
        self.filename = filename
        self.tag = tag

    def __call__(self, args):
        dra, ddec = args
        print('dra={}, ddec={}'.format(*args))
        return perturb_catalog(self.filename, ra_offset=dra, dec_offset=ddec, tag=self.tag)

def write_fake_grid(pool, filename, ra_offsets=range(-10,11), dec_offsets=range(-10,11), tag='fake'):
    worker = write_worker(filename, tag=tag)

    results = pool.map(worker, itertools.product(ra_offsets, dec_offsets))
    pool.close()

if __name__=='__main__':

    import schwimmbad

    from argparse import ArgumentParser
    parser = ArgumentParser(description="Schwimmbad example.")

    group = parser.add_mutually_exclusive_group()
    group.add_argument("--ncores", dest="n_cores", default=1,
                       type=int, help="Number of processes (uses multiprocessing).")
    group.add_argument("--mpi", dest="mpi", default=False,
                       action="store_true", help="Run with MPI.")
    args = parser.parse_args()

    pool = schwimmbad.choose_pool(mpi=args.mpi, processes=args.n_cores)

    filename = '../data/forced_big.h5'
    write_fake_grid(pool, filename, ra_offsets=range(-1,2), dec_offsets=range(-1,2), tag='test')<commit_msg>made default script big writing<commit_after>import pandas as pd
import numpy as np
import fastparquet
import itertools
import re
import schwimmbad

def perturb_catalog(filename, ra_offset=2, dec_offset=2, tag='fake'):

    cat = pd.read_hdf(filename)
    new = cat.copy()
    for i, c in enumerate(cat.columns):
        if c=='coord_ra':
            new.loc[:, c] = cat[c] + np.deg2rad(ra_offset)
        elif c=='coord_dec':
            new.loc[:, c] = cat[c] + np.deg2rad(dec_offset)
        elif re.search('_flux$', c):
            new.loc[:, c] *= (1 + np.random.randn(len(new))*0.01)

    ra_sign = 'm' if ra_offset < 0 else 'p'
    dec_sign = 'm' if dec_offset < 0 else 'p'

    new_filename = filename[:-3] + '_{}_{}{}_{}{}.parq'.format(tag, ra_sign, abs(ra_offset),
                                                                 dec_sign, abs(dec_offset))

    fastparquet.write(new_filename, new)


class write_worker(object):
    def __init__(self, filename, tag='fake'):
        self.filename = filename
        self.tag = tag

    def __call__(self, args):
        dra, ddec = args
        print('dra={}, ddec={}'.format(*args))
        return perturb_catalog(self.filename, ra_offset=dra, dec_offset=ddec, tag=self.tag)

def write_fake_grid(pool, filename, ra_offsets=range(-10,11), dec_offsets=range(-10,11), tag='fake'):
    worker = write_worker(filename, tag=tag)

    results = pool.map(worker, itertools.product(ra_offsets, dec_offsets))
    pool.close()

if __name__=='__main__':

    import schwimmbad

    from argparse import ArgumentParser
    parser = ArgumentParser(description="Schwimmbad example.")

    group = parser.add_mutually_exclusive_group()
    group.add_argument("--ncores", dest="n_cores", default=1,
                       type=int, help="Number of processes (uses multiprocessing).")
    group.add_argument("--mpi", dest="mpi", default=False,
                       action="store_true", help="Run with MPI.")
    args = parser.parse_args()

    pool = schwimmbad.choose_pool(mpi=args.mpi, processes=args.n_cores)

    filename = '../data/forced_big.h5'
    # write_fake_grid(pool, filename, ra_offsets=range(-1,2), dec_offsets=range(-1,2), tag='test')
    write_fake_grid(pool, filename)
{'fraction_non_alphanumeric': 0.14410578216046616, 'fraction_numerical': 0.01165396683101748, 'mean_word_length': 3.053587647593097, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '90706', 'n_tokens_mistral': 1535, 'n_tokens_neox': 1464, 'n_words': 310}