<commit_before>import itertools
import random
import cProfile

import numpy as np
import astropy.table as table

import read_spectrum_fits
from hdf5_spectrum_container import Hdf5SpectrumContainer
import common_settings
from qso_data import QSORecord, QSOData


MAX_SPECTRA = 220000
MAX_WAVELENGTH_COUNT = 4992

settings = common_settings.Settings()


def save_spectrum(qso_spec_obj):
    """

    :type qso_spec_obj: QSOData
    :return:
    """
    qso_rec = qso_spec_obj.qso_rec
    z = qso_rec.z
    ar_wavelength = qso_spec_obj.ar_wavelength
    ar_flux = qso_spec_obj.ar_flux
    ar_ivar = qso_spec_obj.ar_ivar
    return [ar_wavelength, ar_flux, ar_ivar]


sample_fraction = 1


def profile_main():
    index = 0

    qso_record_table = table.Table(np.load(settings.get_qso_metadata_npy()))

    # assume qso_record_table is already sorted
    spec_sample = read_spectrum_fits.enum_spectra(qso_record_table, pre_sort=False)

    qso_spectra_hdf5 = settings.get_qso_spectra_hdf5()
    output_spectra = Hdf5SpectrumContainer(qso_spectra_hdf5, readonly=False, create_new=True,
                                           num_spectra=MAX_SPECTRA)

    if settings.get_single_process():
        result_enum = itertools.imap(save_spectrum,
                                     itertools.ifilter(lambda x: random.random() < sample_fraction, spec_sample))
    else:
        assert False, "Not supported"

    for i in result_enum:
        output_spectra.set_wavelength(index, i[0])
        output_spectra.set_flux(index, i[1])
        output_spectra.set_ivar(index, i[2])
        index += 1


if settings.get_profile():
    cProfile.run('profile_main()', sort=2, filename='extract_sdss_qso_spectra.prof')
else:
    profile_main()<commit_msg>remove sample fraction.<commit_after>import itertools
import random
import cProfile

import numpy as np
import astropy.table as table

import read_spectrum_fits
from hdf5_spectrum_container import Hdf5SpectrumContainer
import common_settings
from qso_data import QSORecord, QSOData


MAX_SPECTRA = 220000
MAX_WAVELENGTH_COUNT = 4992

settings = common_settings.Settings()


def save_spectrum(qso_spec_obj):
    """

    :type qso_spec_obj: QSOData
    :return:
    """
    qso_rec = qso_spec_obj.qso_rec
    z = qso_rec.z
    ar_wavelength = qso_spec_obj.ar_wavelength
    ar_flux = qso_spec_obj.ar_flux
    ar_ivar = qso_spec_obj.ar_ivar
    return [ar_wavelength, ar_flux, ar_ivar]


def profile_main():
    index = 0

    qso_record_table = table.Table(np.load(settings.get_qso_metadata_npy()))

    # assume qso_record_table is already sorted
    spec_sample = read_spectrum_fits.enum_spectra(qso_record_table, pre_sort=False)

    qso_spectra_hdf5 = settings.get_qso_spectra_hdf5()
    output_spectra = Hdf5SpectrumContainer(qso_spectra_hdf5, readonly=False, create_new=True,
                                           num_spectra=MAX_SPECTRA)

    if settings.get_single_process():
        result_enum = itertools.imap(save_spectrum, spec_sample)
    else:
        assert False, "Not supported"

    for i in result_enum:
        output_spectra.set_wavelength(index, i[0])
        output_spectra.set_flux(index, i[1])
        output_spectra.set_ivar(index, i[2])
        index += 1


if settings.get_profile():
    cProfile.run('profile_main()', sort=2, filename='extract_sdss_qso_spectra.prof')
else:
    profile_main()
{'fraction_non_alphanumeric': 0.0787518573551263, 'fraction_numerical': 0.01337295690936107, 'mean_word_length': 3.694560669456067, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6930479', 'n_tokens_mistral': 1229, 'n_tokens_neox': 1162, 'n_words': 213}