<commit_before>
'''
Bulk spectral line fitting with SDSS galaxy spectra
'''

import os
import shutil
from astropy.io import fits
from pandas import DataFrame

# Bring in the package funcs
from specfit import do_specfit
from download_spectra import download_spectra


def bulk_fit(obs_file, output_file, keep_spectra=True, split_save=True,
             num_save=10, num_start=-1):
    '''
    Downloads files based off of the entries in the given file, performs
    spectral line fitting and saves the results to a FITS table.

    Parameters
    ----------

    num_start : int {<num_save}
        Used for restarting at a point set by num_save. Default is -1, which
        starts at the beginning.
    '''

    # Open the file
    data_file = fits.open(obs_file)

    spectra_data = data_file[1].data
    del data_file

    num_spectra = spectra_data.size
    save_nums = [(num_spectra/num_save)*(i+1) for i in range(num_save)]
    save_nums[-1] = num_spectra
    save_nums.append(0)

    for i in range(save_nums[num_start], num_spectra):
        spec_info = spectra_data[i]

        # Download the spectrum
        spec_name = \
            download_spectra(spec_info['PLATE'], spec_info['FIBERID'],
                             spec_info['MJD'], spec_info['SURVEY'])

        spec_name = "spectra/" + spec_name

        try:
            spec_df = do_specfit(spec_name, verbose=False)
        except IOError:
            download_spectra(spec_info['PLATE'], spec_info['FIBERID'],
                             spec_info['MJD'], spec_info['SURVEY'],
                             download=True)
            shutil.move(spec_name.split("/")[-1], spec_name.split("/")[0])
            spec_df = do_specfit(spec_name, verbose=False)

        if i == 0:
            df = DataFrame(spec_df, columns=[spec_name[:-5]])
        else:
            df[spec_name[:-5]] = spec_df

        if split_save and i in save_nums:
            posn = [j for j, x in enumerate(save_nums) if x == i][0]
            df.to_csv(output_file[:-4]+"_"+str(posn+1)+".csv")
        if not keep_spectra:
            os.system('rm ' + spec_name)

    if not split_save:
        df.to_csv(output_file)

    return

if __name__ == "__main__":

    import sys

    input_file = str(sys.argv[1])
    output_file = str(sys.argv[2])

    restart_point = int(sys.argv[3])

    bulk_fit(input_file, output_file, num_start=restart_point)<commit_msg>Set the start point to create the dataframe.<commit_after>
'''
Bulk spectral line fitting with SDSS galaxy spectra
'''

import os
import shutil
from astropy.io import fits
from pandas import DataFrame

# Bring in the package funcs
from specfit import do_specfit
from download_spectra import download_spectra


def bulk_fit(obs_file, output_file, keep_spectra=True, split_save=True,
             num_save=10, num_start=-1):
    '''
    Downloads files based off of the entries in the given file, performs
    spectral line fitting and saves the results to a FITS table.

    Parameters
    ----------

    num_start : int {<num_save}
        Used for restarting at a point set by num_save. Default is -1, which
        starts at the beginning.
    '''

    # Open the file
    data_file = fits.open(obs_file)

    spectra_data = data_file[1].data
    del data_file

    num_spectra = spectra_data.size
    save_nums = [(num_spectra/num_save)*(i+1) for i in range(num_save)]
    save_nums[-1] = num_spectra
    save_nums.append(0)

    start_pt = save_nums[num_start]

    for i in range(start_pt, num_spectra):
        spec_info = spectra_data[i]

        # Download the spectrum
        spec_name = \
            download_spectra(spec_info['PLATE'], spec_info['FIBERID'],
                             spec_info['MJD'], spec_info['SURVEY'])

        spec_name = "spectra/" + spec_name

        try:
            spec_df = do_specfit(spec_name, verbose=False)
        except IOError:
            download_spectra(spec_info['PLATE'], spec_info['FIBERID'],
                             spec_info['MJD'], spec_info['SURVEY'],
                             download=True)
            shutil.move(spec_name.split("/")[-1], spec_name.split("/")[0])
            spec_df = do_specfit(spec_name, verbose=False)

        if i == start_pt:
            df = DataFrame(spec_df, columns=[spec_name[:-5]])
        else:
            df[spec_name[:-5]] = spec_df

        if split_save and i in save_nums:
            posn = [j for j, x in enumerate(save_nums) if x == i][0]
            df.to_csv(output_file[:-4]+"_"+str(posn+1)+".csv")
        if not keep_spectra:
            os.system('rm ' + spec_name)

    if not split_save:
        df.to_csv(output_file)

    return

if __name__ == "__main__":

    import sys

    input_file = str(sys.argv[1])
    output_file = str(sys.argv[2])

    restart_point = int(sys.argv[3])

    bulk_fit(input_file, output_file, num_start=restart_point)
{'fraction_non_alphanumeric': 0.10871794871794872, 'fraction_numerical': 0.007589743589743589, 'mean_word_length': 2.4508138711960368, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1656622', 'n_tokens_mistral': 1615, 'n_tokens_neox': 1573, 'n_words': 386}