<commit_before>from spectral_cube import SpectralCube, BooleanArrayMask
import numpy as np
from astropy import wcs
from astropy.io import fits
from astropy import units as u
from astropy import log
from astropy.utils.console import ProgressBar

from paths import hpath
from constrain_parameters import paraH2COmodel
from masked_cubes import cube303m,cube321m,cube303,cube321
from noise import noise, noise_cube
from common_constants import logabundance,elogabundance
from higal_gridded import column_regridded

nsigma = 5 # start big to minimize # of failures

mf = paraH2COmodel()

#noise = fits.getdata(hpath('APEX_H2CO_303_202_noise.fits'))
#noise = cube303[:50].std(axis=0).value
#noise = fits.getdata(mpath('APEX_H2CO_merge_high_sub_noise.fits'))
#nhits = nhits = fits.getdata(paths.mpath('APEX_H2CO_merge_high_nhits.fits'))
#noise[nhits<20] = np.nan

noise_flat = noise_cube[mask]
var_flat = noise_flat**2

ratio303321 = cube321m.flattened().value / cube303m.flattened().value
eratio303321 = (ratio303321**2 * (var_flat/cube303m.flattened().value**2 + var_flat/cube321m.flattened().value**2))**0.5

indices = np.where(mask)
usable = (eratio303321*nsigma < ratio303321) & (eratio303321 > 0) & (ratio303321 > 0) & (noise_flat > 1e-10) & (noise_flat < 10) & (ratio303321 < 100)
ngood = np.count_nonzero(usable)
usable_indices = [ind[usable] for ind in indices]
uz,uy,ux = usable_indices

column_flat = column_regridded.data[uy,ux]<commit_msg>import fix<commit_after>from spectral_cube import SpectralCube, BooleanArrayMask
import numpy as np
from astropy import wcs
from astropy.io import fits
from astropy import units as u
from astropy import log
from astropy.utils.console import ProgressBar

from paths import hpath
from constrain_parameters import paraH2COmodel
from masked_cubes import cube303m,cube321m,cube303,cube321,mask
from noise import noise, noise_cube
from common_constants import logabundance,elogabundance
from higal_gridded import column_regridded

nsigma = 5 # start big to minimize # of failures

mf = paraH2COmodel()

#noise = fits.getdata(hpath('APEX_H2CO_303_202_noise.fits'))
#noise = cube303[:50].std(axis=0).value
#noise = fits.getdata(mpath('APEX_H2CO_merge_high_sub_noise.fits'))
#nhits = nhits = fits.getdata(paths.mpath('APEX_H2CO_merge_high_nhits.fits'))
#noise[nhits<20] = np.nan

noise_flat = noise_cube[mask]
var_flat = noise_flat**2

ratio303321 = cube321m.flattened().value / cube303m.flattened().value
eratio303321 = (ratio303321**2 * (var_flat/cube303m.flattened().value**2 + var_flat/cube321m.flattened().value**2))**0.5

indices = np.where(mask)
usable = (eratio303321*nsigma < ratio303321) & (eratio303321 > 0) & (ratio303321 > 0) & (noise_flat > 1e-10) & (noise_flat < 10) & (ratio303321 < 100)
ngood = np.count_nonzero(usable)
usable_indices = [ind[usable] for ind in indices]
uz,uy,ux = usable_indices

column_flat = column_regridded.data[uy,ux]
{'fraction_non_alphanumeric': 0.10753803596127248, 'fraction_numerical': 0.07468879668049792, 'mean_word_length': 6.776881720430108, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5195379', 'n_tokens_mistral': 1231, 'n_tokens_neox': 1042, 'n_words': 230}