<commit_before>#! /usr/bin/env python

"""
This script produces the stacks for emission line luminosity limited samples.
"""
import sys
import os 
from os.path import join
import glob
import numpy as n
import SpectraStackingEBOSS as sse

# create all stacks
dataList = n.array(glob.glob(join(os.environ['HOME'],"SDSS/lss/catalogs/3/stacks_v1", "eboss-elg_*.asc")))
dataList.sort()

for specList in dataList:
	print( specList )
	outfile = join(os.environ['HOME'], "stacks", os.path.basename(specList)[:-4]+".stack")
	#outfile = join(os.environ['HOME'], "SDSS", "stacks", os.path.basename(specList)[:-4]+".stack")
	if os.path.isfile(outfile)==False:
		stack=sse.SpectraStackingEBOSS(specList, outfile)
		print(outfile)
		if os.path.isfile(outfile+'.specMatrix.dat')==False:
			stack.createStackMatrix()
		stack.stackSpectra()

	outfile = join(os.environ['HOME'], "stacks", os.path.basename(specList)[:-4]+".UVstack")
	#outfile = join(os.environ['HOME'],"SDSS", "stacks", os.path.basename(specList)[:-4]+".UVstack")
	if os.path.isfile(outfile)==False:
		stack=sse.SpectraStackingEBOSS(specList, outfile)
		print(outfile)
		if os.path.isfile(outfile+'.specMatrix.dat')==False:
			stack.createStackMatrix_UVnormed()
		stack.stackSpectra()
		
<commit_msg>v51010<commit_after>#! /usr/bin/env python

"""
This script produces the stacks for emission line luminosity limited samples.
"""
import sys
import os 
from os.path import join
import glob
import numpy as n
import SpectraStackingEBOSS as sse
import time
t0=time.time()
# create all stacks
dataList = n.array(glob.glob(join(os.environ['HOME'],"SDSS/lss/catalogs/3/stacks_v1", "eboss-elg_*.asc")))
dataList.sort()

for specList in dataList:
	print('considers', specList, time.time()-t0)
	outfile = join(os.environ['HOME'], "stacks", os.path.basename(specList)[:-4]+".stack")
	#outfile = join(os.environ['HOME'], "SDSS", "stacks", os.path.basename(specList)[:-4]+".stack")
	if os.path.isfile(outfile)==False:
		print('starts working on', outfile, time.time()-t0)
		stack=sse.SpectraStackingEBOSS(specList, outfile)
		if os.path.isfile(outfile+'.specMatrix.dat')==False:
			print('creates matrix', time.time()-t0)
			stack.createStackMatrix()
		print('stacks', time.time()-t0)
		stack.stackSpectra()

	outfile = join(os.environ['HOME'], "stacks", os.path.basename(specList)[:-4]+".UVstack")
	#outfile = join(os.environ['HOME'],"SDSS", "stacks", os.path.basename(specList)[:-4]+".UVstack")
	if os.path.isfile(outfile)==False:
		print('starts working on', outfile, time.time()-t0)
		stack=sse.SpectraStackingEBOSS(specList, outfile)
		if os.path.isfile(outfile+'.specMatrix.dat')==False:
			print('creates matrix', time.time()-t0)
			stack.createStackMatrix_UVnormed()
		print('stacks', time.time()-t0)
		stack.stackSpectra()
		

{'fraction_non_alphanumeric': 0.18037518037518038, 'fraction_numerical': 0.009018759018759018, 'mean_word_length': 5.8638613861386135, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2220879', 'n_tokens_mistral': 1093, 'n_tokens_neox': 1035, 'n_words': 182}