<jupyter_start><jupyter_text>Generate FITS Files 25.05.2018<jupyter_code>%matplotlib inline
import warnings; warnings.simplefilter('ignore')
import matplotlib.pyplot as plt
#import matplotlib.gridspec as gridspec
#import matplotlib
import numpy as np
import pandas as pd
from os import listdir
from astropy.io import fits
import scipy.optimize as opt
import rydlab
from tqdm import tqdm_notebook
from matplotlib.patches import Arrow, Circle
#import peakutils
import numpy.ma as ma
from lmfit import  Model
import os
import numpy.ma as ma
import random


pd.options.display.max_colwidth = 120

font = {'family' : 'normal',
        'weight' : 'normal',
        'size'   : 14}

#matplotlib.rc('font', **font)

binning_scale = 1

path = '/home/qd/Schreibtisch/Data'
folders = rydlab.analyze_folder(path)
folders
folder=folders.Name[1]
print(folder)
rydlab.parse_folder(folder)
variable = rydlab.get_variables(folder)
paths = rydlab.get_paths(folder)
#print(paths)
# --------------------------------------------------------------------------
# ----- MAKE CHANGES HERE --------------------------------------------------
# --------------------------------------------------------------------------
date = '2019-07-17' # date of measurement
atoms =  1  # number of image in fits file
beam = 3
bg = 5
# --------------------------------------------------------------------------
# --------------------------------------------------------------------------



# ----- initialisations --------------------------------------------------

os.mkdir(folder+'_averaged_images')
savefolder = folder+'_averaged_images/'

print(savefolder)


iterator = tqdm_notebook(range(len(paths['images'])),desc='calculating...', leave=False) 
variableList = []


# ---- calculate averaged images ---------------------------------------------

for j in iterator:
    
    # ---- remove non existing images, when a run was not completed ---------
    bad_images = []
    for w in range(len(paths['images'].iloc[j])):
        if paths['images'].iloc[j][w][-4:] == 'None':
            bad_images.append(w)

    if len(bad_images)==0:
        image_paths = paths['images'].iloc[j]
        
    else:
        image_paths = paths['images'].iloc[j][:bad_images[0]]
        
    print(len(image_paths))
    
    # ----- initialisations --------------------------------------------------
    dimY=100*binning_scale
    dimX = 512*binning_scale
    image_atoms = np.zeros((dimY,dimX))
    image_beam = np.zeros((dimY,dimX))
    image_bg = np.zeros((dimY,dimX))

    # ---- average images ----------------------------------------------------
    for i in range(len(image_paths)):
        fits_image = fits.open(image_paths[i])
        image_atoms = image_atoms + fits_image[0].data[atoms]
        image_beam = image_beam + fits_image[0].data[beam]
        image_bg = image_bg + fits_image[0].data[bg]
        
    image_atoms = image_atoms /len(image_paths)
    image_beam = image_beam /len(image_paths)
    image_bg = image_bg / len(image_paths)
    
    hdu1 = fits.PrimaryHDU(([image_atoms,image_beam,image_bg]))
    hdul = fits.HDUList([hdu1])
    counter = str(j)
    counter = counter.zfill(2)
    filename = date +'_'+ counter
    hdul.writeto(savefolder+filename+'.fts')
    
    variableList.append((j,image_paths.name))
    
np.savetxt(savefolder +'/'+ date +'_variables.dat',variableList, fmt='%5.6f',delimiter=' ', newline=os.linesep)<jupyter_output><empty_output><jupyter_text>create mean and standard deviation<jupyter_code># --------------------------------------------------------------------------
# ----- MAKE CHANGES HERE --------------------------------------------------
# --------------------------------------------------------------------------
date = '2019-07-17' # date of measurement
atoms =  1  # number of image in fits file
beam = 3
bg = 5
# --------------------------------------------------------------------------
# --------------------------------------------------------------------------



# ----- initialisations --------------------------------------------------

iterator = tqdm_notebook(range(len(paths['images'])),desc='calculating...', leave=False) #len(paths['images'])
variableList = []


# ---- calculate averaged images ---------------------------------------------

for j in iterator:
    
    # ---- remove non excisting images, when a run was not completed ---------
    bad_images = []
    for w in range(len(paths['images'].iloc[j])):
        if paths['images'].iloc[j][w][100:] == 'None':
            bad_images.append(w)

    if len(bad_images)==0:
        print(paths['images'].iloc[j])
        image_paths = paths['images'].iloc[j]
    else:
        image_paths = paths['images'].iloc[j][:bad_images[0]]
        
    
    # ----- initialisations --------------------------------------------------
    dimY=100*binning_scale
    dimX = 512*binning_scale
    L = len(image_paths)
    image_atoms_matrix = np.zeros((L,dimY,dimX))
    image_beam_matrix = np.zeros((L,dimY,dimX))
    image_bg_matrix = np.zeros((L,dimY,dimX))

    # ---- calculated averaged and std images --------------------------------------
    for i in range(len(image_paths)):
        fits_image = fits.open(image_paths[i])
        image_atoms_matrix[i] = fits_image[0].data[1]
        image_beam_matrix[i] = fits_image[0].data[2]
        image_bg_matrix[i] = fits_image[0].data[3]
    
    image_atoms_mean = np.mean(image_atoms_matrix,axis=0)
    image_atoms_std = np.std(image_atoms_matrix,axis=0)
    image_beam_mean = np.mean(image_beam_matrix,axis=0)
    image_beam_std = np.std(image_beam_matrix,axis=0)
    image_bg_mean = np.mean(image_bg_matrix,axis=0)
    image_bg_std = np.std(image_bg_matrix,axis=0)
    
    # filename
    counter = str(j)
    counter = counter.zfill(3)
    filename = date +'_'+ counter
    
    # save mean images
    hdu1_mean = fits.PrimaryHDU(([image_atoms_mean,image_beam_mean,image_bg_mean]))
    hdul_mean = fits.HDUList([hdu1_mean])
    directory_mean = savefolder +  'mean'
    if not os.path.exists(directory_mean):
        os.makedirs(directory_mean)
    
    hdul_mean.writeto(directory_mean +'/'+ filename+'.fts')
    
    
    # save std images
    hdu1_std = fits.PrimaryHDU(([image_atoms_std,image_beam_std,image_bg_std]))
    hdul_std = fits.HDUList([hdu1_std])
    directory_std = savefolder +  'std'
    if not os.path.exists(directory_std):
        os.makedirs(directory_std)
    
    hdul_std.writeto(directory_std +'/'+ filename+'.fts')
    
    # append variable to variableList
    variableList.append((j,image_paths.name))
    
# save variables in both folders
np.savetxt(directory_mean +'/'+ date +'_variables.dat',variableList, fmt='%5.3f',delimiter=' ', newline=os.linesep)
np.savetxt(directory_std +'/'+ date +'_variables.dat',variableList, fmt='%5.3f',delimiter=' ', newline=os.linesep)<jupyter_output><empty_output><jupyter_text>check laser lock<jupyter_code># --------------------------------------------------------------------------
# ----- MAKE CHANGES HERE --------------------------------------------------
# --------------------------------------------------------------------------

atoms =  2  # number of image in fits file
beam = 4
bg = 6
j = 1 # number of variable with which you can test the look (e.g. know that you should see good EIT there) 
# --------------------------------------------------------------------------
# --------------------------------------------------------------------------

print(paths['images'].iloc[j].name)

# ---- remove non existing images, when a run was not completed ---------
bad_images = []
for w in range(len(paths['images'].iloc[j])):
    if paths['images'].iloc[j][w][107:] == 'None':
        bad_images.append(w)

if len(bad_images)==0:
    image_paths = paths['images'].iloc[j]
else:
    image_paths = paths['images'].iloc[j][:bad_images[0]]

print(len(image_paths))

# ----- initialisations --------------------------------------------------
dimY=100
dimX = 512
image_atoms = np.zeros((dimY,dimX))
image_beam = np.zeros((dimY,dimX))
image_bg = np.zeros((dimY,dimX))

minX = 50
maxX = 400
minY = 10
maxY = 90

# ---- average images ----------------------------------------------------
for i in range(len(image_paths)): #
    fits_image = fits.open(image_paths[i])
    image_atoms = fits_image[0].data[atoms][minY:maxY,minX:maxX]
    image_beam = fits_image[0].data[beam][minY:maxY,minX:maxX]
    image_bg = fits_image[0].data[bg][minY:maxY,minX:maxX]
    transmission = (image_atoms - image_bg)/(image_beam - image_bg)
    transmission[transmission > 10] = 1
    plt.figure()
    plt.imshow(transmission)
    plt.title(i)
#    fits_image = fits.open(image_paths[i])
#    image_atoms = image_atoms + fits_image[0].data[atoms]
#    image_beam = image_beam + fits_image[0].data[beam]
#    image_bg = image_bg + fits_image[0].data[bg]<jupyter_output><empty_output><jupyter_text>create averaged images from all data   create mean and standard deviation    create subsets for error estimation<jupyter_code># ---- initialisations --------------------------------------------------
date = '2018-05-09' # date of measurement
part = 3/4          # percentage of images taken for each subset
numSub = 5          # number of subsets

# ---- secify folder for subsets of averaged images ---------------------
savefolderSUB = 'A:/Projekte - Projects/2017_Light_Propagation/2018-05-09_scan_dDDSred_trans_large_averaged_images_subsets/'


for k in range(numSub):
    iterator = tqdm_notebook(range(len(paths['images'])), desc='calculating...', leave=False)
    variableList = []
    
    # generate random numbers
    numAverages = len(paths['images'].iloc[0])
    lenRand = int(numAverages*part)
    randNum = random.sample(range(numAverages), lenRand)
    print(randNum)

    for j in iterator:
        image_paths = paths['images'].iloc[j]
        # ----- initialisations --------------------------------------------------
        dimY=100
        dimX = 512
        image_atoms = np.zeros((dimY,dimX))
        image_beam = np.zeros((dimY,dimX))
        image_bg = np.zeros((dimY,dimX))

        # ---- average images ----------------------------------------------------
        for i in range(len(randNum)):
            number = randNum[i]
            fits_image = fits.open(image_paths[number])
            image_atoms = image_atoms + fits_image[0].data[1]
            image_beam = image_beam + fits_image[0].data[2]
            image_bg = image_bg + fits_image[0].data[3]

        image_atoms = image_atoms /len(randNum)
        image_beam = image_beam /len(randNum)
        image_bg = image_bg / len(randNum)
        
        # ---- save to folder ----------------------------------------------------
        # write into HDUList
        hdu1 = fits.PrimaryHDU(([image_atoms,image_beam,image_bg]))
        hdul = fits.HDUList([hdu1])
        
        # create foldername and create the folder if it does not excists
        counterSubset = str(k)
        counterSubset = counterSubset.zfill(2)
        directory = savefolderSUB +  counterSubset
        if not os.path.exists(directory):
            os.makedirs(directory)
            
        # crate filename
        counter = str(j)
        counter = counter.zfill(2)
        filename = date +'_'+ counter
        hdul.writeto(directory +'/'+ filename+'.fts')

        variableList.append((j,image_paths.name))
    
    # save variables for the subset
    np.savetxt(directory +'/'+ date +'_variables.dat',variableList, fmt='%5.3f',delimiter=' ', newline=os.linesep)<jupyter_output><empty_output><jupyter_text>create averaged images from a single folder<jupyter_code>import os
# --------------------------------------------------------------------------
# ----- MAKE CHANGES HERE --------------------------------------------------
# --------------------------------------------------------------------------
path = 'Z:/rydberg/Projekte - Projects/2017_Light_Propagation/2018/October/22/'
infolder = 'intensity_andor_image'
date = '2018-10-22' # date of measurement
number = 0
atoms =  1  # number of image in fits file
beam = 3
bg = 5
# --------------------------------------------------------------------------
# --------------------------------------------------------------------------


folder  = infolder + '/FITS Files/'
folder=path+folder
filenames= os.listdir(folder)
filenames=filenames[1:]


# ---- initialisations --------------------------------------------------

os.mkdir(path+infolder+'_averaged_images')
savefolder = path+infolder+'_averaged_images/'
iterator = tqdm_notebook(range(len(filenames)),desc='calculating...', leave=False) 
variableList = []


# ---- calculate averaged images ---------------------------------------------
    
# ----- initialisations --------------------------------------------------
dimY=100
dimX = 512
image_atoms = np.zeros((dimY,dimX))
image_beam = np.zeros((dimY,dimX))
image_bg = np.zeros((dimY,dimX))

# ---- average images ----------------------------------------------------
for i in iterator:
    fits_image = fits.open(folder+filenames[i])
    image_atoms = image_atoms + fits_image[0].data[atoms]
    image_beam = image_beam + fits_image[0].data[beam]
    image_bg = image_bg + fits_image[0].data[bg]

image_atoms = image_atoms /len(filenames)
image_beam = image_beam /len(filenames)
image_bg = image_bg / len(filenames)

# ---- save averaged image ----------------------------------------------------
hdu1 = fits.PrimaryHDU(([image_atoms,image_beam,image_bg]))
hdul = fits.HDUList([hdu1])
counter = str(1)
counter = counter.zfill(2)
filename = date +'_'+ str(number)#counter
hdul.writeto(savefolder+filename+'.fts')<jupyter_output><empty_output><jupyter_text>create averaged images from a single folders into one folder with variable file<jupyter_code>import os

# --------------------------------------------------------------------------
# ----- MAKE CHANGES HERE --------------------------------------------------
# --------------------------------------------------------------------------
path = 'Z:/rydberg/Projekte - Projects/2017_Light_Propagation/2018/October/05/'
date = '2018-10-05' # date of measurement
folderBase ='lens_pos_5mW'
numberString = 4 # number of last digits from the filename, that you want to use as a variable
atoms =  1  # number of image in fits file
beam = 3
bg = 5
# --------------------------------------------------------------------------
# --------------------------------------------------------------------------


# --------------------------------------------------------------------------
# ----- Main part ----------------------------------------------------------
# --------------------------------------------------------------------------


# ----- initialisations --------------------------------------------------
number=0
os.mkdir(path+folderBase+'_Averaged_images')
savefolder = path+folderBase+'_Averaged_images/'
variableList = []
allFolders = os.listdir(path)
correctFolders = []

# ----- select good folders --------------------------------------------------
for i in range(len(allFolders)):
    currentFolder = allFolders[i]
    if currentFolder[0:len(folderBase)] == folderBase:
        correctFolders.append(currentFolder)


# ---- calculate averaged images ---------------------------------------------
for j in range(len(correctFolders)-1):
    
# ----- initialisations --------------------------------------------------
    infolder = correctFolders[j]
    folder  = infolder + '/FITS Files/'
    folder = path + folder

    filenames= os.listdir(folder)
    filenames=filenames[1:]
    number = number+1
    counter = str(number)
    counter = counter.zfill(2)
    

    iterator = tqdm_notebook(range(len(filenames)),desc='calculating...', leave=False) 

    
# ----- initialisations for averaging --------------------------------------------------   
    dimY=100
    dimX = 512
    image_atoms = np.zeros((dimY,dimX))
    image_beam = np.zeros((dimY,dimX))
    image_bg = np.zeros((dimY,dimX))

# ---- average images ----------------------------------------------------
    for i in iterator:
        fits_image = fits.open(folder+filenames[i])
        image_atoms = image_atoms + fits_image[0].data[atoms]
        image_beam = image_beam + fits_image[0].data[beam]
        image_bg = image_bg + fits_image[0].data[bg]

    image_atoms = image_atoms /len(filenames)
    image_beam = image_beam /len(filenames)
    image_bg = image_bg / len(filenames)

# ---- save averaged image ----------------------------------------------------
    hdu1 = fits.PrimaryHDU(([image_atoms,image_beam,image_bg]))
    hdul = fits.HDUList([hdu1])
    
    filename = date +'_'+ counter
    hdul.writeto(savefolder+filename+'.fts')
    
    variableList.append((float(number),float(infolder[-numberString::])))
    
    
np.savetxt(savefolder +'/'+ date +'_variables.dat',variableList, fmt='%5.3f',delimiter=' ', newline=os.linesep)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.3035788228334812, 'fraction_numerical': 0.014137829044661343, 'mean_word_length': 1.6754233264757081, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '145439', 'license': 'MIT', 'n_tokens_mistral': 5116, 'n_tokens_neox': 4798, 'n_words': 924, 'path': 'FITSFiles_Generator.ipynb', 'repo_name': 'salzinger/ImageAnalysis', 'size': '16905'}