<jupyter_start><jupyter_text>step 1: define material and other parameters for simulating Laue patterns<jupyter_code>if __name__ == '__main__':     #enclosing required because of multiprocessing
    ## Import modules used for this Notebook
    import os

    ## if LaueToolsNN is properly installed
    try:
        from lauetoolsnn.utils_lauenn import generate_classHKL, generate_dataset, rmv_freq_class, get_material_detail
    except:
        # else import from a path where LaueToolsNN files are
        import sys
        sys.path.append(r"USER_PATH_HERE")
        from utils_lauenn import generate_classHKL, generate_dataset, rmv_freq_class,  get_material_detail
        
    import numpy as np
    import os
    import _pickle as cPickle
    import itertools
    from keras.callbacks import EarlyStopping, ModelCheckpoint
    import matplotlib.pyplot as plt

    ## if LaueToolsNN is properly installed
    try:
        from lauetoolsnn.utils_lauenn import array_generator, array_generator_verify, vali_array
    except:
        # else import from a path where LaueToolsNN files are
        import sys
        sys.path.append(r"C:\Users\purushot\Desktop\github_version_simple\lauetoolsnn")
        from utils_lauenn import array_generator, array_generator_verify, vali_array
        
    import os
    import numpy as np
    import random
    from tqdm import trange

    ## if LaueToolsNN is properly installed
    try:
        from lauetoolsnn.utils_lauenn import get_material_detail, prepare_LP_NB
        from lauetoolsnn.lauetools import dict_LaueTools as dictLT
        from lauetoolsnn.lauetools import IOLaueTools as IOLT
    except:
        # else import from a path where LaueToolsNN files are
        import sys
        sys.path.append(r"C:\Users\purushot\Desktop\github_version_simple\lauetoolsnn")
        from utils_lauenn import get_material_detail, prepare_LP_NB
        sys.path.append(r"C:\Users\purushot\Desktop\github_version_simple\lauetoolsnn\lauetools")
        import dict_LaueTools as dictLT
        import IOLaueTools as IOLT
    # =============================================================================
    ## User Input dictionary with parameters
    ## In case of only one phase/material, keep same value for material_ and material1_ key
    # =============================================================================
    
    text_file_logger = open("logger_ZrO2.txt", "w")
    bss = [1,5,10,15,20,20,20,20,20,50,50,50,50,50,50,50,50,50]
    nmbr = [10,50,100,150,200,250,300,350,400,450,500,550,600,700,800,900,1000,2000]
    for inums in range(len(nmbr)):
        
        epochs = 20
        batch_size = bss[inums]
        inum = nmbr[inums]
        if inum != 2000:
            continue
        
        input_params = {
                        "material_": "ZrO2",             ## same key as used in dict_LaueTools
                        "material1_": "ZrO2",            ## same key as used in dict_LaueTools
                        "prefix" : "data",                 ## prefix for the folder to be created for training dataset
                        "symmetry": "monoclinic",           ## crystal symmetry of material_
                        "symmetry1": "monoclinic",          ## crystal symmetry of material1_
                        "SG": 14,                     ## Space group of material_ (None if not known)
                        "SG1": 14,                    ## Space group of material1_ (None if not known)
                        "hkl_max_identify" : 5,        ## Maximum hkl index to classify in a Laue pattern
                        "maximum_angle_to_search":90,  ## Angle of radial distribution to reconstruct the histogram (in deg)
                        "step_for_binning" : 0.1,      ## bin widht of angular radial distribution in degree
                        "nb_grains_per_lp" : 5,        ## max grains to be generated in a Laue Image
                        "grains_nb_simulate" : inum,    ## Number of orientations to generate (takes advantage of crystal symmetry)
                        ## Detector parameters (roughly) of the Experimental setup
                        ## Sample-detector distance, X center, Y center, two detector angles
                        "detectorparameters" :  [79.553,979.32,932.31,0.37,0.447], 
                        "pixelsize" : 0.0734,          ## Detector pixel size
                        "dim1":2018,                   ## Dimensions of detector in pixels
                        "dim2":2016,
                        "emin" : 5,                    ## Minimum and maximum energy to use for simulating Laue Patterns
                        "emax" : 22,
                        }

        input_params["prefix"] = "_" + str(input_params["grains_nb_simulate"]) + input_params["prefix"]
        text_file_logger.write(input_params["prefix"] + "\n")
        material_= input_params["material_"]
        material1_= input_params["material1_"]
        n = input_params["hkl_max_identify"]
        maximum_angle_to_search = input_params["maximum_angle_to_search"]
        step_for_binning = input_params["step_for_binning"]
        nb_grains_per_lp = input_params["nb_grains_per_lp"]
        grains_nb_simulate = input_params["grains_nb_simulate"]
        detectorparameters = input_params["detectorparameters"]
        pixelsize = input_params["pixelsize"]
        emax = input_params["emax"]
        emin = input_params["emin"]
        symm_ = input_params["symmetry"]
        symm1_ = input_params["symmetry1"]
        SG = input_params["SG"]
        SG1 = input_params["SG1"]

        if material_ != material1_:
            save_directory = os.getcwd()+"//"+material_+"_"+material1_+input_params["prefix"]
        else:
            save_directory = os.getcwd()+"//"+material_+input_params["prefix"]
        print("save directory is : "+save_directory)
        if not os.path.exists(save_directory):
            os.makedirs(save_directory)

        ## get unit cell parameters and other details required for simulating Laue patterns
        rules, symmetry, lattice_material, \
            crystal, SG, rules1, symmetry1,\
            lattice_material1, crystal1, SG1 = get_material_detail(material_, SG, symm_,
                                                                   material1_, SG1, symm1_)

        ## procedure for generation of GROUND TRUTH classes
        # general_diff_cond = True will eliminate the hkl index that does not satisfy the general reflection conditions
        generate_classHKL(n, rules, lattice_material, symmetry, material_, crystal=crystal, SG=SG, general_diff_cond=True,
                  save_directory=save_directory, write_to_console=print, ang_maxx = maximum_angle_to_search, 
                  step = step_for_binning)

        if material_ != material1_:
            generate_classHKL(n, rules1, lattice_material1, symmetry1, material1_, crystal=crystal1, SG=SG1, general_diff_cond=True,
                      save_directory=save_directory, write_to_console=print, ang_maxx = maximum_angle_to_search, 
                      step = step_for_binning)
        ############ GENERATING TRAINING DATA ##############
        # data_realism =True ; will introduce noise and partial Laue patterns in the training dataset
        # modelp can have either "random" for random orientation generation or "uniform" for uniform orientation generation
        # include_scm (if True; misorientation_angle parameter need to be defined): this parameter introduces misoriented crystal of specific angle along a crystal axis in the training dataset
        generate_dataset(material_=material_, material1_=material1_, ang_maxx=maximum_angle_to_search,
                             step=step_for_binning, mode=0, 
                             nb_grains=nb_grains_per_lp, nb_grains1=nb_grains_per_lp, 
                             grains_nb_simulate=grains_nb_simulate, data_realism = True, 
                             detectorparameters=detectorparameters, pixelsize=pixelsize, type_="training_data",
                             var0 = 1, dim1=input_params["dim1"], dim2=input_params["dim2"], 
                             removeharmonics=1, save_directory=save_directory,
                            write_to_console=print, emin=emin, emax=emax, modelp = "random",
                            misorientation_angle = 1, general_diff_rules = False, 
                            crystal = crystal, crystal1 = crystal1, include_scm=False,)

        ############ GENERATING TESTING DATA ##############
        factor = 5 # validation split for the training dataset  --> corresponds to 20% of total training dataset
        generate_dataset(material_=material_, material1_=material1_, ang_maxx=maximum_angle_to_search,
                             step=step_for_binning, mode=0, 
                             nb_grains=nb_grains_per_lp, nb_grains1=nb_grains_per_lp, 
                             grains_nb_simulate=grains_nb_simulate//factor, data_realism = True, 
                             detectorparameters=detectorparameters, pixelsize=pixelsize, type_="testing_data",
                             var0 = 1, dim1=input_params["dim1"], dim2=input_params["dim2"], 
                             removeharmonics=1, save_directory=save_directory,
                            write_to_console=print, emin=emin, emax=emax, modelp = "random",
                            misorientation_angle = 1, general_diff_rules = False, 
                            crystal = crystal, crystal1 = crystal1, include_scm=False,)

        ## Updating the ClassHKL list by removing the non-common HKL or less frequent HKL from the list
        ## The non-common HKL can occur as a result of the detector position and energy used
        # freq_rmv: remove output hkl if the training dataset has less tha 100 occurances of the considered hkl (freq_rmv1 for second phase)
        # Weights (penalty during training) are also calculated based on the occurance
        rmv_freq_class(freq_rmv = 1, freq_rmv1 = 1,
                            save_directory=save_directory, material_=material_, 
                            material1_=material1_, write_to_console=print)

        ## End of data generation for Neural network training: all files are saved in the same folder to be later used for training and prediction
        import numpy as np
        import _pickle as cPickle
        classhkl = np.load(save_directory+"//MOD_grain_classhkl_angbin.npz")["arr_0"]
        angbins = np.load(save_directory+"//MOD_grain_classhkl_angbin.npz")["arr_1"]
        loc_new = np.load(save_directory+"//MOD_grain_classhkl_angbin.npz")["arr_2"]
        with open(save_directory+"//class_weights.pickle", "rb") as input_file:
            class_weights = cPickle.load(input_file)
        class_weights = class_weights[0]

        n_bins = len(angbins)-1
        n_outputs = len(classhkl)
        print(n_bins, n_outputs)

        import tensorflow as tf
        from tensorflow.keras.layers import BatchNormalization
        import keras
        from keras.regularizers import l2
        from keras.models import Sequential
        from keras.layers import Dense, Activation, Dropout
        from keras.constraints import maxnorm

        metricsNN = [
                    keras.metrics.FalseNegatives(name="fn"),
                    keras.metrics.FalsePositives(name="fp"),
                    keras.metrics.TrueNegatives(name="tn"),
                    keras.metrics.TruePositives(name="tp"),
                    keras.metrics.Precision(name="precision"),
                    keras.metrics.Recall(name="accuracy"),
                    ]

        def model_arch_general_optimized(n_bins, n_outputs, kernel_coeff = 0.0005, bias_coeff = 0.0005, lr=None, verbose=0,
                               write_to_console=None):
            """
            Very simple and straight forward Neural Network with few hyperparameters
            straighforward RELU activation strategy with cross entropy to identify the HKL
            Tried BatchNormalization --> no significant impact
            Tried weighted approach --> not better for HCP
            Trying Regularaization 
            l2(0.001) means that every coefficient in the weight matrix of the layer 
            will add 0.001 * weight_coefficient_value**2 to the total loss of the network
            1e-3,1e-5,1e-6
            """
            if n_outputs >= n_bins:
                param = n_bins
                if param*15 < (2*n_outputs): ## quick hack; make Proper implementation
                    param = (n_bins + n_outputs)//2
            else:
                # param = n_outputs ## More reasonable ???
                param = n_outputs*2 ## More reasonable ???
                # param = n_bins//2

            model = Sequential()
            model.add(keras.Input(shape=(n_bins,)))
            ## Hidden layer 1
            model.add(Dense(n_bins, kernel_regularizer=l2(kernel_coeff), bias_regularizer=l2(bias_coeff)))
            # model.add(BatchNormalization())
            model.add(Activation('relu'))
            model.add(Dropout(0.3)) ## Adding dropout as we introduce some uncertain data with noise
            ## Hidden layer 2
            model.add(Dense(((param)*15 + n_bins)//2, kernel_regularizer=l2(kernel_coeff), bias_regularizer=l2(bias_coeff)))
            # model.add(BatchNormalization())
            model.add(Activation('relu'))
            model.add(Dropout(0.3))
            ## Hidden layer 3
            model.add(Dense((param)*15, kernel_regularizer=l2(kernel_coeff), bias_regularizer=l2(bias_coeff)))
            # model.add(BatchNormalization())
            model.add(Activation('relu'))
            model.add(Dropout(0.3))
            ## Output layer 
            model.add(Dense(n_outputs, activation='softmax'))
            ## Compile model
            if lr != None:
                otp = tf.keras.optimizers.Adam(learning_rate=lr)
                model.compile(loss='categorical_crossentropy', optimizer=otp, metrics=[metricsNN])
            else:
                model.compile(loss='categorical_crossentropy', optimizer="adam", metrics=[metricsNN])

            if verbose == 1:
                model.summary()
                stringlist = []
                model.summary(print_fn=lambda x: stringlist.append(x))
                short_model_summary = "\n".join(stringlist)
                if write_to_console!=None:
                    write_to_console(short_model_summary)
            return model

        # load model and train
        #neurons_multiplier is a list with number of neurons per layer, the first value is input shape and last value is output shape, inbetween are the number of neurons per hidden layers
        model = model_arch_general_optimized(  n_bins, n_outputs,
                                               kernel_coeff = 1e-5,
                                               bias_coeff = 1e-6,
                                               lr = 1e-3,
                                                )
        #model = model_arch_general_optimized(n_bins, n_outputs, kernel_coeff = 0.0005, bias_coeff = 0.0005, lr=None, verbose=1,)
        ## temp function to quantify the spots and classes present in a batch
        
        trainy_inbatch = array_generator_verify(save_directory+"//training_data", batch_size, 
                                                len(classhkl), loc_new, print)
        print("Number of spots in a batch of %i files : %i" %(batch_size, len(trainy_inbatch)))
        print("Min, Max class ID is %i, %i" %(np.min(trainy_inbatch), np.max(trainy_inbatch)))

        ## Batch loading for numpy grain files (Keep low value to avoid overcharging the RAM)
        nb_grains_per_lp1 = nb_grains_per_lp
        if material_ != material1_:
            nb_grains_list = list(range(nb_grains_per_lp+1))
            nb_grains1_list = list(range(nb_grains_per_lp1+1))
            list_permute = list(itertools.product(nb_grains_list, nb_grains1_list))
            list_permute.pop(0)
            steps_per_epoch = (len(list_permute) * grains_nb_simulate)//batch_size
        else:
            steps_per_epoch = int((nb_grains_per_lp * grains_nb_simulate) / batch_size)

        val_steps_per_epoch = int(steps_per_epoch / 5)
        if steps_per_epoch == 0:
            steps_per_epoch = 1
        if val_steps_per_epoch == 0:
            val_steps_per_epoch = 1 

        ## Load generator objects from filepaths (iterators for Training and Testing datasets)
        training_data_generator = array_generator(save_directory+"//training_data", batch_size, \
                                                  len(classhkl), loc_new, print)
        testing_data_generator = array_generator(save_directory+"//testing_data", batch_size, \
                                                  len(classhkl), loc_new, print)

        ######### TRAIN THE DATA
        es = EarlyStopping(monitor='val_accuracy', mode='max', patience=2)
        ms = ModelCheckpoint(save_directory+"//best_val_acc_model.h5", monitor='val_accuracy', 
                              mode='max', save_best_only=True)

        # model save directory and filename
        if material_ != material1_:
            model_name = save_directory+"//model_"+material_+"_"+material1_
        else:
            model_name = save_directory+"//model_"+material_

        ## Fitting function
        stats_model = model.fit(
                                training_data_generator, 
                                epochs=epochs, 
                                steps_per_epoch=steps_per_epoch,
                                validation_data=testing_data_generator,
                                validation_steps=val_steps_per_epoch,
                                verbose=1,
                                class_weight=class_weights,
                                callbacks=[es, ms]
                                )

        # Save model config and weights
        model_json = model.to_json()
        with open(model_name+".json", "w") as json_file:
            json_file.write(model_json)            
        # serialize weights to HDF5
        model.save_weights(model_name+".h5")
        print("Saved model to disk")

        print( "Training Accuracy: "+str( stats_model.history['accuracy'][-1]))
        print( "Training Loss: "+str( stats_model.history['loss'][-1]))
        print( "Validation Accuracy: "+str( stats_model.history['val_accuracy'][-1]))
        print( "Validation Loss: "+str( stats_model.history['val_loss'][-1]))

        # Plot the accuracy/loss v Epochs
        epochs = range(1, len(model.history.history['loss']) + 1)
        fig, ax = plt.subplots(1,2)
        ax[0].plot(epochs, model.history.history['loss'], 'r', label='Training loss')
        ax[0].plot(epochs, model.history.history['val_loss'], 'r', ls="dashed", label='Validation loss')
        ax[0].legend()
        ax[1].plot(epochs, model.history.history['accuracy'], 'g', label='Training Accuracy')
        ax[1].plot(epochs, model.history.history['val_accuracy'], 'g', ls="dashed", label='Validation Accuracy')
        ax[1].legend()
        if material_ != material1_:
            plt.savefig(save_directory+"//loss_accuracy_"+material_+"_"+material1_+".png", bbox_inches='tight',format='png', dpi=1000)
        else:
            plt.savefig(save_directory+"//loss_accuracy_"+material_+".png", bbox_inches='tight',format='png', dpi=1000)
        plt.close()

        if material_ != material1_:
            text_file = open(save_directory+"//loss_accuracy_logger_"+material_+"_"+material1_+".txt", "w")
        else:
            text_file = open(save_directory+"//loss_accuracy_logger_"+material_+".txt", "w")

        text_file.write("# EPOCH, LOSS, VAL_LOSS, ACCURACY, VAL_ACCURACY" + "\n")
        for inj in range(len(epochs)):
            string1 = str(epochs[inj]) + ","+ str(model.history.history['loss'][inj])+\
                    ","+str(model.history.history['val_loss'][inj])+","+str(model.history.history['accuracy'][inj])+\
                    ","+str(model.history.history['val_accuracy'][inj])+" \n"  
            text_file.write(string1)
        text_file.close() 

        from lattice import HklPlane
        

        def commonclass(hkl1, hkl2, lattice_material, symmetry_):
            """ test if hkl1 and hkl2 belong to the same class"""
            h,k,l = hkl1
            h1,k1,l1 = hkl2
            h_obj = HklPlane(h,k,l, lattice=lattice_material)
            normal = np.round(h_obj.normal(), 6)

            h_obj1 = HklPlane(h1,k1,l1, lattice=lattice_material)
            family = h_obj1.get_family(h_obj1.miller_indices(), lattice=h_obj1._lattice, 
                                       include_friedel_pairs=True, crystal_structure=symmetry_)

            normals = np.array([np.round(ijk.normal(),6) for ijk in family])

            cond1 = h_obj in family
            cond2 = np.any(np.all(normal == normals, axis=1))
            return cond1 or cond2


        ### verify prediction
        import time
        from fast_histogram import histogram1d
        import random

        length = []
        nbgrains = 5
        nbgrains1 = 5
        material1_ = material_
        total_prediction = 0
        count_good = 0
        count_bad = 0
        for _ in trange(50):
            noisy_data = bool(random.getrandbits(1)) 
            remove_peaks = bool(random.getrandbits(1)) 
            seednumber = np.random.randint(1e6)
            tabledistancerandom, hkl_sol, \
                    s_posx, s_posy, s_I, s_tth, s_chi, g, g1  = prepare_LP_NB(nbgrains, nbgrains1,
                                                                            material_, 0,
                                                                            material1_ = material1_,
                                                                            seed = seednumber,sortintensity=True,
                                                                            detectorparameters=[79.553,979.32,932.31,0.37,0.447], 
                                                                            pixelsize=0.0734,
                                                                            dim1=2018, dim2=2016, 
                                                                            emin=5, emax=23,
                                                                            flag = 10, noisy_data=noisy_data,
                                                                            remove_peaks = remove_peaks)
            length.append(len(s_posx))

            ## Not all spots needs to be recognized for now
            spots_in_center = np.arange(0,len(s_tth))

            start_time = time.time()
            codebars_all = []
            #print("Spots in center: ",len(spots_in_center))
            for i in spots_in_center: ## identify the center HKL spots 
                spotangles = tabledistancerandom[i]
                spotangles = np.delete(spotangles, i)# removing the self distance
                # codebars = np.histogram(spotangles, bins=angbins)[0]
                codebars = histogram1d(spotangles, range=[min(angbins),max(angbins)], bins=len(angbins)-1)
                ## normalize the same way as training data
                max_codebars = np.max(codebars)
                codebars = codebars/ max_codebars
                codebars_all.append(codebars)
            ## reshape for the model to predict all spots at once
            codebars = np.array(codebars_all)
            ## Do prediction of all spots at once
            prediction = model.predict(codebars)
            max_pred = np.max(prediction, axis = 1)
            class_predicted = np.argmax(prediction, axis = 1)
            ## verify prediction (Assuming we dont know HKL solution)
            for i, gg in enumerate(class_predicted):

                if np.any(np.abs(hkl_sol[spots_in_center[i],:3]) > 5):
                    continue

                total_prediction += 1
                isgoodprediction = commonclass(classhkl[gg], hkl_sol[spots_in_center[i],:3], lattice_material, symmetry)
                if isgoodprediction:
                    count_good += 1
                else:
                    #print("Predicted HKL: ", classhkl[gg], " Actual HKL: ", hkl_sol[spots_in_center[i],:3])
                    count_bad += 1
            #print("Seed number for LP:",seednumber)

            end_time = time.time()
        length = np.array(length)
        print("****************************************************************************")
        print("Total spots attempted:",total_prediction)
        print("Total Good prediction:",count_good)
        print("Total Bad prediction:",count_bad)
        print(material_, np.average(length), np.std(length))
        
        text_file_logger.write("Total spots attempted:" +str(total_prediction) + "\n")
        text_file_logger.write("Total Good prediction:" +str(count_good) + "\n")
        text_file_logger.write("Total Bad prediction:" +str(count_bad) + "\n")
        text_file_logger.write(material_ +", "+ str(np.average(length))+", "+ str(np.std(length)) + "\n")
        
    text_file_logger.close()
if __name__ == '__main__':     #enclosing required because of multiprocessing
    ## Import modules used for this Notebook
    import os

    ## if LaueToolsNN is properly installed
    try:
        from lauetoolsnn.utils_lauenn import generate_classHKL, generate_dataset, rmv_freq_class, get_material_detail, read_hdf5
    except:
        # else import from a path where LaueToolsNN files are
        import sys
        sys.path.append(r"USER_PATH_HERE")
        from utils_lauenn import generate_classHKL, generate_dataset, rmv_freq_class,  get_material_detail, read_hdf5
        
    import numpy as np
    import os
    import _pickle as cPickle
    import itertools
    from keras.callbacks import EarlyStopping, ModelCheckpoint
    import matplotlib.pyplot as plt

    ## if LaueToolsNN is properly installed
    try:
        from lauetoolsnn.utils_lauenn import array_generator, array_generator_verify, vali_array
    except:
        # else import from a path where LaueToolsNN files are
        import sys
        sys.path.append(r"C:\Users\purushot\Desktop\github_version_simple\lauetoolsnn")
        from utils_lauenn import array_generator, array_generator_verify, vali_array
        
    import os
    import numpy as np
    import random
    from tqdm import trange
    from keras.models import model_from_json

    ## if LaueToolsNN is properly installed
    try:
        from lauetoolsnn.utils_lauenn import get_material_detail, prepare_LP_NB
        from lauetoolsnn.lauetools import dict_LaueTools as dictLT
        from lauetoolsnn.lauetools import IOLaueTools as IOLT
    except:
        # else import from a path where LaueToolsNN files are
        import sys
        sys.path.append(r"C:\Users\purushot\Desktop\github_version_simple\lauetoolsnn")
        from utils_lauenn import get_material_detail, prepare_LP_NB
        sys.path.append(r"C:\Users\purushot\Desktop\github_version_simple\lauetoolsnn\lauetools")
        import dict_LaueTools as dictLT
        import IOLaueTools as IOLT
    # =============================================================================
    ## User Input dictionary with parameters
    ## In case of only one phase/material, keep same value for material_ and material1_ key
    # =============================================================================
    
    text_file_logger = open("logger_ZrO2.txt", "w")
    bss = [1,5,10,15,20,20,20,20,20,50,50,50,50,50,50,50,50,50]
    nmbr = [10,50,100,150,200,250,300,350,400,450,500,550,600,700,800,900,1000,2000]
    for inums in range(len(nmbr)):
        
        epochs = 20
        batch_size = bss[inums]
        inum = nmbr[inums]

        
        input_params = {
                        "material_": "ZrO2",             ## same key as used in dict_LaueTools
                        "material1_": "ZrO2",            ## same key as used in dict_LaueTools
                        "prefix" : "data",                 ## prefix for the folder to be created for training dataset
                        "symmetry": "monoclinic",           ## crystal symmetry of material_
                        "symmetry1": "monoclinic",          ## crystal symmetry of material1_
                        "SG": 14,                     ## Space group of material_ (None if not known)
                        "SG1": 14,                    ## Space group of material1_ (None if not known)
                        "hkl_max_identify" : 5,        ## Maximum hkl index to classify in a Laue pattern
                        "maximum_angle_to_search":90,  ## Angle of radial distribution to reconstruct the histogram (in deg)
                        "step_for_binning" : 0.1,      ## bin widht of angular radial distribution in degree
                        "nb_grains_per_lp" : 5,        ## max grains to be generated in a Laue Image
                        "grains_nb_simulate" : inum,    ## Number of orientations to generate (takes advantage of crystal symmetry)
                        ## Detector parameters (roughly) of the Experimental setup
                        ## Sample-detector distance, X center, Y center, two detector angles
                        "detectorparameters" :  [79.553,979.32,932.31,0.37,0.447], 
                        "pixelsize" : 0.0734,          ## Detector pixel size
                        "dim1":2018,                   ## Dimensions of detector in pixels
                        "dim2":2016,
                        "emin" : 5,                    ## Minimum and maximum energy to use for simulating Laue Patterns
                        "emax" : 22,
                        }

        input_params["prefix"] = "_" + str(input_params["grains_nb_simulate"]) + input_params["prefix"]
        text_file_logger.write(input_params["prefix"] + "\n")
        material_= input_params["material_"]
        material1_= input_params["material1_"]
        n = input_params["hkl_max_identify"]
        maximum_angle_to_search = input_params["maximum_angle_to_search"]
        step_for_binning = input_params["step_for_binning"]
        nb_grains_per_lp = input_params["nb_grains_per_lp"]
        grains_nb_simulate = input_params["grains_nb_simulate"]
        detectorparameters = input_params["detectorparameters"]
        pixelsize = input_params["pixelsize"]
        emax = input_params["emax"]
        emin = input_params["emin"]
        symm_ = input_params["symmetry"]
        symm1_ = input_params["symmetry1"]
        SG = input_params["SG"]
        SG1 = input_params["SG1"]

        if material_ != material1_:
            save_directory = os.getcwd()+"//"+material_+"_"+material1_+input_params["prefix"]
        else:
            save_directory = os.getcwd()+"//"+material_+input_params["prefix"]
        print("save directory is : "+save_directory)
        if not os.path.exists(save_directory):
            os.makedirs(save_directory)

        ## get unit cell parameters and other details required for simulating Laue patterns
        rules, symmetry, lattice_material, \
            crystal, SG, rules1, symmetry1,\
            lattice_material1, crystal1, SG1 = get_material_detail(material_, SG, symm_,
                                                                   material1_, SG1, symm1_)

        ## End of data generation for Neural network training: all files are saved in the same folder to be later used for training and prediction
        import numpy as np
        import _pickle as cPickle
        if material_ != material1_:
            prefix1 = material_+"_"+material1_
        else:
            prefix1 = material_
        ## load model related files and generate the model
        json_file = open(save_directory+"//model_"+prefix1+".json", 'r')
        classhkl = np.load(save_directory+"//MOD_grain_classhkl_angbin.npz")["arr_0"]
        angbins = np.load(save_directory+"//MOD_grain_classhkl_angbin.npz")["arr_1"]
        ind_mat = None
        ind_mat1 = None  
        load_weights = save_directory + "//model_"+prefix1+".h5"
        wb = read_hdf5(load_weights)
        temp_key = list(wb.keys())

        # # load json and create model
        loaded_model_json = json_file.read()
        json_file.close()
        model = model_from_json(loaded_model_json)
        print("Constructing model")
        model.load_weights(load_weights)
        print("Uploading weights to model")
        print("All model files found and loaded")

        import tensorflow as tf
        from tensorflow.keras.layers import BatchNormalization
        import keras
        from keras.regularizers import l2
        from keras.models import Sequential
        from keras.layers import Dense, Activation, Dropout
        from keras.constraints import maxnorm


        from lattice import HklPlane
        

        def commonclass(hkl1, hkl2, lattice_material, symmetry_):
            """ test if hkl1 and hkl2 belong to the same class"""
            h,k,l = hkl1
            h1,k1,l1 = hkl2
            h_obj = HklPlane(h,k,l, lattice=lattice_material)
            normal = np.round(h_obj.normal(), 6)

            h_obj1 = HklPlane(h1,k1,l1, lattice=lattice_material)
            family = h_obj1.get_family(h_obj1.miller_indices(), lattice=h_obj1._lattice, 
                                       include_friedel_pairs=True, crystal_structure=symmetry_)

            normals = np.array([np.round(ijk.normal(),6) for ijk in family])

            cond1 = h_obj in family
            cond2 = np.any(np.all(normal == normals, axis=1))
            return cond1 or cond2


        ### verify prediction
        import time
        from fast_histogram import histogram1d
        import random

        length = []
        nbgrains = 5
        nbgrains1 = 5
        material1_ = material_
        total_prediction = 0
        count_good = 0
        count_bad = 0
        for _ in trange(50):
            noisy_data = bool(random.getrandbits(1)) 
            remove_peaks = bool(random.getrandbits(1)) 
            seednumber = np.random.randint(1e6)
            tabledistancerandom, hkl_sol, \
                    s_posx, s_posy, s_I, s_tth, s_chi, g, g1  = prepare_LP_NB(nbgrains, nbgrains1,
                                                                            material_, 0,
                                                                            material1_ = material1_,
                                                                            seed = seednumber,sortintensity=True,
                                                                            detectorparameters=[79.553,979.32,932.31,0.37,0.447], 
                                                                            pixelsize=0.0734,
                                                                            dim1=2018, dim2=2016, 
                                                                            emin=5, emax=23,
                                                                            flag = 10, noisy_data=noisy_data,
                                                                            remove_peaks = remove_peaks)
            length.append(len(s_posx))

            ## Not all spots needs to be recognized for now
            spots_in_center = np.arange(0,len(s_tth))

            start_time = time.time()
            codebars_all = []
            #print("Spots in center: ",len(spots_in_center))
            for i in spots_in_center: ## identify the center HKL spots 
                spotangles = tabledistancerandom[i]
                spotangles = np.delete(spotangles, i)# removing the self distance
                # codebars = np.histogram(spotangles, bins=angbins)[0]
                codebars = histogram1d(spotangles, range=[min(angbins),max(angbins)], bins=len(angbins)-1)
                ## normalize the same way as training data
                max_codebars = np.max(codebars)
                codebars = codebars/ max_codebars
                codebars_all.append(codebars)
            ## reshape for the model to predict all spots at once
            codebars = np.array(codebars_all)
            ## Do prediction of all spots at once
            prediction = model.predict(codebars)
            max_pred = np.max(prediction, axis = 1)
            class_predicted = np.argmax(prediction, axis = 1)
            ## verify prediction (Assuming we dont know HKL solution)
            for i, gg in enumerate(class_predicted):

                if np.any(np.abs(hkl_sol[spots_in_center[i],:3]) > 5):
                    continue

                total_prediction += 1
                isgoodprediction = commonclass(classhkl[gg], hkl_sol[spots_in_center[i],:3], lattice_material, symmetry)
                if isgoodprediction:
                    count_good += 1
                else:
                    #print("Predicted HKL: ", classhkl[gg], " Actual HKL: ", hkl_sol[spots_in_center[i],:3])
                    count_bad += 1
            #print("Seed number for LP:",seednumber)

            end_time = time.time()
        length = np.array(length)
        print("****************************************************************************")
        print("Total spots attempted:",total_prediction)
        print("Total Good prediction:",count_good)
        print("Total Bad prediction:",count_bad)
        print(material_, np.average(length), np.std(length))
        
        text_file_logger.write("Total spots attempted:" +str(total_prediction) + "\n")
        text_file_logger.write("Total Good prediction:" +str(count_good) + "\n")
        text_file_logger.write("Total Bad prediction:" +str(count_bad) + "\n")
        text_file_logger.write(material_ +", "+ str(np.average(length))+", "+ str(np.std(length)) + "\n")
        
    text_file_logger.close()<jupyter_output>save directory is : C:\Users\purushot\Anaconda3\envs\laueNN\Lib\site-packages\lauetoolsnn\example_notebook_scripts//ZrO2_10data
Constructing model
Uploading weights to model
All model files found and loaded
{'fraction_non_alphanumeric': 0.0999056010069226, 'fraction_numerical': 0.01885357667295993, 'mean_word_length': 1.9766625039025914, 'pattern_counts': {'":': 18, '<': 5, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 74, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '326990', 'license': 'MIT', 'n_tokens_mistral': 11035, 'n_tokens_neox': 10274, 'n_words': 2428, 'path': 'lauetoolsnn/example_notebook_scripts/Effect_of_training_dataset.ipynb', 'repo_name': 'ravipurohit1991/lauetoolsnn', 'size': '38136'}