<jupyter_start><jupyter_text>Params and Init<jupyter_code>print("Importing    ")
!pip -q install --upgrade pip
# !pip -q install --upgrade tensorflow keras
!pip -q install plyfile deepdiff talos GitPython numba
!pip -q install --upgrade imgaug
import talos
from deepdiff import DeepDiff
import re
import numpy as np
from progressbar import progressbar, ProgressBar
import sys
import os
import cv2
import math
import json
import importlib
import matplotlib.pyplot as plt
import tensorflow as tf
import git
from tensorflow import keras
import keras.backend as K
import datetime
import imgaug.augmenters as iaa
cuda_paths=!ls /usr/local | grep cuda-
os.environ['CUDA_HOME'] = os.path.join("/usr/local/",cuda_paths[-1])
K.manual_variable_initialization(True)
print("Importing deep_ga")
if using_colab:
    !git -C /content/deep_ga/ pull || git clone https://github.com/InigoMoreno/deep_ga
    sys.path.append('/content/deep_ga')
    import deep_ga  # nopep8
    deep_ga=importlib.reload(deep_ga)
else:
    MODULE_PATH = "deep_ga/deep_ga/__init__.py"
    MODULE_NAME = "deep_ga"
    git.cmd.Git(MODULE_NAME).pull()
    spec = importlib.util.spec_from_file_location(MODULE_NAME, MODULE_PATH)
    deep_ga = importlib.util.module_from_spec(spec)
    sys.modules[spec.name] = deep_ga
    spec.loader.exec_module(deep_ga)

p={
  "resolution" : .5,                     # resolution of map [meters per pixel]
  "mapLength"  : 40,                     # size of one side of the map [meters]
  "minSlopeThreshold"  : 0.5,            # minimum slope to be counted [proportion]
  "maxNanPercentage"   : 5/100,          # maximum percentage of NaNs in a patch [%]
  "minSlopePercentage" : 0/100,          # minimum percentage of slope in a patch [%]
  "maxSlopePercentage" : 100/100,        # maximum percentage of slope in a patch [%]
  "stdPatchShift"          : 15,         # standard deviation of shift between to patches [m]
  "local_resolution" : .5,
  "local_mapLength"  : 40,
  "local_maxNanPercentage": 65/100,
  "local_global_minCcorr": 15/100,
   "booleanDist": False
   
}
p["mapLengthPixels"]=math.ceil(p["mapLength"]/p["resolution"])
p["local_mapLengthPixels"]=math.ceil(p["local_mapLength"]/p["local_resolution"])
deep_ga.set_scale(p["stdPatchShift"])

p["augment_a"] = iaa.Sequential([
    iaa.Add((-4, 4)),
    iaa.PerspectiveTransform(scale=0.01),
    iaa.BlendAlphaSimplexNoise(iaa.Add(0.2), upscale_method="cubic", size_px_max=4),
])

hyperparams = {
    # "input": ["raw", "rawmask", "sobel", "fixsobel", "PConv", "SymConv", "Conv"],
    "input": ["PConv"],
    # "batch_size": [32, 64, 128, 256],
    "batch_size": [4],
    "mobileNet_alpha": [1.4,1.00,0.75,0.5,0.35],
    # "mobileNet_alpha": [0.35],
    # "mobileNet_pooling": ["avg", "max", None],
    "mobileNet_pooling": ["avg"],
    # "mobileNet_weights": ["imagenet"],
    "mobileNet_weights": [None],
    # "firstLayerSize": [4096, 1280, 1000],"
    "firstLayerSize": [0],
    # "dropout": [0, 0.2, 0.5],
    "dropout": [0.01],
    # "secondLayerSize": [1000, 500, 100],
    "secondLayerSize": [1000,500,100,50,0],
    # "activation": ["relu", None],
    "activation": ["relu"],
    "sharedWeights": [True],
    # "sharedWeights": [True],
    # "loss": ["MSE", "MAE", "DOOMSE", "PCL", "BCE"],
    "loss": ["DOOMSE"],
    # "arepeat": [1,2],
    # "los_scale": [1],
    "learnSize": [10],
    "optimizer": ["SGD"],
    "learning_rate": [0.0001],
    # "learning_rate": [0.003,0.0001,0.00003],
    # "learnEnding": [True, False],
    "learnEnding": [False],
    # "displace": [True, False],
    # "loadFolder": [os.path.join(drive_folder,
    #                             "talos_oxia_mobileNet_alpha__secondLayerSize")],
    # "loadBranch": [None, "1_0__0"]
}
# hyperparams["loadBranch"]=[file for file in os.listdir(hyperparams["loadFolder"][0]) if re.match(".*\.h5",file) and "weights" not in file]

keys= [k for k,v in hyperparams.items() if len(v)>1]
experiment_name="talos_tenerife_"+"__".join(keys)
experiment_path = os.path.relpath(os.path.join(drive_folder,experiment_name))
print(experiment_name)
first_hyperparams={k:v[0] for k,v in hyperparams.items()}<jupyter_output>talos_tenerife_mobileNet_alpha__secondLayerSize<jupyter_text>Model preparation<jupyter_code>values=[hyperparams[k] for k in keys]
params=first_hyperparams
print(" & "+ " & ".join([str(v) for v in values[1]])+" \\\\\\hline")
for i,v0 in enumerate(values[0]):
  params[keys[0]]=v0
  print(v0,end="")
  for j,v1 in enumerate(values[1]):
    params[keys[1]]=v1
    input_a = keras.Input(shape=(
        p["local_mapLengthPixels"], p["local_mapLengthPixels"]), name="patch_a")
    input_b = keras.Input(
        shape=(p["mapLengthPixels"], p["mapLengthPixels"]), name="patch_b")

    model = deep_ga.get_model(params,input_a,input_b)
    model = deep_ga.compile_model(model, 0, params)
    print(" & ",deep_ga.count(model), end="")
    # print(" & ",v1, end="")
  print(" \\\\")
import tensorflow as tf

# Convert Keras model to a tflite model
converter = tf.lite.TFLiteConverter.from_keras_model(model)
converter.optimizations = [tf.lite.Optimize.OPTIMIZE_FOR_SIZE]
# converter.representative_dataset = representative_data_gen
converter.target_spec.supported_ops = [
  tf.lite.OpsSet.SELECT_TF_OPS # enable TensorFlow ops.
]
tflite_model = converter.convert()

filename=os.path.join(drive_folder, "a.tflite")
open(filename, 'wb').write(tflite_model);
print(f"{os.path.getsize(filename)/pow(2,20):.2f}")
os.remove(filename)<jupyter_output>INFO:tensorflow:Assets written to: /tmp/tmp9lv7d5do/assets
{'fraction_non_alphanumeric': 0.16320260256641966, 'fraction_numerical': 0.030182541116934754, 'mean_word_length': 4.331406551059731, 'pattern_counts': {'":': 36, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '631018', 'license': 'MIT', 'n_tokens_mistral': 2145, 'n_tokens_neox': 2018, 'n_words': 404, 'path': 'notebooks/Data_usage.ipynb', 'repo_name': 'InigoMoreno/deep-ga', 'size': '5533'}