<jupyter_start><jupyter_text>Script to move raw MACS files based on extracted footprints files **author:** <NAME>, <NAME>**contact:** <EMAIL>**version date:** 2021-10-22  Imports<jupyter_code>import geopandas as gpd
import shutil
import os
import glob
import pandas as pd
from IPython.display import clear_output
import sys
import numpy as np
import tqdm
import zipfile
from pathlib import Path
from joblib import delayed, Parallel
pwd = Path(os.getcwd())<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code>def prepare_df_for_mipps(path_footprints, path_infiles):
    # Load filtered footprints files
    df = gpd.read_file(path_footprints)
    
    flist = glob.glob(path_infiles + '/*/*.macs')
    
    df_full = pd.DataFrame()
    df_full['full_path'] = flist
    df_full['basename'] = pd.DataFrame(df_full['full_path'].apply(lambda x: os.path.basename(x)))
    # return Inner join of lists - create filtered list of filepaths 
    return df.set_index('Basename').join(df_full.set_index('basename'))

def write_exif(outdir, tag, exifpath):
    s = f'{exifpath} -overwrite_original -Model="{tag}" {outdir}'
    print(s)
    os.system(s)<jupyter_output><empty_output><jupyter_text>Settings * prefer full/absolute paths* Create processing template automatically  Variable Settings<jupyter_code># Set project directory here, where you want to process your data
PROJECT_DIR = r'D:\pix4d\Cape_Blossom_2021-06-25' # SET Project output

# Set raw data dir here for the speicific image acquisition project
path_infiles = r'N:\response\Restricted_Airborne\MACs\2021_Perma-X_Alaska\01_raw_data\20210626-014401_52_BP_CapeBlossom_1500m'<jupyter_output><empty_output><jupyter_text>Fixed Settings<jupyter_code>#CODE_DIR = r'D:\pix4d_Processing\code\MACS_tools'
CODE_DIR = pwd
MIPPS_DIR = r'C:\Program Files\DLR MACS-Box\bin'
MIPPS_BIN = r'..\tools\MACS\mipps.exe'
EXIF_PATH = Path(CODE_DIR / Path(r'exiftool\exiftool.exe'))
mipps_script_nir = r'mipps_scripts\33552_all_taps_2018-09-26_12-58-15_modelbased.mipps'
mipps_script_right = r'mipps_scripts\33576_all_taps_2018-09-26_13-13-43_modelbased.mipps'
mipps_script_left = r'mipps_scripts\33577_all_taps_2018-09-26_13-21-24_modelbased.mipps'

mipps_script_nir = pwd / mipps_script_nir
mipps_script_right = pwd / mipps_script_right
mipps_script_left = pwd / mipps_script_left

Path(DATA_DIR / Path('33576_Right')).exists
DATA_DIR = Path(PROJECT_DIR / Path(r'01_rawdata\tif'))
outdir = {'right': DATA_DIR / Path('33576_Right'),
          'left':DATA_DIR / Path('33577_Left'),
          'nir':DATA_DIR / Path('33552_NIR')}
tag = {'right':'MACS_RGB_Right_33576',
       'left':'MACS_RGB_Left_33577',
       'nir':'MACS_NIR_33552'}
# Path of filtered footprints file (.shp file)
path_footprints = os.path.join(PROJECT_DIR, '02_studysites','footprints.shp')
outdir = os.path.join(PROJECT_DIR, '01_rawdata','tif')<jupyter_output><empty_output><jupyter_text>Prepare processing dir * check if exists<jupyter_code>zippath = os.path.join(CODE_DIR, 'processing_folder_structure_template.zip')
nav_script_path = os.path.join(CODE_DIR, 'pix4dnav.py')
with zipfile.ZipFile(zippath, 'r') as zip_ref:
    zip_ref.extractall(PROJECT_DIR)
shutil.copy(nav_script_path, outdir)<jupyter_output><empty_output><jupyter_text>Manual Selection of footprints  Now select footprints and export selection as ***footprints.shp*** to ***02_footprints*** in your working directory  Load filtered footprints file<jupyter_code>df_final = prepare_df_for_mipps(path_footprints, path_infiles)
print("Total number of images:", len(df_final))
print("NIR images:", (df_final['Looking'] == 'center').sum())
print("RGB right images:", (df_final['Looking'] == 'right').sum())
print("RGB left images:", (df_final['Looking'] == 'left').sum())<jupyter_output><empty_output><jupyter_text>Run Process<jupyter_code>os.chdir(MIPPS_DIR)
max_roll = 3 # Select maximum roll angle to avoid image issues
chunksize = 20
looking = 'center'
q = (np.abs(df_final['Roll[deg]']) < max_roll) & (df_final['Looking'] == looking)
df_nir = df_final[q]
print(len(df_nir))
for df in tqdm.tqdm_notebook(np.array_split(df_nir, len(df_nir) // chunksize)):
    outlist = ' '.join(df['full_path'].values[:])
    s = f'{MIPPS_BIN} -c={mipps_script_nir} -o={outdir} -j=4 {outlist}'
    os.system(s)
looking = 'right'
q = (np.abs(df_final['Roll[deg]']) < max_roll) & (df_final['Looking'] == looking)
df_right = df_final[q]
for df in tqdm.tqdm_notebook(np.array_split(df_right, len(df_right) // chunksize)):
    outlist = ' '.join(df['full_path'].values[:])
    s = f'{MIPPS_BIN} -c={mipps_script_right} -o={outdir} -j=4 {outlist}'
    os.system(s)
looking = 'left'
q = (np.abs(df_final['Roll[deg]']) < max_roll) & (df_final['Looking'] == looking)
df_left = df_final[q]
for df in tqdm.tqdm_notebook(np.array_split(df_left, len(df_left) // chunksize)):
    outlist = ' '.join(df['full_path'].values[:])
    s = f'{MIPPS_BIN} -c={mipps_script_left} -o={outdir} -j=4 {outlist}'
    os.system(s)<jupyter_output><empty_output><jupyter_text>Write exif information into all images<jupyter_code>for sensor in ['right', 'left', 'nir']:
    %time write_exif(outdir[sensor], tag[sensor], EXIF_PATH)<jupyter_output><empty_output><jupyter_text>Nav<jupyter_code>list(Path(path_infiles).glob('*nav.txt'))[0]
navfile = list(Path(path_infiles).glob('*nav.txt'))[0]
shutil.copy(navfile, outdir['nir'].parent / 'nav.txt')
os.chdir(outdir['nir'].parent)
os.system('python pix4dnav.py')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14719327116474676, 'fraction_numerical': 0.02815871274456025, 'mean_word_length': 6.493150684931507, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '348999', 'license': 'FTL', 'n_tokens_mistral': 2212, 'n_tokens_neox': 2097, 'n_words': 362, 'path': 'Prepare_MACS_Processing.ipynb', 'repo_name': 'awi-response/MACS_tools', 'size': '5497'}