<commit_before>#! /bin/bash


# TMP sourcing for Sepal env.
source /data/home/Andreas.Vollrath/github/OpenSARKit_source.bash

#----------------------------------------------------------------------
#	0 Set up Script variables
#----------------------------------------------------------------------

# 	0.1 Check for right usage
if [ "$#" != "1" ]; then
  echo -e "Usage: osk_ALOS_merge_path /path/to/satellite/tracks"
  echo -e "The path will be your Project folder!"
  exit 1
else
  cd $1
  export PROC_DIR=`pwd`
  echo "Welcome to OpenSARKit!"
  echo "Processing folder: ${PROC_DIR}"
fi

# Number of CPUs (for SAGA GIS)
CPU=`lscpu | grep "CPU(s):" | awk $'{print $2}' | head -1`

# Path structure
cd ${PROC_DIR}
TMP_DIR=${PROC_DIR}/TMP
mkdir -p ${TMP_DIR}
mkdir -p ${PROC_DIR}/PATH_MOSAICS
mkdir -p ${PROC_DIR}/PATH_SHP_OUTLINES
mkdir -p ${PROC_DIR}/PATH_KMZS

# Loop for Acquisition Date
for ACQ_DATE in `ls -1 -d [1,2]*`;do 

	cd ${ACQ_DATE}

	# Loop for Frames
	for FRAME in `ls -1 -d [0-9]*`;do 
		

		cd $FRAME
		ORBIT=${FRAME:5:10}
	
		echo "Translate to SAGA GIS format (powerful and fast raster manipulation)"				
		gdalwarp -multi -wo NUM_THREADS=ALL_CPUS -t_srs EPSG:4326 -tr 0.000138889 0.000138889 -overwrite -of SAGA -srcnodata "0" -dstnodata "-99999" Gamma0_HH.tif ${TMP_DIR}/${FRAME}"_Gamma0_HH_saga.sdat"
		gdalwarp -multi -wo NUM_THREADS=ALL_CPUS -t_srs EPSG:4326 -tr 0.000138889 0.000138889 -overwrite -of SAGA -srcnodata "0" -dstnodata "-99999" Gamma0_HV.tif ${TMP_DIR}/${FRAME}"_Gamma0_HV_saga.sdat"
	
		echo ${TMP_DIR}/${FRAME}"_Gamma0_HH_saga.sgrd" >> ${TMP_DIR}/Gamma0_HH_db_list
		echo ${TMP_DIR}/${FRAME}"_Gamma0_HV_saga.sgrd" >> ${TMP_DIR}/Gamma0_HV_db_list
	
		cd ../
	
	done

	LIST_HH=`cat ${TMP_DIR}/Gamma0_HH_db_list | tr '\n' ';' | rev | cut -c 2- | rev`
	saga_cmd -c=${CPU} grid_tools 3 -GRIDS:"${LIST_HH}" -TYPE:7 -OVERLAP:4 -BLEND_DIST:0 -TARGET_OUT_GRID:${PROC_DIR}/PATH_MOSAICS/${ACQ_DATE}"_Gamma0_HH_"${ORBIT}.sgrd

	LIST_HV=`cat ${TMP_DIR}/Gamma0_HV_db_list | tr '\n' ';' | rev | cut -c 2- | rev`
	saga_cmd -c=${CPU} grid_tools 3 -GRIDS:"${LIST_HV}" -TYPE:7 -OVERLAP:4 -BLEND_DIST:0 -TARGET_OUT_GRID:${PROC_DIR}/PATH_MOSAICS/${ACQ_DATE}"_Gamma0_HV_"${ORBIT}.sgrd
	
	echo "creating a shapefile for all valid points"
	gdal_trace_outline ${ACQ_DATE}"_Gamma0_HV_"${ORBIT}.sdat -ndv -99999 -dp-toler 0 -out-cs en -ogr-out ${PROC_DIR}/PATH_OUTLINES/${ACQ_DATE}"_Gamma0_HV_"${ORBIT}.shp
	echo "creating a KMZ file for GoogleEarth"
	gdal_translate -of KMLSUPEROVERLAY -a_nodata 0 -outsize 25% 25% -scale 0.001 0.1 0 255 ${PROC_DIR}/PATH_MOSAICS/${ACQ_DATE}"_Gamma0_HV_"${ORBIT}.sgrd  ${PROC_DIR}/PATH_KMZS/${ACQ_DATE}"_Gamma0_HV_"${ORBIT}.kmz -co FORMAT=JPEG

	rm -rf ${TMP_DIR}/*
	cd ${PROC_DIR}
	
done

cd ${PROC_DIR}
<commit_msg>Updating<commit_after>#! /bin/bash

# dependecies
#	SAGA GIS (Version > 2.1.4)
#  dans-gdal-scripts
#  gdal-libs

# TMP sourcing for Sepal env.
source /data/home/Andreas.Vollrath/github/OpenSARKit_source.bash

#----------------------------------------------------------------------
#	0 Set up Script variables
#----------------------------------------------------------------------

# 	0.1 Check for right usage
if [ "$#" != "1" ]; then
  echo -e "Usage: osk_ALOS_merge_path /path/to/satellite/tracks"
  echo -e "The path will be your Project folder!"
  exit 1
else
  cd $1
  export PROC_DIR=`pwd`
  echo "Welcome to OpenSARKit!"
  echo "Processing folder: ${PROC_DIR}"
fi

# Number of CPUs (for SAGA GIS)
CPU=`lscpu | grep "CPU(s):" | awk $'{print $2}' | head -1`

# Path structure
cd ${PROC_DIR}
TMP_DIR=${PROC_DIR}/TMP
mkdir -p ${TMP_DIR}
mkdir -p ${PROC_DIR}/PATH_MOSAICS
mkdir -p ${PROC_DIR}/PATH_SHP_OUTLINES
mkdir -p ${PROC_DIR}/PATH_KMZS

# Loop for Acquisition Date
for ACQ_DATE in `ls -1 -d [1,2]*`;do 

	cd ${ACQ_DATE}

	# Loop for Frames
	for FRAME in `ls -1 -d [0-9]*`;do 
		

		cd $FRAME
		ORBIT=${FRAME:5:10}
	
		echo "Translate to SAGA GIS format (powerful and fast raster manipulation)"				
		gdalwarp -multi -wo NUM_THREADS=ALL_CPUS -t_srs EPSG:4326 -tr 0.000138889 0.000138889 -overwrite -of SAGA -srcnodata "0" -dstnodata "-99999" Gamma0_HH.tif ${TMP_DIR}/${FRAME}"_Gamma0_HH_saga.sdat"
		gdalwarp -multi -wo NUM_THREADS=ALL_CPUS -t_srs EPSG:4326 -tr 0.000138889 0.000138889 -overwrite -of SAGA -srcnodata "0" -dstnodata "-99999" Gamma0_HV.tif ${TMP_DIR}/${FRAME}"_Gamma0_HV_saga.sdat"
	
		echo ${TMP_DIR}/${FRAME}"_Gamma0_HH_saga.sgrd" >> ${TMP_DIR}/Gamma0_HH_db_list
		echo ${TMP_DIR}/${FRAME}"_Gamma0_HV_saga.sgrd" >> ${TMP_DIR}/Gamma0_HV_db_list
	
		cd ../
	
	done

	LIST_HH=`cat ${TMP_DIR}/Gamma0_HH_db_list | tr '\n' ';' | rev | cut -c 2- | rev`
	saga_cmd -c=${CPU} grid_tools 3 -GRIDS:"${LIST_HH}" -TYPE:7 -OVERLAP:4 -BLEND_DIST:0 -TARGET_OUT_GRID:${PROC_DIR}/PATH_MOSAICS/${ACQ_DATE}"_Gamma0_HH_"${ORBIT}.sgrd

	LIST_HV=`cat ${TMP_DIR}/Gamma0_HV_db_list | tr '\n' ';' | rev | cut -c 2- | rev`
	saga_cmd -c=${CPU} grid_tools 3 -GRIDS:"${LIST_HV}" -TYPE:7 -OVERLAP:4 -BLEND_DIST:0 -TARGET_OUT_GRID:${PROC_DIR}/PATH_MOSAICS/${ACQ_DATE}"_Gamma0_HV_"${ORBIT}.sgrd
	
	echo "creating a shapefile for all valid points"
	gdal_trace_outline ${ACQ_DATE}"_Gamma0_HV_"${ORBIT}.sdat -ndv -99999 -dp-toler 0 -out-cs en -ogr-out ${PROC_DIR}/PATH_OUTLINES/${ACQ_DATE}"_Gamma0_HV_"${ORBIT}.shp
	echo "creating a KMZ file for GoogleEarth"
	gdal_translate -of KMLSUPEROVERLAY -a_nodata 0 -outsize 25% 25% -scale 0.001 0.1 0 255 ${PROC_DIR}/PATH_MOSAICS/${ACQ_DATE}"_Gamma0_HV_"${ORBIT}.sdat  ${PROC_DIR}/PATH_KMZS/${ACQ_DATE}"_Gamma0_HV_"${ORBIT}.kmz -co FORMAT=PNG

	rm -rf ${TMP_DIR}/*
	cd ${PROC_DIR}
	
done

cd ${PROC_DIR}

{'fraction_non_alphanumeric': 0.202893373816753, 'fraction_numerical': 0.04482943382746919, 'mean_word_length': 3.4374009508716323, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '892660', 'n_tokens_mistral': 2625, 'n_tokens_neox': 2362, 'n_words': 507}