<commit_before>'''
Utilities to parse pyrate.conf config files. Includes numerous general PyRate
constants relating to options in config files.

Created on 17/09/2012
@author: <NAME>, NCI
'''

# TODO: add regex column to check if some values are within bounds? Potential
# problem with the checking being done in the middle of the runs, as bad values
# could cause crashes & destroying some of the results.

from orbital import NETWORK_METHOD, QUADRATIC


# general constants
NO_MULTILOOKING = 1

# constants for lookups
NUMBER_OF_SETS = 'nsets'
IFG_FILE_LIST = 'ifgfilelist'
OBS_DIR = 'obsdir'
OUT_DIR = 'outdir'
NUM_SETS = 'nsets'
SIM_DIR = 'simdir'
DEM_FILE = 'demfile'
AMPLITUDE_FLAG = 'ampflag'
PERP_BASELINE_FLAG = 'basepflag'
PROJECTION_FLAG = 'prjflag'

IFG_CROP_OPT = 'ifgcropopt' # 1=min, 2=max, 3=custom, 4=all ifgs already same size
IFG_LKSX = 'ifglksx'
IFG_LKSY = 'ifglksy'

IFG_XFIRST = 'ifgxfirst'
IFG_XLAST = 'ifgxlast'
IFG_YFIRST = 'ifgyfirst'
IFG_YLAST = 'ifgylast'

# reference pixel parameters
REFX = 'refx'
REFY = 'refy'
REFNX = "refnx"
REFNY = "refny"
REF_CHIP_SIZE = 'refchipsize'
REF_MIN_FRAC = 'refminfrac'

# orbital error correction/parameters
ORBITAL_FIT = 'orbfit' # numerical BOOL (1/0), do/don't do orbital correction
ORBITAL_FIT_METHOD = 'orbfitmethod'  # BOOL (1/2) 1: ifg by ifg/independent, 2: epoch by epoch
ORBITAL_FIT_DEGREE = 'orbfitdegrees' # BOOL (1/2) 1=planar, 2=quadratic
ORBITAL_FIT_LOOKS_X = 'orbfitlksx' # int of 1+, X multi looking factor
ORBITAL_FIT_LOOKS_Y = 'orbfitlksy' # int of 1+, Y multi looking factor
# ORBITAL_FIT_orbrefest:     1 BOOLEAN (1/0) # remove reference phase
# ORBITAL_FIT_ orbmaskflag:   1 BOOLEAN (1/0) # mask some patches for orbital correction

# Linear rate/stacking parameters
LR_NSIG = 'nsig' # n-sigma ratio used to threshold 'model minus observation' residuals
LR_PTHRESH = 'pthr'  # minimum number of coherent observations for a pixel
LR_MAXSIG = 'maxsig' # maximum allowable standard error

# Lookup to help convert args to correct type/defaults
# format is    key : (conversion, default value)
# None = no conversion
PARAM_CONVERSION = { OBS_DIR : (None, "obs"),
					IFG_FILE_LIST : (None, "ifg.list"),
					OUT_DIR : (None, "out"),
					DEM_FILE : (str, None),
					PERP_BASELINE_FLAG : (bool, True),
					AMPLITUDE_FLAG : (bool, False),
					NUM_SETS : (int, 1),
					IFG_CROP_OPT : (int, None),
					IFG_LKSX : (int, NO_MULTILOOKING),
					IFG_LKSY : (int, NO_MULTILOOKING),
					IFG_XFIRST : (float, None),
					IFG_XLAST : (float, None),
					IFG_YFIRST : (float, None),
					IFG_YLAST : (float, None),
					PROJECTION_FLAG : (int, 3),

					REFX : (int, -1),
					REFY : (int, -1),
					REFNX : (int, None), # was 50 in original Pirate code
					REFNY : (int, None), # was 50 in original Pirate code
					REF_CHIP_SIZE : (int, None), # defaults to 21 in orig
					REF_MIN_FRAC : (float, 0.8), # uses Pirate default

					ORBITAL_FIT : (bool, True),
					ORBITAL_FIT_METHOD : (int, NETWORK_METHOD),
					ORBITAL_FIT_DEGREE : (int, QUADRATIC),
					ORBITAL_FIT_LOOKS_X : (int, NO_MULTILOOKING),
					ORBITAL_FIT_LOOKS_Y : (int, NO_MULTILOOKING),

					LR_NSIG : (int, 3), # Pirate default
					LR_PTHRESH : (int, 20), # should be based on nepochs since not every project may have 20 epochs
					LR_MAXSIG : (int, 2), # Pirate default
				}

def get_config_params(path):
	with open(path) as f:
		txt = f.read().splitlines()

	return _parse_conf_file(txt)

def _parse_conf_file(content):
	"""Returns a dict for the key:value pairs from the .conf file"""

	def is_valid(line):
		return line != "" and line[0] not in "%#"

	lines = [line.split() for line in content if is_valid(line)]

	# convert "field:   value" lines to [field, value]  
	kvpair = [(e[0].rstrip(":"), e[1]) for e in lines if len(e) == 2]
	parameters = dict(kvpair)
	_parse_pars(parameters)
	return parameters


def _parse_pars(pars):
	"""Parses and converts config file params from text"""
	for k in PARAM_CONVERSION.keys():
		if pars.has_key(k):
			conversion_func = PARAM_CONVERSION[k][0]
			if conversion_func:<commit_msg>Config doco cleanup.<commit_after>'''
Utilities to parse pyrate.conf config files. Includes numerous general PyRate
constants relating to options in config files.

Created on 17/09/2012
@author: <NAME>, NCI
'''

# TODO: add regex column to check if some values are within bounds? Potential
# problem with the checking being done in the middle of the runs, as bad values
# could cause crashes & destroying some of the results.

from orbital import NETWORK_METHOD, QUADRATIC


# general constants
NO_MULTILOOKING = 1

# constants for lookups
NUMBER_OF_SETS = 'nsets'
IFG_FILE_LIST = 'ifgfilelist'
OBS_DIR = 'obsdir'
OUT_DIR = 'outdir'
NUM_SETS = 'nsets'
SIM_DIR = 'simdir'
DEM_FILE = 'demfile'
AMPLITUDE_FLAG = 'ampflag'
PERP_BASELINE_FLAG = 'basepflag'
PROJECTION_FLAG = 'prjflag'

IFG_CROP_OPT = 'ifgcropopt' # 1=min, 2=max, 3=custom, 4=all ifgs already same size
IFG_LKSX = 'ifglksx'
IFG_LKSY = 'ifglksy'

IFG_XFIRST = 'ifgxfirst'
IFG_XLAST = 'ifgxlast'
IFG_YFIRST = 'ifgyfirst'
IFG_YLAST = 'ifgylast'

# reference pixel parameters
REFX = 'refx'
REFY = 'refy'
REFNX = "refnx"
REFNY = "refny"
REF_CHIP_SIZE = 'refchipsize'
REF_MIN_FRAC = 'refminfrac'

# orbital error correction/parameters
ORBITAL_FIT = 'orbfit' # numerical BOOL (1/0), do/don't do orbital correction
ORBITAL_FIT_METHOD = 'orbfitmethod'  # BOOL (1/2) 1: ifg by ifg/independent, 2: epoch by epoch
ORBITAL_FIT_DEGREE = 'orbfitdegrees' # BOOL (1/2) 1=planar, 2=quadratic
ORBITAL_FIT_LOOKS_X = 'orbfitlksx' # int of 1+, X multi looking factor
ORBITAL_FIT_LOOKS_Y = 'orbfitlksy' # int of 1+, Y multi looking factor
# ORBITAL_FIT_orbrefest:     1 BOOLEAN (1/0) # remove reference phase
# ORBITAL_FIT_ orbmaskflag:   1 BOOLEAN (1/0) # mask some patches for orbital correction

# Linear rate/stacking parameters
LR_NSIG = 'nsig' # n-sigma ratio used to threshold 'model minus observation' residuals
LR_PTHRESH = 'pthr'  # minimum number of coherent observations for a pixel
LR_MAXSIG = 'maxsig' # maximum allowable standard error

# Lookup to help convert args to correct type/defaults
# format is    key : (conversion, default value)
# None = no conversion
PARAM_CONVERSION = { OBS_DIR : (None, "obs"),
					IFG_FILE_LIST : (None, "ifg.list"),
					OUT_DIR : (None, "out"),
					DEM_FILE : (str, None),
					PERP_BASELINE_FLAG : (bool, True),
					AMPLITUDE_FLAG : (bool, False),
					NUM_SETS : (int, 1),
					IFG_CROP_OPT : (int, None),
					IFG_LKSX : (int, NO_MULTILOOKING),
					IFG_LKSY : (int, NO_MULTILOOKING),
					IFG_XFIRST : (float, None),
					IFG_XLAST : (float, None),
					IFG_YFIRST : (float, None),
					IFG_YLAST : (float, None),
					PROJECTION_FLAG : (int, 3),

					REFX : (int, -1),
					REFY : (int, -1),
					REFNX : (int, None), # was 50 in original Pirate code
					REFNY : (int, None), # was 50 in original Pirate code
					REF_CHIP_SIZE : (int, None), # defaults to 21 in orig
					REF_MIN_FRAC : (float, 0.8), # uses Pirate default

					ORBITAL_FIT : (bool, True),
					ORBITAL_FIT_METHOD : (int, NETWORK_METHOD),
					ORBITAL_FIT_DEGREE : (int, QUADRATIC),
					ORBITAL_FIT_LOOKS_X : (int, NO_MULTILOOKING),
					ORBITAL_FIT_LOOKS_Y : (int, NO_MULTILOOKING),

					LR_NSIG : (int, 3), # Pirate default
					LR_PTHRESH : (int, 20), # should be based on nepochs since not every project may have 20 epochs
					LR_MAXSIG : (int, 2), # Pirate default
				}

def get_config_params(path):
	'Returns a dict for the key:value pairs from the .conf file'
	with open(path) as f:
		txt = f.read().splitlines()

	return _parse_conf_file(txt)

def _parse_conf_file(content):
	'Parser for converting text content into a dict of parameters'
	def is_valid(line):
		return line != "" and line[0] not in "%#"

	lines = [line.split() for line in content if is_valid(line)]

	# convert "field:   value" lines to [field, value]  
	kvpair = [(e[0].rstrip(":"), e[1]) for e in lines if len(e) == 2]
	parameters = dict(kvpair)
	_parse_pars(parameters)
	return parameters


def _parse_pars(pars):
	"""Parses and converts config file params from text"""
	for k in PARAM_CONVERSION.keys():
		if pars.has_key(k):
			conversion_func = PARAM_CONVERSION[k][0]
			if conversion_func:
{'fraction_non_alphanumeric': 0.11158432708688246, 'fraction_numerical': 0.013141883669992699, 'mean_word_length': 4.1562107904642405, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2047146', 'n_tokens_mistral': 3470, 'n_tokens_neox': 3073, 'n_words': 907}