<commit_before>import os
import sys
import fnmatch
import subprocess
import datetime
import argparse
import logging
import re


def isLeapYear(year):
	'''Returns TRUE if the given year (int) is leap and FALSE otherwise'''
	leapyear = False
	if year % 4 != 0:
		leapyear = False
	elif year % 100 != 0:
		leapyear = True
	elif year % 400 == 0:
		leapyear = True
	else:
		leapyear = False
	return leapyear


def listFiles(path, pattern):
	res = list()
	for path, subdirs, files in os.walk(path):
		for name in files:
			filepath = os.path.join(path, name)
			if(re.match(pattern, filepath, flags=0)):
				res.append(filepath)
	return res

	
def testGribModis2SciDB():
	'''Test if GRibeiro's modis2scidb is installed'''
	res = True
	try:
		subprocess.check_output(["modis2scidb", "--h"])
	except:
		res = False
	return res

	
def date2grid(dateFileName, period, startyear):
	'''Return an time index (timid) from the input date (MODIS DOY) and time period (e.g 8 days). '''
	res = -1
	if period > 0:
		dateYYYYDOY = dateFileName[1:] #Remove the A precedding the date
		year = int(dateYYYYDOY[0:4])
		doy = int(dateYYYYDOY[4:7])
		ppy = int(365 / period) + 1 # Periods per year
		if(period > 0 and (doy - 1) % period == 0):
			idd = (doy - 1) / period
			idy = (year - startyear) * ppy
			res = idy + idd
		else:
			logging.error("date2grid: Invalid date")
	elif period == -319980101: # Monthly - given as YYYYMMDD i.e 19980101, 19980201, 19980301
		dateYYYYMMDD = dateFileName
		year = int(dateYYYYMMDD[0:4])
		mm = int(dateYYYYMMDD[4:6])
		#dd = int(dateYYYYMMDD[6:8])
		idy = (year - startyear) * 12
		idd = mm - 1
		res = idy + idd
	return res	


#********************************************************
# MAIN
#********************************************************
def main(argv):
	t0 = datetime.datetime.now()
	parser = argparse.ArgumentParser(description = "Exports MODIS-HDFs to binary files for uploading to SCIDB using GRibeiro's tool")
	parser.add_argument("hdfFile", help = "Path to the HDF")
	parser.add_argument("loadFolder", help = "Folder from where the binary files are uploaded to SCIDB")
	parser.add_argument("product", help = "Product. e.g MOD09Q1")
	parser.add_argument("--log", help = "Log level. Default = WARNING", default = 'WARNING')
	#Get paramters
	args = parser.parse_args()
	hdfFile = os.path.join(args.hdfFile, '')
	loadFolder = os.path.join(args.loadFolder, '')
	product = args.product
	log = args.log
	####################################################
	# CONFIG
	####################################################
	prodList = ['MOD09Q1', 'MOD13Q1', 'TRMM3B43']
	prodTemporalResolution = {
		'MOD09Q1':	8,
		'MOD13Q1':	16,
		'TRMM3B43':	-319980101
	}
	prodStartYear = {
		'MOD09Q1':	2000,
		'MOD13Q1':	2000,
		'TRMM3B43':	1998
	}
	prodBands = {
		'MOD09Q1':	'0,1,2',
		'MOD13Q1':	'0,1,2,3,4,5,6,7,8,9,10,11',
		'TRMM3B43':	'0,1,2'
	}<commit_msg>Fixed bug on file path building<commit_after>import os
import sys
import fnmatch
import subprocess
import datetime
import argparse
import logging
import re


def isLeapYear(year):
	'''Returns TRUE if the given year (int) is leap and FALSE otherwise'''
	leapyear = False
	if year % 4 != 0:
		leapyear = False
	elif year % 100 != 0:
		leapyear = True
	elif year % 400 == 0:
		leapyear = True
	else:
		leapyear = False
	return leapyear


def listFiles(path, pattern):
	res = list()
	for path, subdirs, files in os.walk(path):
		for name in files:
			filepath = os.path.join(path, name)
			if(re.match(pattern, filepath, flags=0)):
				res.append(filepath)
	return res

	
def testGribModis2SciDB():
	'''Test if GRibeiro's modis2scidb is installed'''
	res = True
	try:
		subprocess.check_output(["modis2scidb", "--h"])
	except:
		res = False
	return res

	
def date2grid(dateFileName, period, startyear):
	'''Return an time index (timid) from the input date (MODIS DOY) and time period (e.g 8 days). '''
	res = -1
	if period > 0:
		dateYYYYDOY = dateFileName[1:] #Remove the A precedding the date
		year = int(dateYYYYDOY[0:4])
		doy = int(dateYYYYDOY[4:7])
		ppy = int(365 / period) + 1 # Periods per year
		if(period > 0 and (doy - 1) % period == 0):
			idd = (doy - 1) / period
			idy = (year - startyear) * ppy
			res = idy + idd
		else:
			logging.error("date2grid: Invalid date")
	elif period == -319980101: # Monthly - given as YYYYMMDD i.e 19980101, 19980201, 19980301
		dateYYYYMMDD = dateFileName
		year = int(dateYYYYMMDD[0:4])
		mm = int(dateYYYYMMDD[4:6])
		#dd = int(dateYYYYMMDD[6:8])
		idy = (year - startyear) * 12
		idd = mm - 1
		res = idy + idd
	return res	


#********************************************************
# MAIN
#********************************************************
def main(argv):
	t0 = datetime.datetime.now()
	parser = argparse.ArgumentParser(description = "Exports MODIS-HDFs to binary files for uploading to SCIDB using GRibeiro's tool")
	parser.add_argument("hdfFile", help = "Path to the HDF")
	parser.add_argument("loadFolder", help = "Folder from where the binary files are uploaded to SCIDB")
	parser.add_argument("product", help = "Product. e.g MOD09Q1")
	parser.add_argument("--log", help = "Log level. Default = WARNING", default = 'WARNING')
	#Get paramters
	args = parser.parse_args()
	hdfFile = args.hdfFile
	loadFolder = os.path.join(args.loadFolder, '')
	product = args.product
	log = args.log
	####################################################
	# CONFIG
	####################################################
	prodList = ['MOD09Q1', 'MOD13Q1', 'TRMM3B43']
	prodTemporalResolution = {
		'MOD09Q1':	8,
		'MOD13Q1':	16,
		'TRMM3B43':	-319980101
	}
	prodStartYear = {
		'MOD09Q1':	2000,
		'MOD13Q1':	2000,
		'TRMM3B43':	1998
	}
	prodBands = {
		'MOD09Q1':	'0,1,2',
		'MOD13Q1':	'0,1,2,3,4,5,6,7,8,9,10,11',
		'TRMM3B43':	'0,1,2'
	}
{'fraction_non_alphanumeric': 0.2029513888888889, 'fraction_numerical': 0.05486111111111111, 'mean_word_length': 4.270814272644099, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6291242', 'n_tokens_mistral': 2402, 'n_tokens_neox': 2110, 'n_words': 571}