<commit_before>import os
from archives_tools import dacs
import openpyxl
import sys

__location__ = (os.path.dirname(os.path.realpath(__file__)))
inputPath = os.path.join(__location__, "input")

for file in os.listdir(inputPath):
	if file.endswith(".xlsx"):
		filePath = os.path.join(inputPath, file)
		
		wb = openpyxl.load_workbook(filename=filePath, read_only=True)
				
		#validate sheets
		for sheet in wb.worksheets:
			checkSwitch = True
			try:
				if sheet["H1"].value.lower().strip() != "title":
					checkSwitch = False
				elif sheet["H2"].value.lower().strip() != "level":
					checkSwitch = False
				elif sheet["H3"].value.lower().strip() != "ref id":
					checkSwitch = False
				elif sheet["J6"].value.lower().strip() != "date 1 display":
					checkSwitch = False
				elif sheet["D6"].value.lower().strip() != "container uri":
					checkSwitch = False
			except:
				print ("ERROR: incorrect sheet " + sheet.title + " in file " + file)
				
			if checkSwitch == False:
				print ("ERROR: incorrect sheet " + sheet.title + " in file " + file)
			else:
			
				#Read sheet info
				print ("Reading sheet: " + sheet.title)
				lineCount = 0
				errorCount = 0
				for row in sheet.rows:
					lineCount += 1
					if lineCount > 6:
						try:
							date = dacs.iso2DACS(str(row[10].value))
						except:
							errorCount += 1
							print ("DATE ERROR: (" + str(row[10].value) + ") line " + str(lineCount) + " title: " + str(row[8].value))
						try:
							date = dacs.iso2DACS(str(row[12].value))
						except:
							errorCount += 1
							print ("DATE ERROR: (" + str(row[12].value) + ") line " + str(lineCount) + " title: " + str(row[8].value))
						try:
							date = dacs.iso2DACS(str(row[14].value))
						except:
							errorCount += 1
							print ("DATE ERROR: (" + str(row[14].value) + ") line " + str(lineCount) + " title: " + str(row[8].value))
						try:
							date = dacs.iso2DACS(str(row[16].value))
						except:
							errorCount += 1
							print ("DATE ERROR: (" + str(row[16].value) + ") line " + str(lineCount) + " title: " + str(row[8].value))
						try:
							date = dacs.iso2DACS(str(row[18].value))
						except:
							errorCount += 1
							print ("DATE ERROR: (" + str(row[18].value) + ") line " + str(lineCount) + " title: " + str(row[8].value))
							
						if not row[22].value is None:
							if len(str(row[22].value).strip()) > 0:
								daoName = str(row[22].value).strip()
								daoPath = os.path.join(__location__, "dao", daoName)
								
								if not os.path.isfile(daoPath):
									errorCount += 1
									print ("DAO ERROR: File Not Present in dao (" + str(row[22].value) + ") line " + str(lineCount))<commit_msg>updated validator, will now check for files in dao dir<commit_after>import os
from archives_tools import dacs
import openpyxl
import sys

__location__ = (os.path.dirname(os.path.realpath(__file__)))
inputPath = os.path.join(__location__, "input")

for file in os.listdir(inputPath):
	if file.endswith(".xlsx"):
		filePath = os.path.join(inputPath, file)
		
		wb = openpyxl.load_workbook(filename=filePath, read_only=True)
				
		#validate sheets
		for sheet in wb.worksheets:
			checkSwitch = True
			try:
				if sheet["H1"].value.lower().strip() != "title":
					checkSwitch = False
				elif sheet["H2"].value.lower().strip() != "level":
					checkSwitch = False
				elif sheet["H3"].value.lower().strip() != "ref id":
					checkSwitch = False
				elif sheet["J6"].value.lower().strip() != "date 1 display":
					checkSwitch = False
				elif sheet["D6"].value.lower().strip() != "container uri":
					checkSwitch = False
			except:
				print ("ERROR: incorrect sheet " + sheet.title + " in file " + file)
				
			if checkSwitch == False:
				print ("ERROR: incorrect sheet " + sheet.title + " in file " + file)
			else:
			
				#Read sheet info
				print ("Reading sheet: " + sheet.title)
				lineCount = 0
				errorCount = 0
				for row in sheet.rows:
					lineCount += 1
					if lineCount > 6:
						try:
							date = dacs.iso2DACS(str(row[10].value))
						except:
							errorCount += 1
							print ("DATE ERROR: (" + str(row[10].value) + ") line " + str(lineCount) + " title: " + str(row[8].value))
						try:
							date = dacs.iso2DACS(str(row[12].value))
						except:
							errorCount += 1
							print ("DATE ERROR: (" + str(row[12].value) + ") line " + str(lineCount) + " title: " + str(row[8].value))
						try:
							date = dacs.iso2DACS(str(row[14].value))
						except:
							errorCount += 1
							print ("DATE ERROR: (" + str(row[14].value) + ") line " + str(lineCount) + " title: " + str(row[8].value))
						try:
							date = dacs.iso2DACS(str(row[16].value))
						except:
							errorCount += 1
							print ("DATE ERROR: (" + str(row[16].value) + ") line " + str(lineCount) + " title: " + str(row[8].value))
						try:
							date = dacs.iso2DACS(str(row[18].value))
						except:
							errorCount += 1
							print ("DATE ERROR: (" + str(row[18].value) + ") line " + str(lineCount) + " title: " + str(row[8].value))
							
						if not row[22].value is None:
							if len(str(row[22].value).strip()) > 0:
								daoName = str(row[22].value).strip()
								if not daoName.lower().startswith("http"):
									daoPath = os.path.join(__location__, "dao", daoName)
									
									if not os.path.isfile(daoPath):
										errorCount += 1
										print ("DAO ERROR: File Not Present in dao (" + str(row[22].value) + ") line " + str(lineCount))
{'fraction_non_alphanumeric': 0.1662699139351767, 'fraction_numerical': 0.020142830983336385, 'mean_word_length': 2.6510695187165774, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6288413', 'n_tokens_mistral': 2481, 'n_tokens_neox': 1819, 'n_words': 423}