<commit_before>import arcpy
import math
from Parcel import Parcel
import re

class Error:

	def __init__(self,featureClass,coName):
		self.coName = coName
		self.generalErrorCount = 0
		self.geometricErrorCount = 0
		self.addressErrorCount = 0
		self.taxErrorCount = 0
		self.comparisonDict = {}
		self.attributeFileErrors = []
		self.geometricFileErrors = []
		#self.geometricPlacementErrors = ["Several parcel geometries appear to be spatially misplaced when comparing them against last year's parcel geometries. This issue is indicative of a re-projection error. Please see the following documentation: http://www.sco.wisc.edu/parcels/tools/FieldMapping/Parcel_Schema_Field_Mapping_Guide.pdf section #2, for advice on how to project native data to the Statewide Parcel CRS."]
		self.geometricPlacementErrors = []
		self.pinSkipCount = 0
		self.trYearPast = 0
		self.trYearExpected = 0
		self.trYearFuture = 0
		self.trYearOther = 0
		self.coNameMiss = 0
		self.fipsMiss = 0
		self.srcMiss = 0
		self.netMoreGrsCnt = 0
		self.recordIterationCount = 0;
		self.recordTotalCount = int(arcpy.GetCount_management(featureClass).getOutput(0)) # Total number of records in the feature class
		self.checkEnvelopeInterval = math.trunc(self.recordTotalCount / 100) # Interval value used to apply 10 total checks on records at evenly spaced intervals throughout the dataset.
		self.nextEnvelopeInterval = self.checkEnvelopeInterval
		self.notConfirmGeomCount = 0 #counts parcels with invalid Geometry
		self.validatedGeomCount = 0 #counts parcels whose geometry is validated
		self.geometryNotValidated = False
		self.geometryNotChecked = True
		self.diffxy = 0
		self.xyShift = 0
		self.codedDomainfields = []

	# Test records throughout the dataset to ensure that polygons exist within an actual county envelope ("Waukesha" issue or the "Lake Michigan" issue).
	def checkGeometricQuality(self,Parcel,ignoreList):
		#arcpy.AddMessage(self.nextEnvelopeInterval)
		#arcpy.AddMessage(self.recordIterationCount)
		if self.nextEnvelopeInterval == self.recordIterationCount:
			if str(Parcel.parcelid).upper() in ignoreList:
				self.nextEnvelopeInterval = self.nextEnvelopeInterval + 1
			else:
				countyEnvelope = self.testCountyEnvelope(Parcel)
				if countyEnvelope == "Valid" and self.validatedGeomCount == 50: # would mean that the "Waukesha" issue or the "Lake Michigan" issue does not exist in this dataset.
					self.nextEnvelopeInterval = 4000000
					if self.notConfirmGeomCount > 0:
						self.xyShift = round((self.diffxy/self.notConfirmGeomCount),2)
					arcpy.AddMessage("Parcel geometry validated.")
					self.geometricPlacementErrors = []
				elif countyEnvelope == "Not Confirmed" and self.notConfirmGeomCount == 50:
					self.nextEnvelopeInterval = 4000000
					self.xyShift = round((self.diffxy/self.notConfirmGeomCount),2)
					#arcpy.AddMessage("couldn't validate")
					if  self.xyShift >= 6:
						self.geometryNotValidated = True
					elif self.xyShift >= 1.2 and self.xyShift < 6:
						arcpy.AddMessage("Parcel geometry validated.")
						arcpy.AddMessage("Note that several parcel geometries appear to be spatially misplaced by about: " + str(self.xyShift) + " meters." )
						self.geometricPlacementErrors = ["Several parcel geometries appear to be spatially misplaced " + str(self.xyShift) + " meters when comparing them against last year's parcel geometries. This issue is indicative of a re-projection error. Please see the following documentation: http://www.sco.wisc.edu/parcels/tools/FieldMapping/Parcel_Schema_Field_Mapping_Guide.pdf section #2, for advice on how to project native data to the Statewide Parcel CRS."]
					else:
						arcpy.AddMessage("Parcel geometry validated.")
						self.geometricPlacementErrors = []
						#arcpy.AddMessage("Geometry validated -- but several parcels are misplaced: " + str(self.xyShift) + " meters.")
				self.nextEnvelopeInterval = self.nextEnvelopeInterval + self.checkEnvelopeInterval
		elif self.nextEnvelopeInterval < 4000000 and self.nextEnvelopeInterval >= (100 * self.checkEnvelopeInterval):
			if self.validatedGeomCount == 0 and self.notConfirmGeomCount == 0: #no parcel geometry was checked -- likely ParcelIds are different from previous years
				arcpy.AddMessage("couldn't check parcels " )
				self.nextEnvelopeInterval = 4000000
				self.geometryNotChecked = False   # flag for county centroid check funcion
			elif self.notConfirmGeomCount  > 0:
				self.nextEnvelopeInterval = 4000000
				self.xyShift = round((self.diffxy/self.notConfirmGeomCount),2)
				arcpy.AddMessage("Several parcel geometries appear to be spatially misplaced by about: " + str(self.xyShift) + " meters." )
				self.geometricPlacementErrors = ["Several parcel geometries appear to be spatially misplaced " + str(self.xyShift) + " meters when comparing them against last year's parcel geometries. This issue is indicative of a re-projection error. Please see the following documentation: http://www.sco.wisc.edu/parcels/tools/FieldMapping/Parcel_Schema_Field_Mapping_Guide.pdf section #2, for advice on how to project native data to the Statewide Parcel CRS."]
		self,Parcel = self.testParcelGeometry(Parcel)
		self.recordIterationCount += 1
		return (self, Parcel)


	# Will test the row against LTSB's feature service to identify if the feature is in the correct location.
	def testCountyEnvelope(self,Parcel):
		specialchars = ['/', '#', '&']  #this special characters occurs in some ParcelIDs
		charsdict = {'&': '%26', '#': '%23', '/': '%2F'}
		parcelid =  str(Parcel.parcelid).upper()
		for i in specialchars:
			while parcelid is not None and i in parcelid:
				parcelid = parcelid[:parcelid.find(i)] + charsdict[i] + parcelid[parcelid.find(i)+1:]
		try:
			baseURL = "http://mapservices.legis.wisconsin.gov/arcgis/rest/services/WLIP_V3/V3_Parcels/FeatureServer/0/query"
			#baseURL = "http://mapservices.legis.wisconsin.gov/arcgis/rest/services/WLIP/Parcels/FeatureServer/0/query"
			where =  str(Parcel.parcelfips) + parcelid
			#arcpy.AddMessage(where)
			query = "?f=json&where=STATEID+%3D+%27{0}%27&geometry=true&returnGeometry=true&spatialRel=esriSpatialRelIntersects&outFields=OBJECTID%2CPARCELID%2CTAXPARCELID%2CCONAME%2CPARCELSRC&outSR=3071&resultOffset=0&resultRecordCount=10000".format(where)
			fsURL = baseURL + query
			#arcpy.AddMessage(fsURL)
			fs = arcpy.FeatureSet()
			fs.load(fsURL)
			with arcpy.da.UpdateCursor(fs,["SHAPE@XY"]) as cursorLTSB:
				for rowLTSB in cursorLTSB:
					v2x = round(rowLTSB[0][0],2)
					v1x = round(Parcel.shapeXY[0],2)
					v2y = round(rowLTSB[0][1],2)
					v1y = round(Parcel.shapeXY[1],2)
					diffx = v2x - v1x
					diffy = v2y - v1y
					#if (v2x == v1x) and (v2y == v1y):
					if (diffx == 0) and (diffy == 0):
						self.validatedGeomCount += 1
						if (self.validatedGeomCount % 10 == 0):
							arcpy.AddMessage("Parcel geometry validated.")
						return "Valid"
					else:
						#arcpy.AddMessage("difference x :"  + str(diffx) + ' = ' + str(v2x) + " - " + str(v1x) )
						#arcpy.AddMessage("difference y :"  + str(diffy) + ' = ' + str(v2y) + " - " + str(v1y) )
						#diffxy = round(math.sqrt (diffx*diffx + diffy*diffy),2)
						arcpy.AddMessage("difference xy :"  + str(diffxy))
						self.diffxy = self.diffxy + diffxy
						self.notConfirmGeomCount += 1
						if (self.notConfirmGeomCount % 10 == 0):
							arcpy.AddMessage("Parcel geometry not yet validated, will attempt another record.")
						return "Not Confirmed"
						# Call it valid If the query returns no features (failure to return features would not be caused by a misalignment)
			return "Valid"
		except:
			# Call it valid if an error happens (error would not be caused by a misalignment)
			return "Valid"
		return "Valid"

	def testParcelGeometry(self,Parcel):
		# Test for null geometries or other oddities
		try:
			geom = Parcel.shape
			xCent = geom.centroid.X
			yCent = geom.centroid.Y
		except:
			Parcel.geometricErrors.append("Corrupt Geometry: The feature's geometry could not be accessed.")<commit_msg>more corrections in Error object functions<commit_after>import arcpy
import math
from Parcel import Parcel
import re

class Error:

	def __init__(self,featureClass,coName):
		self.coName = coName
		self.generalErrorCount = 0
		self.geometricErrorCount = 0
		self.addressErrorCount = 0
		self.taxErrorCount = 0
		self.comparisonDict = {}
		self.attributeFileErrors = []
		self.geometricFileErrors = []
		#self.geometricPlacementErrors = ["Several parcel geometries appear to be spatially misplaced when comparing them against last year's parcel geometries. This issue is indicative of a re-projection error. Please see the following documentation: http://www.sco.wisc.edu/parcels/tools/FieldMapping/Parcel_Schema_Field_Mapping_Guide.pdf section #2, for advice on how to project native data to the Statewide Parcel CRS."]
		self.geometricPlacementErrors = []
		self.pinSkipCount = 0
		self.trYearPast = 0
		self.trYearExpected = 0
		self.trYearFuture = 0
		self.trYearOther = 0
		self.coNameMiss = 0
		self.fipsMiss = 0
		self.srcMiss = 0
		self.netMoreGrsCnt = 0
		self.recordIterationCount = 0;
		self.recordTotalCount = int(arcpy.GetCount_management(featureClass).getOutput(0)) # Total number of records in the feature class
		self.checkEnvelopeInterval = math.trunc(self.recordTotalCount / 100) # Interval value used to apply 10 total checks on records at evenly spaced intervals throughout the dataset.
		self.nextEnvelopeInterval = self.checkEnvelopeInterval
		self.notConfirmGeomCount = 0 #counts parcels with invalid Geometry
		self.validatedGeomCount = 0 #counts parcels whose geometry is validated
		self.geometryNotValidated = False
		self.geometryNotChecked = True
		self.diffxy = 0
		self.xyShift = 0
		self.codedDomainfields = []

	# Test records throughout the dataset to ensure that polygons exist within an actual county envelope ("Waukesha" issue or the "Lake Michigan" issue).
	def checkGeometricQuality(self,Parcel,ignoreList):
		#arcpy.AddMessage(self.nextEnvelopeInterval)
		#arcpy.AddMessage(self.recordIterationCount)
		if self.nextEnvelopeInterval == self.recordIterationCount:
			if str(Parcel.parcelid).upper() in ignoreList:
				self.nextEnvelopeInterval = self.nextEnvelopeInterval + 1
			else:
				countyEnvelope = self.testCountyEnvelope(Parcel)
				if countyEnvelope == "Valid" and self.validatedGeomCount == 50: # would mean that the "Waukesha" issue or the "Lake Michigan" issue does not exist in this dataset.
					self.nextEnvelopeInterval = 4000000
					if self.notConfirmGeomCount > 0:
						self.xyShift = round((self.diffxy/self.notConfirmGeomCount),2)
					arcpy.AddMessage("Parcel geometry validated.")
					self.geometricPlacementErrors = []
				elif countyEnvelope == "Not Confirmed" and self.notConfirmGeomCount == 50:
					self.nextEnvelopeInterval = 4000000
					self.xyShift = round((self.diffxy/self.notConfirmGeomCount),2)
					#arcpy.AddMessage("couldn't validate")
					if  self.xyShift >= 6:
						self.geometryNotValidated = True
					elif self.xyShift >= 1.2 and self.xyShift < 6:
						arcpy.AddMessage("\n ")
						arcpy.AddMessage("Parcel geometry validated, but several parcel geometries appear to be spatially misplaced by about: " + str(self.xyShift) + " meters. \n" )
						self.geometricPlacementErrors = ["Several parcel geometries appear to be spatially misplaced " + str(self.xyShift) + " meters when comparing them against last year's parcel geometries. This issue is indicative of a re-projection error. Please see the following documentation: http://www.sco.wisc.edu/parcels/tools/FieldMapping/Parcel_Schema_Field_Mapping_Guide.pdf section #2, for advice on how to project native data to the Statewide Parcel CRS."]
					else:
						arcpy.AddMessage("\nParcel geometry validated.")
						self.geometricPlacementErrors = []
						#arcpy.AddMessage("Geometry validated -- but several parcels are misplaced: " + str(self.xyShift) + " meters.")
				self.nextEnvelopeInterval = self.nextEnvelopeInterval + self.checkEnvelopeInterval
		elif self.nextEnvelopeInterval < 4000000 and self.nextEnvelopeInterval >= (100 * self.checkEnvelopeInterval):
			if self.validatedGeomCount == 0 and self.notConfirmGeomCount == 0: #no parcel geometry was checked -- likely ParcelIds are different from previous years
				arcpy.AddMessage("The PARCELID within the dataset may not match the PARCELID submitted the previous year. \n" )
				self.nextEnvelopeInterval = 4000000
				self.geometryNotChecked = False   # flag for county centroid check funcion
			elif self.notConfirmGeomCount  > 0:
				self.nextEnvelopeInterval = 4000000
				self.xyShift = round((self.diffxy/self.notConfirmGeomCount),2)
				arcpy.AddMessage("Several parcel geometries appear to be spatially misplaced by about: " + str(self.xyShift) + " meters." )
				self.geometricPlacementErrors = ["Several parcel geometries appear to be spatially misplaced " + str(self.xyShift) + " meters when comparing them against last year's parcel geometries. This issue is indicative of a re-projection error. Please see the following documentation: http://www.sco.wisc.edu/parcels/tools/FieldMapping/Parcel_Schema_Field_Mapping_Guide.pdf section #2, for advice on how to project native data to the Statewide Parcel CRS."]
		self,Parcel = self.testParcelGeometry(Parcel)
		self.recordIterationCount += 1
		return (self, Parcel)


	# Will test the row against LTSB's feature service to identify if the feature is in the correct location.
	def testCountyEnvelope(self,Parcel):
		specialchars = ['/', '#', '&']  #this special characters occurs in some ParcelIDs
		charsdict = {'&': '%26', '#': '%23', '/': '%2F'}
		parcelid =  str(Parcel.parcelid).upper()
		for i in specialchars:
			while parcelid is not None and i in parcelid:
				parcelid = parcelid[:parcelid.find(i)] + charsdict[i] + parcelid[parcelid.find(i)+1:]
		try:
			baseURL = "http://mapservices.legis.wisconsin.gov/arcgis/rest/services/WLIP_V3/V3_Parcels/FeatureServer/0/query"
			#baseURL = "http://mapservices.legis.wisconsin.gov/arcgis/rest/services/WLIP/Parcels/FeatureServer/0/query"
			where =  str(Parcel.parcelfips) + parcelid
			query = "?f=json&where=STATEID+%3D+%27{0}%27&geometry=true&returnGeometry=true&spatialRel=esriSpatialRelIntersects&outFields=OBJECTID%2CPARCELID%2CTAXPARCELID%2CCONAME%2CPARCELSRC&outSR=3071&resultOffset=0&resultRecordCount=10000".format(where)
			fsURL = baseURL + query
			#arcpy.AddMessage(fsURL)
			fs = arcpy.FeatureSet()
			fs.load(fsURL)
			#arcpy.AddMessage(where)
			with arcpy.da.UpdateCursor(fs,["SHAPE@XY"]) as cursorLTSB:
				for rowLTSB in cursorLTSB:
					v2x = round(rowLTSB[0][0],2)
					v1x = round(Parcel.shapeXY[0],2)
					v2y = round(rowLTSB[0][1],2)
					v1y = round(Parcel.shapeXY[1],2)
					diffx = v2x - v1x
					diffy = v2y - v1y
					#if (v2x == v1x) and (v2y == v1y):
					if (diffx == 0) and (diffy == 0):
						self.validatedGeomCount += 1
						if (self.validatedGeomCount % 10 == 0):
							arcpy.AddMessage("Checking parcel geometry ...")
						return "Valid"
					else:
						#arcpy.AddMessage("difference x :"  + str(diffx) + ' = ' + str(v2x) + " - " + str(v1x) )
						#arcpy.AddMessage("difference y :"  + str(diffy) + ' = ' + str(v2y) + " - " + str(v1y) )
						diffxy = round(math.sqrt (diffx*diffx + diffy*diffy),2)
						#arcpy.AddMessage("difference xy :"  + str(diffxy))
						self.diffxy = self.diffxy + diffxy
						self.notConfirmGeomCount += 1
						if (self.notConfirmGeomCount % 10 == 0):
							arcpy.AddMessage("Parcel geometry not yet validated, will attempt another record.")
						return "Not Confirmed"
						# Call it valid If the query returns no features (failure to return features would not be caused by a misalignment)
			return "Valid"
		except:
			# Call it valid if an error happens (error would not be caused by a misalignment)
			return "Valid"
		return "Valid"

	def testParcelGeometry(self,Parcel):
		# Test for null geometries or other oddities
		try:
			geom = Parcel.shape
			xCent = geom.centroid.X
			yCent = geom.centroid.Y
		except:
			Parcel.geometricErrors.append("Corrupt Geometry: The feature's geometry could not be accessed.")
{'fraction_non_alphanumeric': 0.10138363218961345, 'fraction_numerical': 0.018489793385865858, 'mean_word_length': 4.21280724450194, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3255220', 'n_tokens_mistral': 6025, 'n_tokens_neox': 4970, 'n_words': 1341}