<commit_before>//
//  OperationParseGeoJson.swift
//  tapmap
//
//  Created by <NAME> on 2017-04-16.
//  Copyright © 2017 Wildbrain. All rights reserved.
//

import Foundation
import SwiftyJSON

class OperationParseGeoJson : Operation {
	let json : JSON
	let report : ProgressReport
	var continents : [GeoMultiFeature]?
	
	init(_ geoJson: JSON, reporter: @escaping ProgressReport) {
		json = geoJson
		report = reporter
	}
	
	override func main() {
		guard !isCancelled else { print("Cancelled before starting"); return }

		var loadedContinents: [GeoMultiFeature] = []
		let numContinents = json.dictionaryValue.keys.count

		for continentJson in json.dictionaryValue.values {
			let loadedContinent = parseContinent(continentJson)
			loadedContinents.append(loadedContinent)

			report(Double(loadedContinents.count) / Double(numContinents), loadedContinent.name, false)
		}
		
		continents = loadedContinents
		
	}
	
	fileprivate func parseContinent(_ json: JSON) -> GeoMultiFeature {
		let continentName = json["name"].stringValue
		let regions = json["regions"].dictionaryValue
		
		let loadedRegions = regions.values.flatMap { parseRegion($0) }
		
		return GeoMultiFeature(name: continentName,
													 subFeatures: [],
													 subMultiFeatures: loadedRegions)
	}
	
	fileprivate func parseRegion(_ json: JSON) -> GeoMultiFeature? {
		guard let regionName = json["name"].string else { print("No name in region"); return nil }
		guard let features = json["coordinates"].array else { print("No coordinates in region"); return nil}
		
		var loadedFeatures: [GeoFeature] = []
		
		for featureJson in features {
			let featureType = featureJson.array?.first?.type ?? .unknown
			switch featureType {
			case .array:
				for subFeatureJson in featureJson.arrayValue {
					loadedFeatures.append(parseFeature(subFeatureJson))
				}
			case .dictionary:
				loadedFeatures.append(parseFeature(featureJson))
			case .unknown:
				break
			default:
				print("Malformed feature in \(regionName)")
			}
		}
		return GeoMultiFeature(name: regionName, subFeatures: loadedFeatures, subMultiFeatures: [])
	}
	
	fileprivate func parseFeature(_ json: JSON) -> GeoFeature {
		return GeoFeature(vertices: [])
	}
	
	func buildVertices(_ coords: [JSON]) -> [Vertex] {
		var outVertices: [Vertex] = []
		for c in coords {
			if c.type == .dictionary {
				let v = Vertex(v: (c["lng"].floatValue,
													 c["lat"].floatValue))
				outVertices.append(v)
			} else {
				print("Feature has unexpected internal type: \(c.type)")
			}
		}<commit_msg>Parse actual vertex geometry.<commit_after>//
//  OperationParseGeoJson.swift
//  tapmap
//
//  Created by <NAME> on 2017-04-16.
//  Copyright © 2017 Wildbrain. All rights reserved.
//

import Foundation
import SwiftyJSON

class OperationParseGeoJson : Operation {
	let json : JSON
	let report : ProgressReport
	var continents : [GeoMultiFeature]?
	
	init(_ geoJson: JSON, reporter: @escaping ProgressReport) {
		json = geoJson
		report = reporter
	}
	
	override func main() {
		guard !isCancelled else { print("Cancelled before starting"); return }

		var loadedContinents: [GeoMultiFeature] = []
		let numContinents = json.dictionaryValue.keys.count

		for continentJson in json.dictionaryValue.values {
			let loadedContinent = parseContinent(continentJson)
			loadedContinents.append(loadedContinent)

			report(Double(loadedContinents.count) / Double(numContinents), loadedContinent.name, false)
		}
		
		continents = loadedContinents
		
	}
	
	fileprivate func parseContinent(_ json: JSON) -> GeoMultiFeature {
		let continentName = json["name"].stringValue
		let regions = json["regions"].dictionaryValue
		
		let loadedRegions = regions.values.flatMap { parseRegion($0) }
		
		return GeoMultiFeature(name: continentName,
													 subFeatures: [],
													 subMultiFeatures: loadedRegions)
	}
	
	fileprivate func parseRegion(_ json: JSON) -> GeoMultiFeature? {
		guard let regionName = json["name"].string else { print("No name in region"); return nil }
		guard let features = json["coordinates"].array else { print("No coordinates in region"); return nil}
		
		var loadedFeatures: [GeoFeature] = []
		
		for featureJson in features {
			let featureType = featureJson.array?.first?.type ?? .unknown
			switch featureType {
			case .array:
				for subFeatureJson in featureJson.arrayValue {
					loadedFeatures.append(parseFeature(subFeatureJson))
				}
			case .dictionary:
				loadedFeatures.append(parseFeature(featureJson))
			case .unknown:
				break
			default:
				print("Malformed feature in \(regionName)")
			}
		}
		return GeoMultiFeature(name: regionName, subFeatures: loadedFeatures, subMultiFeatures: [])
	}
	
	fileprivate func parseFeature(_ json: JSON) -> GeoFeature {
		let vertices : [Vertex]
		if let coordinates = json.array {
			vertices = buildVertices(coordinates)
		} else {
			print("Vertices not found in feature.")
			vertices = []
		}
		return GeoFeature(vertices: vertices)
	}
	
	func buildVertices(_ coords: [JSON]) -> [Vertex] {
		var outVertices: [Vertex] = []
		for c in coords {
			if c.type == .dictionary {
				let v = Vertex(v: (c["lng"].floatValue,
													 c["lat"].floatValue))
				outVertices.append(v)
			} else {
				print("Feature has unexpected internal type: \(c.type)")
			}
		}
{'fraction_non_alphanumeric': 0.10945179584120983, 'fraction_numerical': 0.004914933837429111, 'mean_word_length': 3.719892952720785, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1350733', 'n_tokens_mistral': 1877, 'n_tokens_neox': 1521, 'n_words': 408}