<commit_before>//
//  ViewController.swift
//  Masterlister
//
//  Created by Curylo, Alex (Agoda) on 7/20/17.
//  Copyright © 2017 Trollwerks Inc. All rights reserved.
//

import Cocoa

class SitelistVC: NSViewController {

    enum Visited: String {
        case yes = "✅"
        case no = "◻️"
    }

    enum Document {
        case html
        case wordpress
    }
    
    struct Country: Codable {
        let alpha2: String
        let alpha3: String
        let name: String
        let officialName: String
        let numeric: String
        // all except Kosovo
        let wikiUrl: URL?
        // Kosovo only
        let unofficial: Bool?
        let wikiEntry: URL?
    }

    let countries: [Country] = {
        let path = Bundle.main.path(forResource: "iso_3166-1", ofType: "json")
        let data = try! Data(contentsOf: URL(fileURLWithPath: path!))
        let dict = try! JSONDecoder().decode([String: Country].self, from: data)
        let array = Array(dict.values).sorted { (lhs, rhs) in
            lhs.name < rhs.name
        }
        return array
    }()
    
    struct Member: Codable {
        let iso: String
        let name: String
        let joined: String
        let region: String
    }

    let members: [Member] = {
        let path = Bundle.main.path(forResource: "unesco_members", ofType: "json")
        let data = try! Data(contentsOf: URL(fileURLWithPath: path!))
        let array = try! JSONDecoder().decode([Member].self, from: data)
        assert(array.count == 205, "Should be 195 states and 10 associates in 2017")
        return array
    }()

    // NB whc-en has other fields:
    // historical_description, long_description
    // "http_url": "http://whc.unesco.org/en/list/208",
    // "image_url": "http://whc.unesco.org/uploads/sites/site_208.jpg",
    struct Site: Codable {
        let id_no: String
        let iso_code: String
        let category: String
        let region_en: String
        let date_inscribed: String
        let name_en: String
        let short_description_en: String
        let justification_en: String

        var id: Int {
            return Int(id_no) ?? 0
        }
    }
    
    let sites: [Site] = {
        let path = Bundle.main.path(forResource: "whc-sites-2017", ofType: "json")
        let data = try! Data(contentsOf: URL(fileURLWithPath: path!))
        let jsonArray = try! JSONDecoder().decode([Site].self, from: data)
        let array = jsonArray.sorted { (lhs, rhs) in
            lhs.id_no < rhs.id_no
        }
        assert(array.count == 1073, "Should be 1073 WHS in 2017")
        return array
    }()
    
    struct Tentative: Codable {
        let id_no: String
        let iso: String
        let submitted: String
        let name: String
        let category: String

        var id: Int {
            return Int(id_no) ?? 0
        }
    }

    let tentatives: [Tentative] = {
        let path = Bundle.main.path(forResource: "whtl-20170806", ofType: "json")
        let data = try! Data(contentsOf: URL(fileURLWithPath: path!))
        let jsonArray = try! JSONDecoder().decode([Tentative].self, from: data)
        let array = jsonArray.sorted { (lhs, rhs) in
            lhs.id_no < rhs.id_no
        }
        // Contrary to http://whc.unesco.org/en/tentativelists/
        // there does, in fact, appear to be 1696
        //assert(array.count == 1669, "Should be 1669 TWHS on 2017.08.06")
        return array
    }()

    struct Visit: Codable {
        let wonder: Int?
        let whs: Int?
        let twhs: Int?
        let visit: URL?
        let stay: URL?
        let eat: URL?
    }
    
    lazy var visits: [Visit] = {
        let path = Bundle.main.path(forResource: "visits", ofType: "json")
        var array: [Visit] = []
        do {
            let data = try Data(contentsOf: URL(fileURLWithPath: path!))
            array = try JSONDecoder().decode([Visit].self, from: data)
        } catch let jsonErr {
            print("Error decoding visits", jsonErr)
        }
        return array
    }()
    lazy var whsVisits: [Visit] = {
        let array = visits.filter { $0.whs != nil }
        let ids = array.compactMap { $0.whs }
        let duplicates = Array(Set(ids.filter({ (i: Int) in ids.filter({ $0 == i }).count > 1})))
        assert(duplicates.isEmpty, "Should not have duplicate WHS visits \(duplicates)")
        return array
    }()
    lazy var twhsVisits: [Visit] = {
        let array = visits.filter { $0.twhs != nil }
        let ids = visits.compactMap { $0.twhs }
        let duplicates = Array(Set(ids.filter({ (i: Int) in ids.filter({ $0 == i }).count > 1})))
        assert(duplicates.isEmpty, "Should not have duplicate TWHS visits \(duplicates)")
        return array
    }()

    struct CountryFile: Codable {
        let iso: String
        let file: URL?
        let name: String? // for easily locating unfiled countries
    }

    lazy var countryFiles: [CountryFile] = {
        let path = Bundle.main.path(forResource: "country-files", ofType: "json")<commit_msg>Check for duped visit IDs<commit_after>//
//  ViewController.swift
//  Masterlister
//
//  Created by <NAME>) on 7/20/17.
//  Copyright © 2017 Trollwerks Inc. All rights reserved.
//

import Cocoa

class SitelistVC: NSViewController {

    enum Visited: String {
        case yes = "✅"
        case no = "◻️"
    }

    enum Document {
        case html
        case wordpress
    }
    
    struct Country: Codable {
        let alpha2: String
        let alpha3: String
        let name: String
        let officialName: String
        let numeric: String
        // all except Kosovo
        let wikiUrl: URL?
        // Kosovo only
        let unofficial: Bool?
        let wikiEntry: URL?
    }

    let countries: [Country] = {
        let path = Bundle.main.path(forResource: "iso_3166-1", ofType: "json")
        let data = try! Data(contentsOf: URL(fileURLWithPath: path!))
        let dict = try! JSONDecoder().decode([String: Country].self, from: data)
        let array = Array(dict.values).sorted { (lhs, rhs) in
            lhs.name < rhs.name
        }
        return array
    }()
    
    struct Member: Codable {
        let iso: String
        let name: String
        let joined: String
        let region: String
    }

    let members: [Member] = {
        let path = Bundle.main.path(forResource: "unesco_members", ofType: "json")
        let data = try! Data(contentsOf: URL(fileURLWithPath: path!))
        let array = try! JSONDecoder().decode([Member].self, from: data)
        assert(array.count == 205, "Should be 195 states and 10 associates in 2017")
        return array
    }()

    // NB whc-en has other fields:
    // historical_description, long_description
    // "http_url": "http://whc.unesco.org/en/list/208",
    // "image_url": "http://whc.unesco.org/uploads/sites/site_208.jpg",
    struct Site: Codable {
        let id_no: String
        let iso_code: String
        let category: String
        let region_en: String
        let date_inscribed: String
        let name_en: String
        let short_description_en: String
        let justification_en: String

        var id: Int {
            return Int(id_no) ?? 0
        }
    }
    
    let sites: [Site] = {
        let path = Bundle.main.path(forResource: "whc-sites-2017", ofType: "json")
        let data = try! Data(contentsOf: URL(fileURLWithPath: path!))
        let jsonArray = try! JSONDecoder().decode([Site].self, from: data)
        let array = jsonArray.sorted { (lhs, rhs) in
            lhs.id_no < rhs.id_no
        }
        assert(array.count == 1073, "Should be 1073 WHS in 2017")
        return array
    }()
    
    struct Tentative: Codable {
        let id_no: String
        let iso: String
        let submitted: String
        let name: String
        let category: String

        var id: Int {
            return Int(id_no) ?? 0
        }
    }

    let tentatives: [Tentative] = {
        let path = Bundle.main.path(forResource: "whtl-20170806", ofType: "json")
        let data = try! Data(contentsOf: URL(fileURLWithPath: path!))
        let jsonArray = try! JSONDecoder().decode([Tentative].self, from: data)
        let array = jsonArray.sorted { (lhs, rhs) in
            lhs.id_no < rhs.id_no
        }
        // Contrary to http://whc.unesco.org/en/tentativelists/
        // there does, in fact, appear to be 1696
        //assert(array.count == 1669, "Should be 1669 TWHS on 2017.08.06")
        return array
    }()

    struct Visit: Codable {
        let wonder: Int?
        let whs: Int?
        let twhs: Int?
        let visit: URL?
        let stay: URL?
        let eat: URL?
    }
    
    lazy var visits: [Visit] = {
        let path = Bundle.main.path(forResource: "visits", ofType: "json")
        var array: [Visit] = []
        do {
            let data = try Data(contentsOf: URL(fileURLWithPath: path!))
            array = try JSONDecoder().decode([Visit].self, from: data)
        } catch let jsonErr {
            print("Error decoding visits", jsonErr)
        }
        return array
    }()
    lazy var whsVisits: [Visit] = {
        let array = visits.filter { $0.whs != nil }
        let ids = array.compactMap { $0.whs }
        let duplicates = Array(Set(ids.filter({ (i: Int) in ids.filter({ $0 == i }).count > 1})))
        assert(duplicates.isEmpty, "Should not have duplicate WHS visits \(duplicates)")
        let wrong = Set(ids).subtracting(Set(sites.map( { $0.id } )))
        assert(wrong.isEmpty, "Should not have wrong WHS visits \(wrong)")
        return array
    }()
    lazy var twhsVisits: [Visit] = {
        let array = visits.filter { $0.twhs != nil }
        let ids = visits.compactMap { $0.twhs }
        let duplicates = Array(Set(ids.filter({ (i: Int) in ids.filter({ $0 == i }).count > 1})))
        assert(duplicates.isEmpty, "Should not have duplicate TWHS visits \(duplicates)")
        let wrong = Set(ids).subtracting(Set(tentatives.map( { $0.id } )))
        assert(wrong.isEmpty, "Should not have wrong TWHS visits \(wrong)")
        return array
    }()

    struct CountryFile: Codable {
        let iso: String
        let file: URL?
        let name: String? // for easily locating unfiled countries
    }

    lazy var countryFiles: [CountryFile] = {
        let path = Bundle.main.path(forResource: "country-files", ofType: "json")
{'fraction_non_alphanumeric': 0.11827024944621015, 'fraction_numerical': 0.017143407493017432, 'mean_word_length': 2.017727404824179, 'pattern_counts': {'":': 4, '<': 10, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4321238', 'n_tokens_mistral': 3421, 'n_tokens_neox': 3175, 'n_words': 941}