<commit_before>//
//  Adds.swift
//  CAVOK
//
//  Created by <NAME> on 24.01.15.
//  Copyright © 2016 <NAME>. All rights reserved.
//

import Foundation
import SWXMLHash
import PMKFoundation
import PromiseKit
import Gzip

enum AddsSource: String {
    case STATION = "stations", METAR = "metars", TAF = "tafs"
}

public class AddsService {
    
    class func fetchStations(at region: WeatherRegion) -> Promise<[Station]> {
        return fetch(dataSource: "stations", at: region).map { data -> [Station] in
            print("Found \(data.children.count) ADDS stations")
            
            return data.children.map { station in
                Station(
                    identifier: station["station_id"].element!.text,
                    name: station["site"].element!.text,
                    latitude: Float(station["latitude"].element!.text)!,
                    longitude: Float(station["longitude"].element!.text)!,
                    elevation: Float(station["elevation_m"].element!.text)!,
                    hasMetar: station["site_type"]["METAR"].element != nil,
                    hasTaf: station["site_type"]["TAF"].element != nil
                )
            }
        }
    }
    
    class func fetchObservations(_ source: AddsSource, history: Bool, at region: WeatherRegion) -> Promise<[String]> {
        let query = [
            "hoursBeforeNow": "3",
            "mostRecentForEachStation": String(history == false),
            "fields": "raw_text"
        ]
        return fetch(dataSource: source.rawValue, with: query, at: region).map { data -> [String] in
            let count = data.children.count
            print("Found \(count) ADDS \(source.rawValue)")
            
            let raws = data.children.compactMap { item in
                item["raw_text"].element?.text
            }
            // remove possible duplicate entries
            return Array(Set(raws))
        }
    }
    
    private class func parse(data: Data) throws -> XMLIndexer {
        let response = SWXMLHash.parse(data)["response"]
    
        let errors = response["errors"]
        guard errors.children.count == 0  else {
            let msgs = errors["error"].all.compactMap { $0.element?.text }
            throw Weather.error(msg: msgs.joined(separator: ", "))
        }
        
        let warnings = response["warnings"]
        if warnings.children.count > 0 {
            let msgs = warnings["warning"].all.compactMap { $0.element?.text }
            print("ADDS warning: \(msgs.joined())")
        }
        
        return response["data"]
    }
    
    private class func fetch(dataSource: String, with: [String: String] = [:], at region: WeatherRegion) -> Promise<XMLIndexer> {
        var params = [
            "dataSource": dataSource,
            "requestType": "retrieve",
            "format": "xml",
//            "compression": "gzip",
            "minLat": String(region.minLat),
            "minLon": String(region.minLon),
            "maxLat": String(region.maxLat),
            "maxLon": String(region.maxLon)
        ]<commit_msg>ADDS service returns sometimes "Invalid field name(s) found: raw_text" errors for Metar query. Without fields filter this doesn't happen. The downloaded payload is larger but it works reliably.<commit_after>//
//  Adds.swift
//  CAVOK
//
//  Created by <NAME> on 24.01.15.
//  Copyright © 2016 <NAME>. All rights reserved.
//

import Foundation
import SWXMLHash
import PMKFoundation
import PromiseKit
import Gzip

enum AddsSource: String {
    case STATION = "stations", METAR = "metars", TAF = "tafs"
}

public class AddsService {
    
    class func fetchStations(at region: WeatherRegion) -> Promise<[Station]> {
        return fetch(dataSource: "stations", at: region).map { data -> [Station] in
            print("Found \(data.children.count) ADDS stations")
            
            return data.children.map { station in
                Station(
                    identifier: station["station_id"].element!.text,
                    name: station["site"].element!.text,
                    latitude: Float(station["latitude"].element!.text)!,
                    longitude: Float(station["longitude"].element!.text)!,
                    elevation: Float(station["elevation_m"].element!.text)!,
                    hasMetar: station["site_type"]["METAR"].element != nil,
                    hasTaf: station["site_type"]["TAF"].element != nil
                )
            }
        }
    }
    
    class func fetchObservations(_ source: AddsSource, history: Bool, at region: WeatherRegion) -> Promise<[String]> {
        let query = [
            "hoursBeforeNow": "3",
            "mostRecentForEachStation": String(history == false)
//            "fields": "raw_text"
        ]
        return fetch(dataSource: source.rawValue, with: query, at: region).map { data -> [String] in
            let count = data.children.count
            print("Found \(count) ADDS \(source.rawValue)")
            
            let raws = data.children.compactMap { item in
                item["raw_text"].element?.text
            }
            // remove possible duplicate entries
            return Array(Set(raws))
        }
    }
    
    private class func parse(data: Data) throws -> XMLIndexer {
        let response = SWXMLHash.parse(data)["response"]
    
        let errors = response["errors"]
        guard errors.children.count == 0  else {
            let msgs = errors["error"].all.compactMap { $0.element?.text }
            throw Weather.error(msg: msgs.joined(separator: ", "))
        }
        
        let warnings = response["warnings"]
        if warnings.children.count > 0 {
            let msgs = warnings["warning"].all.compactMap { $0.element?.text }
            print("ADDS warning: \(msgs.joined())")
        }
        
        return response["data"]
    }
    
    private class func fetch(dataSource: String, with: [String: String] = [:], at region: WeatherRegion) -> Promise<XMLIndexer> {
        var params = [
            "dataSource": dataSource,
            "requestType": "retrieve",
            "format": "xml",
            "compression": "gzip",
            "minLat": String(region.minLat),
            "minLon": String(region.minLon),
            "maxLat": String(region.maxLat),
            "maxLon": String(region.maxLon)
        ]
{'fraction_non_alphanumeric': 0.12930627654554036, 'fraction_numerical': 0.004719207173194903, 'mean_word_length': 1.9779859484777518, 'pattern_counts': {'":': 22, '<': 13, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6874605', 'n_tokens_mistral': 1806, 'n_tokens_neox': 1719, 'n_words': 417}