<commit_before>//
//  CloudKitAppDelegate.swift
//  Account
//
//  Created by <NAME> on 3/18/20.
//  Copyright © 2020 Ranchero Software, LLC. All rights reserved.
//

import Foundation
import CloudKit
import os.log
import SyncDatabase
import RSCore
import RSParser
import Articles
import RSWeb

public enum CloudKitAccountDelegateError: String, Error {
	case invalidParameter = "An invalid parameter was used."
}

final class CloudKitAccountDelegate: AccountDelegate {

	private var log = OSLog(subsystem: Bundle.main.bundleIdentifier!, category: "CloudKit")

	private let database: SyncDatabase
	
	private let container: CKContainer = {
		let orgID = Bundle.main.object(forInfoDictionaryKey: "OrganizationIdentifier") as! String
		return CKContainer(identifier: "iCloud.\(orgID).NetNewsWire")
	}()
	
	private lazy var zones: [CloudKitZone] = [accountZone, articlesZone, publicZone]
	private let accountZone: CloudKitAccountZone
	private let articlesZone: CloudKitArticlesZone
	private let publicZone: CloudKitPublicZone
	
	private let refresher = LocalAccountRefresher()

	let behaviors: AccountBehaviors = []
	let isOPMLImportInProgress = false
	
	let server: String? = nil
	var credentials: Credentials?
	var accountMetadata: AccountMetadata?

	var refreshProgress = DownloadProgress(numberOfTasks: 0)
	
	init(dataFolder: String) {
		accountZone = CloudKitAccountZone(container: container)
		articlesZone = CloudKitArticlesZone(container: container)
		publicZone = CloudKitPublicZone(container: container)
		
		let databaseFilePath = (dataFolder as NSString).appendingPathComponent("Sync.sqlite3")
		database = SyncDatabase(databaseFilePath: databaseFilePath)
	}
	
	func receiveRemoteNotification(for account: Account, userInfo: [AnyHashable : Any], completion: @escaping () -> Void) {
		os_log(.debug, log: log, "Processing remote notification...")

		let group = DispatchGroup()
		
		zones.forEach { zone in
			group.enter()
			zone.receiveRemoteNotification(userInfo: userInfo) {
				group.leave()
			}
		}
		
		group.notify(queue: DispatchQueue.main) {
			os_log(.debug, log: self.log, "Done processing remote notification...")
			completion()
		}
	}
	
	func refreshAll(for account: Account, completion: @escaping (Result<Void, Error>) -> Void) {
		refreshAll(for: account, downloadFeeds: true, completion: completion)
	}

	func sendArticleStatus(for account: Account, completion: @escaping ((Result<Void, Error>) -> Void)) {
		os_log(.debug, log: log, "Sending article statuses...")

		database.selectForProcessing { result in

			func processStatuses(_ syncStatuses: [SyncStatus]) {
				guard syncStatuses.count > 0 else {
					completion(.success(()))
					return
				}
				
				let starredArticleIDs = syncStatuses.filter({ $0.key == .starred && $0.flag == true }).map({ $0.articleID })
				account.fetchArticlesAsync(.articleIDs(Set(starredArticleIDs))) { result in
					
					func processWithArticles(_ starredArticles: Set<Article>) {
						
						self.articlesZone.sendArticleStatus(syncStatuses, starredArticles: starredArticles) { result in
							switch result {
							case .success:
								self.database.deleteSelectedForProcessing(syncStatuses.map({ $0.articleID }) )
								os_log(.debug, log: self.log, "Done sending article statuses.")
								completion(.success(()))
							case .failure(let error):
								self.database.resetSelectedForProcessing(syncStatuses.map({ $0.articleID }) )
								completion(.failure(error))
							}
						}
						
					}

					switch result {
					case .success(let starredArticles):
						processWithArticles(starredArticles)
					case .failure(let databaseError):
						completion(.failure(databaseError))
					}

				}
				
			}

			switch result {
			case .success(let syncStatuses):
				processStatuses(syncStatuses)
			case .failure(let databaseError):
				completion(.failure(databaseError))
			}
		}
	}
	
	
	func refreshArticleStatus(for account: Account, completion: @escaping ((Result<Void, Error>) -> Void)) {
		os_log(.debug, log: log, "Refreshing article statuses...")
		
		articlesZone.fetchChangesInZone() { result in
			os_log(.debug, log: self.log, "Done refreshing article statuses.")
			switch result {
			case .success:
				completion(.success(()))
			case .failure(let error):
				completion(.failure(error))
			}
		}
	}
	
	func importOPML(for account:Account, opmlFile: URL, completion: @escaping (Result<Void, Error>) -> Void) {
		var fileData: Data?
		
		do {
			fileData = try Data(contentsOf: opmlFile)
		} catch {
			completion(.failure(error))
			return
		}
		
		guard let opmlData = fileData else {
			completion(.success(()))
			return
		}
		
		let parserData = ParserData(url: opmlFile.absoluteString, data: opmlData)
		var opmlDocument: RSOPMLDocument?
		
		do {
			opmlDocument = try RSOPMLParser.parseOPML(with: parserData)
		} catch {
			completion(.failure(error))
			return
		}
		
		guard let loadDocument = opmlDocument else {
			completion(.success(()))
			return
		}

		guard let opmlItems = loadDocument.children, let rootExternalID = account.externalID else {
			return
		}

		let normalizedItems = OPMLNormalizer.normalize(opmlItems)
		
		// Combine all existing web feed URLs with all the new ones
		
		var webFeedURLs = account.flattenedWebFeedURLs
		for opmlItem in normalizedItems {
			if let webFeedURL = opmlItem.feedSpecifier?.feedURL {
				webFeedURLs.insert(webFeedURL)
			} else {
				if let childItems = opmlItem.children {
					for childItem in childItems {
						if let webFeedURL = childItem.feedSpecifier?.feedURL {
							webFeedURLs.insert(webFeedURL)
						}
					}
				}
			}
		}
		
//		os_log(.error, log: self.log, "Error while subscribing to the feed: %@", error.localizedDescription)

		refreshProgress.addToNumberOfTasksAndRemaining(2)
		publicZone.manageSubscriptions(webFeedURLs) { result in
			self.refreshProgress.completeTask()
			switch result {
			case .success:
				self.accountZone.importOPML(rootExternalID: rootExternalID, items: normalizedItems) { _ in
					self.refreshAll(for: account, completion: completion)
				}
			case .failure(let error):
				completion(.failure(error))
			}<commit_msg>Fix bug that was causing duplicate downloads on OPML import.<commit_after>//
//  CloudKitAppDelegate.swift
//  Account
//
//  Created by <NAME> on 3/18/20.
//  Copyright © 2020 Ranchero Software, LLC. All rights reserved.
//

import Foundation
import CloudKit
import os.log
import SyncDatabase
import RSCore
import RSParser
import Articles
import RSWeb

public enum CloudKitAccountDelegateError: String, Error {
	case invalidParameter = "An invalid parameter was used."
}

final class CloudKitAccountDelegate: AccountDelegate {

	private var log = OSLog(subsystem: Bundle.main.bundleIdentifier!, category: "CloudKit")

	private let database: SyncDatabase
	
	private let container: CKContainer = {
		let orgID = Bundle.main.object(forInfoDictionaryKey: "OrganizationIdentifier") as! String
		return CKContainer(identifier: "iCloud.\(orgID).NetNewsWire")
	}()
	
	private lazy var zones: [CloudKitZone] = [accountZone, articlesZone, publicZone]
	private let accountZone: CloudKitAccountZone
	private let articlesZone: CloudKitArticlesZone
	private let publicZone: CloudKitPublicZone
	
	private let refresher = LocalAccountRefresher()

	let behaviors: AccountBehaviors = []
	let isOPMLImportInProgress = false
	
	let server: String? = nil
	var credentials: Credentials?
	var accountMetadata: AccountMetadata?

	var refreshProgress = DownloadProgress(numberOfTasks: 0)
	
	init(dataFolder: String) {
		accountZone = CloudKitAccountZone(container: container)
		articlesZone = CloudKitArticlesZone(container: container)
		publicZone = CloudKitPublicZone(container: container)
		
		let databaseFilePath = (dataFolder as NSString).appendingPathComponent("Sync.sqlite3")
		database = SyncDatabase(databaseFilePath: databaseFilePath)
	}
	
	func receiveRemoteNotification(for account: Account, userInfo: [AnyHashable : Any], completion: @escaping () -> Void) {
		os_log(.debug, log: log, "Processing remote notification...")

		let group = DispatchGroup()
		
		zones.forEach { zone in
			group.enter()
			zone.receiveRemoteNotification(userInfo: userInfo) {
				group.leave()
			}
		}
		
		group.notify(queue: DispatchQueue.main) {
			os_log(.debug, log: self.log, "Done processing remote notification...")
			completion()
		}
	}
	
	func refreshAll(for account: Account, completion: @escaping (Result<Void, Error>) -> Void) {
		refreshAll(for: account, downloadFeeds: true, completion: completion)
	}

	func sendArticleStatus(for account: Account, completion: @escaping ((Result<Void, Error>) -> Void)) {
		os_log(.debug, log: log, "Sending article statuses...")

		database.selectForProcessing { result in

			func processStatuses(_ syncStatuses: [SyncStatus]) {
				guard syncStatuses.count > 0 else {
					completion(.success(()))
					return
				}
				
				let starredArticleIDs = syncStatuses.filter({ $0.key == .starred && $0.flag == true }).map({ $0.articleID })
				account.fetchArticlesAsync(.articleIDs(Set(starredArticleIDs))) { result in
					
					func processWithArticles(_ starredArticles: Set<Article>) {
						
						self.articlesZone.sendArticleStatus(syncStatuses, starredArticles: starredArticles) { result in
							switch result {
							case .success:
								self.database.deleteSelectedForProcessing(syncStatuses.map({ $0.articleID }) )
								os_log(.debug, log: self.log, "Done sending article statuses.")
								completion(.success(()))
							case .failure(let error):
								self.database.resetSelectedForProcessing(syncStatuses.map({ $0.articleID }) )
								completion(.failure(error))
							}
						}
						
					}

					switch result {
					case .success(let starredArticles):
						processWithArticles(starredArticles)
					case .failure(let databaseError):
						completion(.failure(databaseError))
					}

				}
				
			}

			switch result {
			case .success(let syncStatuses):
				processStatuses(syncStatuses)
			case .failure(let databaseError):
				completion(.failure(databaseError))
			}
		}
	}
	
	
	func refreshArticleStatus(for account: Account, completion: @escaping ((Result<Void, Error>) -> Void)) {
		os_log(.debug, log: log, "Refreshing article statuses...")
		
		articlesZone.fetchChangesInZone() { result in
			os_log(.debug, log: self.log, "Done refreshing article statuses.")
			switch result {
			case .success:
				completion(.success(()))
			case .failure(let error):
				completion(.failure(error))
			}
		}
	}
	
	func importOPML(for account:Account, opmlFile: URL, completion: @escaping (Result<Void, Error>) -> Void) {
		var fileData: Data?
		
		do {
			fileData = try Data(contentsOf: opmlFile)
		} catch {
			completion(.failure(error))
			return
		}
		
		guard let opmlData = fileData else {
			completion(.success(()))
			return
		}
		
		let parserData = ParserData(url: opmlFile.absoluteString, data: opmlData)
		var opmlDocument: RSOPMLDocument?
		
		do {
			opmlDocument = try RSOPMLParser.parseOPML(with: parserData)
		} catch {
			completion(.failure(error))
			return
		}
		
		guard let loadDocument = opmlDocument else {
			completion(.success(()))
			return
		}

		guard let opmlItems = loadDocument.children, let rootExternalID = account.externalID else {
			return
		}

		let normalizedItems = OPMLNormalizer.normalize(opmlItems)
		
		// Combine all existing web feed URLs with all the new ones
		
		var webFeedURLs = account.flattenedWebFeedURLs
		for opmlItem in normalizedItems {
			if let webFeedURL = opmlItem.feedSpecifier?.feedURL {
				webFeedURLs.insert(webFeedURL)
			} else {
				if let childItems = opmlItem.children {
					for childItem in childItems {
						if let webFeedURL = childItem.feedSpecifier?.feedURL {
							webFeedURLs.insert(webFeedURL)
						}
					}
				}
			}
		}
		
		refreshProgress.addToNumberOfTasksAndRemaining(2)
		publicZone.manageSubscriptions(webFeedURLs) { result in
			self.refreshProgress.completeTask()
			switch result {
			case .success:
				self.accountZone.importOPML(rootExternalID: rootExternalID, items: normalizedItems) { _ in
					self.refreshAll(for: account, downloadFeeds: false, completion: completion)
				}
			case .failure(let error):
				completion(.failure(error))
			}
{'fraction_non_alphanumeric': 0.10407979723653013, 'fraction_numerical': 0.0029433406916850625, 'mean_word_length': 3.857823669579031, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '674493', 'n_tokens_mistral': 4467, 'n_tokens_neox': 3518, 'n_words': 887}