<commit_before>//
//  CloudKitArticlesZoneDelegate.swift
//  Account
//
//  Created by <NAME> on 4/1/20.
//  Copyright © 2020 Ranchero Software, LLC. All rights reserved.
//

import Foundation
import os.log
import RSParser
import RSWeb
import CloudKit
import SyncDatabase
import Articles
import ArticlesDatabase

class CloudKitArticlesZoneDelegate: CloudKitZoneDelegate {

	private var log = OSLog(subsystem: Bundle.main.bundleIdentifier!, category: "CloudKit")
	
	weak var account: Account?
	var database: SyncDatabase
	weak var articlesZone: CloudKitArticlesZone?
	
	init(account: Account, database: SyncDatabase, articlesZone: CloudKitArticlesZone) {
		self.account = account
		self.database = database
		self.articlesZone = articlesZone
	}
	
	func cloudKitDidModify(changed: [CKRecord], deleted: [CloudKitRecordKey], completion: @escaping (Result<Void, Error>) -> Void) {
		
		database.selectPendingReadStatusArticleIDs() { result in
			switch result {
			case .success(let pendingReadStatusArticleIDs):

				self.database.selectPendingStarredStatusArticleIDs() { result in
					switch result {
					case .success(let pendingStarredStatusArticleIDs):

						self.delete(recordKeys: deleted, pendingStarredStatusArticleIDs: pendingStarredStatusArticleIDs) {
							self.update(records: changed,
										 pendingReadStatusArticleIDs: pendingReadStatusArticleIDs,
										 pendingStarredStatusArticleIDs: pendingStarredStatusArticleIDs,
										 completion: completion)
						}
						
					case .failure(let error):
						os_log(.error, log: self.log, "Error occurred geting pending starred records: %@", error.localizedDescription)
						completion(.failure(CloudKitZoneError.unknown))
					}
				}
			case .failure(let error):
				os_log(.error, log: self.log, "Error occurred getting pending read status records: %@", error.localizedDescription)
				completion(.failure(CloudKitZoneError.unknown))
			}

		}
		
	}
	
}

private extension CloudKitArticlesZoneDelegate {

	func delete(recordKeys: [CloudKitRecordKey], pendingStarredStatusArticleIDs: Set<String>, completion: @escaping () -> Void) {
		let receivedRecordIDs = recordKeys.filter({ $0.recordType == CloudKitArticlesZone.CloudKitArticleStatus.recordType }).map({ $0.recordID })
		let receivedArticleIDs = Set(receivedRecordIDs.map({ stripPrefix($0.externalID) }))
		let deletableArticleIDs = receivedArticleIDs.subtracting(pendingStarredStatusArticleIDs)
		
		guard !deletableArticleIDs.isEmpty else {
			completion()
			return
		}
		
		database.deleteSelectedForProcessing(Array(deletableArticleIDs)) { _ in
			self.account?.delete(articleIDs: deletableArticleIDs) { _ in
				completion()
			}
		}
	}

	func update(records: [CKRecord], pendingReadStatusArticleIDs: Set<String>, pendingStarredStatusArticleIDs: Set<String>, completion: @escaping (Result<Void, Error>) -> Void) {

		let receivedUnreadArticleIDs = Set(records.filter({ $0[CloudKitArticlesZone.CloudKitArticleStatus.Fields.read] == "0" }).map({ stripPrefix($0.externalID) }))
		let receivedReadArticleIDs =  Set(records.filter({ $0[CloudKitArticlesZone.CloudKitArticleStatus.Fields.read] == "1" }).map({ stripPrefix($0.externalID) }))
		let receivedUnstarredArticleIDs =  Set(records.filter({ $0[CloudKitArticlesZone.CloudKitArticleStatus.Fields.starred] == "0" }).map({ stripPrefix($0.externalID) }))
		let receivedStarredArticleIDs =  Set(records.filter({ $0[CloudKitArticlesZone.CloudKitArticleStatus.Fields.starred] == "1" }).map({ stripPrefix($0.externalID) }))

		let updateableUnreadArticleIDs = receivedUnreadArticleIDs.subtracting(pendingReadStatusArticleIDs)
		let updateableReadArticleIDs = receivedReadArticleIDs.subtracting(pendingReadStatusArticleIDs)
		let updateableUnstarredArticleIDs = receivedUnstarredArticleIDs.subtracting(pendingStarredStatusArticleIDs)
		let updateableStarredArticleIDs = receivedStarredArticleIDs.subtracting(pendingStarredStatusArticleIDs)

		var errorOccurred = false
		let group = DispatchGroup()
		
		group.enter()
		account?.markAsUnread(updateableUnreadArticleIDs) { result in
			if case .failure(let databaseError) = result {
				errorOccurred = true
				os_log(.error, log: self.log, "Error occurred while storing unread statuses: %@", databaseError.localizedDescription)
			}
			group.leave()
		}
		
		group.enter()
		account?.markAsRead(updateableReadArticleIDs) { result in
			if case .failure(let databaseError) = result {
				errorOccurred = true
				os_log(.error, log: self.log, "Error occurred while storing read statuses: %@", databaseError.localizedDescription)
			}
			group.leave()
		}
		
		group.enter()
		account?.markAsUnstarred(updateableUnstarredArticleIDs) { result in
			if case .failure(let databaseError) = result {
				errorOccurred = true
				os_log(.error, log: self.log, "Error occurred while storing unstarred statuses: %@", databaseError.localizedDescription)
			}
			group.leave()
		}
		
		group.enter()
		account?.markAsStarred(updateableStarredArticleIDs) { result in
			if case .failure(let databaseError) = result {
				errorOccurred = true
				os_log(.error, log: self.log, "Error occurred while storing starred statuses: %@", databaseError.localizedDescription)
			}
			group.leave()
		}
		
		let parsedItems = records.compactMap { makeParsedItem($0) }
		let webFeedIDsAndItems = Dictionary(grouping: parsedItems, by: { item in item.feedURL } ).mapValues { Set($0) }
		for (webFeedID, parsedItems) in webFeedIDsAndItems {
			group.enter()
			self.account?.update(webFeedID, with: parsedItems, deleteOlder: false) { result in
				switch result {
				case .success(let articleChanges):
					guard let deletes = articleChanges.deletedArticles, !deletes.isEmpty else {
						group.leave()
						return
					}
					let syncStatuses = deletes.map { SyncStatus(articleID: $0.articleID, key: .deleted, flag: true) }
					try? self.database.insertStatuses(syncStatuses)
					group.leave()
				case .failure(let databaseError):
					errorOccurred = true
					os_log(.error, log: self.log, "Error occurred while storing articles: %@", databaseError.localizedDescription)
					group.leave()
				}
			}
		}
		
		group.notify(queue: DispatchQueue.main) {
			if errorOccurred {<commit_msg>Do CloudKit record process loop on a high priority background thread so that it doesn't block the main thread<commit_after>//
//  CloudKitArticlesZoneDelegate.swift
//  Account
//
//  Created by <NAME> on 4/1/20.
//  Copyright © 2020 Ranchero Software, LLC. All rights reserved.
//

import Foundation
import os.log
import RSParser
import RSWeb
import CloudKit
import SyncDatabase
import Articles
import ArticlesDatabase

class CloudKitArticlesZoneDelegate: CloudKitZoneDelegate {

	private var log = OSLog(subsystem: Bundle.main.bundleIdentifier!, category: "CloudKit")
	
	weak var account: Account?
	var database: SyncDatabase
	weak var articlesZone: CloudKitArticlesZone?
	
	init(account: Account, database: SyncDatabase, articlesZone: CloudKitArticlesZone) {
		self.account = account
		self.database = database
		self.articlesZone = articlesZone
	}
	
	func cloudKitDidModify(changed: [CKRecord], deleted: [CloudKitRecordKey], completion: @escaping (Result<Void, Error>) -> Void) {
		
		database.selectPendingReadStatusArticleIDs() { result in
			switch result {
			case .success(let pendingReadStatusArticleIDs):

				self.database.selectPendingStarredStatusArticleIDs() { result in
					switch result {
					case .success(let pendingStarredStatusArticleIDs):

						self.delete(recordKeys: deleted, pendingStarredStatusArticleIDs: pendingStarredStatusArticleIDs) {
							self.update(records: changed,
										 pendingReadStatusArticleIDs: pendingReadStatusArticleIDs,
										 pendingStarredStatusArticleIDs: pendingStarredStatusArticleIDs,
										 completion: completion)
						}
						
					case .failure(let error):
						os_log(.error, log: self.log, "Error occurred geting pending starred records: %@", error.localizedDescription)
						completion(.failure(CloudKitZoneError.unknown))
					}
				}
			case .failure(let error):
				os_log(.error, log: self.log, "Error occurred getting pending read status records: %@", error.localizedDescription)
				completion(.failure(CloudKitZoneError.unknown))
			}

		}
		
	}
	
}

private extension CloudKitArticlesZoneDelegate {

	func delete(recordKeys: [CloudKitRecordKey], pendingStarredStatusArticleIDs: Set<String>, completion: @escaping () -> Void) {
		let receivedRecordIDs = recordKeys.filter({ $0.recordType == CloudKitArticlesZone.CloudKitArticleStatus.recordType }).map({ $0.recordID })
		let receivedArticleIDs = Set(receivedRecordIDs.map({ stripPrefix($0.externalID) }))
		let deletableArticleIDs = receivedArticleIDs.subtracting(pendingStarredStatusArticleIDs)
		
		guard !deletableArticleIDs.isEmpty else {
			completion()
			return
		}
		
		database.deleteSelectedForProcessing(Array(deletableArticleIDs)) { _ in
			self.account?.delete(articleIDs: deletableArticleIDs) { _ in
				completion()
			}
		}
	}

	func update(records: [CKRecord], pendingReadStatusArticleIDs: Set<String>, pendingStarredStatusArticleIDs: Set<String>, completion: @escaping (Result<Void, Error>) -> Void) {

		let receivedUnreadArticleIDs = Set(records.filter({ $0[CloudKitArticlesZone.CloudKitArticleStatus.Fields.read] == "0" }).map({ stripPrefix($0.externalID) }))
		let receivedReadArticleIDs =  Set(records.filter({ $0[CloudKitArticlesZone.CloudKitArticleStatus.Fields.read] == "1" }).map({ stripPrefix($0.externalID) }))
		let receivedUnstarredArticleIDs =  Set(records.filter({ $0[CloudKitArticlesZone.CloudKitArticleStatus.Fields.starred] == "0" }).map({ stripPrefix($0.externalID) }))
		let receivedStarredArticleIDs =  Set(records.filter({ $0[CloudKitArticlesZone.CloudKitArticleStatus.Fields.starred] == "1" }).map({ stripPrefix($0.externalID) }))

		let updateableUnreadArticleIDs = receivedUnreadArticleIDs.subtracting(pendingReadStatusArticleIDs)
		let updateableReadArticleIDs = receivedReadArticleIDs.subtracting(pendingReadStatusArticleIDs)
		let updateableUnstarredArticleIDs = receivedUnstarredArticleIDs.subtracting(pendingStarredStatusArticleIDs)
		let updateableStarredArticleIDs = receivedStarredArticleIDs.subtracting(pendingStarredStatusArticleIDs)

		var errorOccurred = false
		let group = DispatchGroup()
		
		group.enter()
		account?.markAsUnread(updateableUnreadArticleIDs) { result in
			if case .failure(let databaseError) = result {
				errorOccurred = true
				os_log(.error, log: self.log, "Error occurred while storing unread statuses: %@", databaseError.localizedDescription)
			}
			group.leave()
		}
		
		group.enter()
		account?.markAsRead(updateableReadArticleIDs) { result in
			if case .failure(let databaseError) = result {
				errorOccurred = true
				os_log(.error, log: self.log, "Error occurred while storing read statuses: %@", databaseError.localizedDescription)
			}
			group.leave()
		}
		
		group.enter()
		account?.markAsUnstarred(updateableUnstarredArticleIDs) { result in
			if case .failure(let databaseError) = result {
				errorOccurred = true
				os_log(.error, log: self.log, "Error occurred while storing unstarred statuses: %@", databaseError.localizedDescription)
			}
			group.leave()
		}
		
		group.enter()
		account?.markAsStarred(updateableStarredArticleIDs) { result in
			if case .failure(let databaseError) = result {
				errorOccurred = true
				os_log(.error, log: self.log, "Error occurred while storing starred statuses: %@", databaseError.localizedDescription)
			}
			group.leave()
		}
		
		group.enter()
		DispatchQueue.global(qos: .userInitiated).async {
			let parsedItems = records.compactMap { self.makeParsedItem($0) }
			let webFeedIDsAndItems = Dictionary(grouping: parsedItems, by: { item in item.feedURL } ).mapValues { Set($0) }
			
			DispatchQueue.main.async {
				for (webFeedID, parsedItems) in webFeedIDsAndItems {
					group.enter()
					self.account?.update(webFeedID, with: parsedItems, deleteOlder: false) { result in
						switch result {
						case .success(let articleChanges):
							guard let deletes = articleChanges.deletedArticles, !deletes.isEmpty else {
								group.leave()
								return
							}
							let syncStatuses = deletes.map { SyncStatus(articleID: $0.articleID, key: .deleted, flag: true) }
							try? self.database.insertStatuses(syncStatuses)
							group.leave()
						case .failure(let databaseError):
							errorOccurred = true
							os_log(.error, log: self.log, "Error occurred while storing articles: %@", databaseError.localizedDescription)
							group.leave()
						}
					}
				}
				group.leave()
			}
			
		}
		
		group.notify(queue: DispatchQueue.main) {
			if errorOccurred {
{'fraction_non_alphanumeric': 0.10289236605026078, 'fraction_numerical': 0.004109372530425162, 'mean_word_length': 4.629448398576512, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4739152', 'n_tokens_mistral': 4456, 'n_tokens_neox': 3622, 'n_words': 795}