<commit_before>
import CoreData
#if os(iOS)
import UIKit
#endif

struct UrlBackOffEntry {
	var nextAttemptAt: NSDate
	var duration: NSTimeInterval
}

final class API {

	var refreshesSinceLastStatusCheck = [NSManagedObjectID:Int]()
	var refreshesSinceLastLabelsCheck = [NSManagedObjectID:Int]()
	var currentNetworkStatus: NetworkStatus

	private let mediumFormatter: NSDateFormatter
	private let cacheDirectory: String
	private let urlSession: NSURLSession
	private var badLinks = [String:UrlBackOffEntry]()
	#if os(iOS)
	private var networkIndicationCount: Int = 0
	#endif

	init() {

		mediumFormatter = NSDateFormatter()
		mediumFormatter.dateStyle = NSDateFormatterStyle.MediumStyle
		mediumFormatter.timeStyle = NSDateFormatterStyle.MediumStyle

		var reachability = Reachability.reachabilityForInternetConnection()
		reachability.startNotifier()
		currentNetworkStatus = reachability.currentReachabilityStatus()

		let fileManager = NSFileManager.defaultManager()
		let appSupportURL = fileManager.URLsForDirectory(NSSearchPathDirectory.CachesDirectory, inDomains: NSSearchPathDomainMask.UserDomainMask).first! as! NSURL
		cacheDirectory = appSupportURL.URLByAppendingPathComponent("com.housetrip.Trailer").path!

        #if DEBUG
            #if os(iOS)
                let userAgent = "HouseTrip-Trailer-v\(currentAppVersion)-iOS-Development"
            #else
                let userAgent = "HouseTrip-Trailer-v\(currentAppVersion)-OSX-Development"
            #endif
        #else
            #if os(iOS)
                let userAgent = "HouseTrip-Trailer-v\(currentAppVersion)-iOS-Release"
            #else
                let userAgent = "HouseTrip-Trailer-v\(currentAppVersion)-OSX-Release"
            #endif
        #endif

		let config = NSURLSessionConfiguration.defaultSessionConfiguration()
		config.HTTPMaximumConnectionsPerHost = 4
		config.HTTPShouldUsePipelining = true
		config.timeoutIntervalForResource = NETWORK_TIMEOUT
		config.timeoutIntervalForRequest = NETWORK_TIMEOUT
		config.HTTPAdditionalHeaders = ["User-Agent" : userAgent]
		urlSession = NSURLSession(configuration: config)

		if fileManager.fileExistsAtPath(cacheDirectory) {
			clearImageCache()
		} else {
			fileManager.createDirectoryAtPath(cacheDirectory, withIntermediateDirectories: true, attributes: nil, error: nil)
		}

		NSNotificationCenter.defaultCenter().addObserverForName(kReachabilityChangedNotification, object: nil, queue: NSOperationQueue.mainQueue()) { [weak self] n in
			let newStatus = (n.object as! Reachability).currentReachabilityStatus()
			if  newStatus != self!.currentNetworkStatus {
				self!.currentNetworkStatus = newStatus
				if newStatus == NetworkStatus.NotReachable {
					DLog("Network went down: %d", newStatus.rawValue)
				} else {
					DLog("Network came up: %d", newStatus.rawValue)
					app.startRefreshIfItIsDue()
				}
			}
		}
	}

	/////////////////////////////////////////////////////// Utilities

	func resetBadLinks() {
		badLinks.removeAll(keepCapacity: false)
	}

	func lastUpdateDescription() -> String {
		if app.isRefreshing {
			return "Refreshing..."
		} else if ApiServer.shouldReportRefreshFailureInMoc(mainObjectContext) {
			return "Last update failed"
		} else {
			let lastSuccess = Settings.lastSuccessfulRefresh ?? NSDate()
			let ago = NSDate().timeIntervalSinceDate(lastSuccess)
			if ago<10 {
				return "Just updated"
			} else {
				return "Updated \(Int(ago)) seconds ago"
			}
		}
	}

	private func markLongCleanReposAsDirtyInMoc(moc: NSManagedObjectContext) {
		let f = NSFetchRequest(entityName: "Repo")
		f.predicate = NSPredicate(format: "dirty != YES and lastDirtied < %@", NSDate(timeInterval: -3600, sinceDate: NSDate()))
		f.includesPropertyValues = false
		f.returnsObjectsAsFaults = false
		let reposNotFetchedRecently = moc.executeFetchRequest(f, error: nil) as! [Repo]
		for r in reposNotFetchedRecently {
			r.resetSyncState()
		}

		if reposNotFetchedRecently.count>0 {
			DLog("Marked dirty %d repos which haven't been refreshed in over an hour", reposNotFetchedRecently.count)
		}
	}

	///////////////////////////////////////////////////////// Images

	private func clearImageCache() {
		let fileManager = NSFileManager.defaultManager()
		let files = fileManager.contentsOfDirectoryAtPath(cacheDirectory, error:nil) as? [String]
		for f in files! {
			if startsWith(f, "imgcache-") {
				let path = cacheDirectory.stringByAppendingPathComponent(f)
				fileManager.removeItemAtPath(path, error:nil)
			}
		}
	}

	func expireOldImageCacheEntries() {
		let fileManager = NSFileManager.defaultManager()
		let files = fileManager.contentsOfDirectoryAtPath(cacheDirectory, error:nil) as? [String]
		let now = NSDate()
		for f in files ?? [] {
			if startsWith(f, "imgcache-") {
				let path = cacheDirectory.stringByAppendingPathComponent(f)
				let attributes = fileManager.attributesOfItemAtPath(path, error: nil)!
				let date = attributes[NSFileCreationDate] as! NSDate
				if now.timeIntervalSinceDate(date) > (3600.0*24) {
					fileManager.removeItemAtPath(path, error:nil)
				}
			}
		}
	}

	// warning: now calls back on thread!!
	func getImage(url: NSURL, completion:(response: NSHTTPURLResponse?, data: NSData?, error: NSError?) -> Void) {

            let task = urlSession.dataTaskWithURL(url) { [weak self] data, res, e in

				let response = res as? NSHTTPURLResponse
				var error = e
				if error == nil && response?.statusCode>399 {
					error = NSError(domain: "Error response received", code: response!.statusCode, userInfo: nil)
				}
                completion(response: response, data: data, error: error)
				#if os(iOS)
					self!.networkIndicationEnd()
				#endif
			}

            #if os(iOS)
                task.priority = NSURLSessionTaskPriorityHigh
                atNextEvent { [weak self] in
					self!.networkIndicationStart()
                }
            #endif

            task.resume()
	}

	func haveCachedAvatar(path: String, tryLoadAndCallback: (IMAGE_CLASS?) -> Void) -> Bool {

		#if os(iOS)
			let absolutePath = path + (contains(path, "?") ? "&" : "?") + "s=\(40.0*GLOBAL_SCREEN_SCALE)"
        #else
			let absolutePath = path + (contains(path, "?") ? "&" : "?") + "s=88"
		#endif

		let imageKey = absolutePath + " " + currentAppVersion
		let cachePath = cacheDirectory.stringByAppendingPathComponent("imgcache-" + md5hash(imageKey))

		let fileManager = NSFileManager.defaultManager()
		if fileManager.fileExistsAtPath(cachePath) {
			#if os(iOS)
				let imgData = NSData(contentsOfFile: cachePath)
				let imgDataProvider = CGDataProviderCreateWithCFData(imgData)
				let cfImage = CGImageCreateWithJPEGDataProvider(imgDataProvider, nil, false, kCGRenderingIntentDefault)
				let ret = UIImage(CGImage: cfImage, scale: GLOBAL_SCREEN_SCALE, orientation:UIImageOrientation.Up)
            #else
				let ret = NSImage(contentsOfFile: cachePath)
			#endif
			if let r = ret {
				tryLoadAndCallback(r)
				return true
			} else {
				fileManager.removeItemAtPath(cachePath, error: nil)
			}
		}

        getImage(NSURL(string: absolutePath)!) { response, data, error in

            #if os(iOS)
                if let d = data, i = IMAGE_CLASS(data: d, scale:GLOBAL_SCREEN_SCALE) {
                    UIImageJPEGRepresentation(i, 1.0).writeToFile(cachePath, atomically: true)
                    dispatch_sync(dispatch_get_main_queue()) { tryLoadAndCallback(i) }
                }
            #else
                if let d = data, i = IMAGE_CLASS(data: d) {
                    i.TIFFRepresentation?.writeToFile(cachePath, atomically: true)
                    dispatch_sync(dispatch_get_main_queue()) { tryLoadAndCallback(i) }
                }
            #endif
        }
		return false
	}

	////////////////////////////////////// API interface

	func syncItemsForActiveReposAndCallback(callback: Completion) {
		let syncContext = DataManager.tempContext()

		let shouldRefreshReposToo = (app.lastRepoCheck.isEqualToDate(never())
			|| (NSDate().timeIntervalSinceDate(app.lastRepoCheck) < NSTimeInterval(Settings.newRepoCheckPeriod*3600.0))
			|| (Repo.countVisibleReposInMoc(syncContext)==0))

		if shouldRefreshReposToo {
			fetchRepositoriesToMoc(syncContext) { [weak self] in
				self!.syncToMoc(syncContext, callback: callback)
			}
		} else {<commit_msg>Repo refresh frequency problem fixed<commit_after>
import CoreData
#if os(iOS)
import UIKit
#endif

struct UrlBackOffEntry {
	var nextAttemptAt: NSDate
	var duration: NSTimeInterval
}

final class API {

	var refreshesSinceLastStatusCheck = [NSManagedObjectID:Int]()
	var refreshesSinceLastLabelsCheck = [NSManagedObjectID:Int]()
	var currentNetworkStatus: NetworkStatus

	private let mediumFormatter: NSDateFormatter
	private let cacheDirectory: String
	private let urlSession: NSURLSession
	private var badLinks = [String:UrlBackOffEntry]()
	#if os(iOS)
	private var networkIndicationCount: Int = 0
	#endif

	init() {

		mediumFormatter = NSDateFormatter()
		mediumFormatter.dateStyle = NSDateFormatterStyle.MediumStyle
		mediumFormatter.timeStyle = NSDateFormatterStyle.MediumStyle

		var reachability = Reachability.reachabilityForInternetConnection()
		reachability.startNotifier()
		currentNetworkStatus = reachability.currentReachabilityStatus()

		let fileManager = NSFileManager.defaultManager()
		let appSupportURL = fileManager.URLsForDirectory(NSSearchPathDirectory.CachesDirectory, inDomains: NSSearchPathDomainMask.UserDomainMask).first! as! NSURL
		cacheDirectory = appSupportURL.URLByAppendingPathComponent("com.housetrip.Trailer").path!

        #if DEBUG
            #if os(iOS)
                let userAgent = "HouseTrip-Trailer-v\(currentAppVersion)-iOS-Development"
            #else
                let userAgent = "HouseTrip-Trailer-v\(currentAppVersion)-OSX-Development"
            #endif
        #else
            #if os(iOS)
                let userAgent = "HouseTrip-Trailer-v\(currentAppVersion)-iOS-Release"
            #else
                let userAgent = "HouseTrip-Trailer-v\(currentAppVersion)-OSX-Release"
            #endif
        #endif

		let config = NSURLSessionConfiguration.defaultSessionConfiguration()
		config.HTTPMaximumConnectionsPerHost = 4
		config.HTTPShouldUsePipelining = true
		config.timeoutIntervalForResource = NETWORK_TIMEOUT
		config.timeoutIntervalForRequest = NETWORK_TIMEOUT
		config.HTTPAdditionalHeaders = ["User-Agent" : userAgent]
		urlSession = NSURLSession(configuration: config)

		if fileManager.fileExistsAtPath(cacheDirectory) {
			clearImageCache()
		} else {
			fileManager.createDirectoryAtPath(cacheDirectory, withIntermediateDirectories: true, attributes: nil, error: nil)
		}

		NSNotificationCenter.defaultCenter().addObserverForName(kReachabilityChangedNotification, object: nil, queue: NSOperationQueue.mainQueue()) { [weak self] n in
			let newStatus = (n.object as! Reachability).currentReachabilityStatus()
			if  newStatus != self!.currentNetworkStatus {
				self!.currentNetworkStatus = newStatus
				if newStatus == NetworkStatus.NotReachable {
					DLog("Network went down: %d", newStatus.rawValue)
				} else {
					DLog("Network came up: %d", newStatus.rawValue)
					app.startRefreshIfItIsDue()
				}
			}
		}
	}

	/////////////////////////////////////////////////////// Utilities

	func resetBadLinks() {
		badLinks.removeAll(keepCapacity: false)
	}

	func lastUpdateDescription() -> String {
		if app.isRefreshing {
			return "Refreshing..."
		} else if ApiServer.shouldReportRefreshFailureInMoc(mainObjectContext) {
			return "Last update failed"
		} else {
			let lastSuccess = Settings.lastSuccessfulRefresh ?? NSDate()
			let ago = NSDate().timeIntervalSinceDate(lastSuccess)
			if ago<10 {
				return "Just updated"
			} else {
				return "Updated \(Int(ago)) seconds ago"
			}
		}
	}

	private func markLongCleanReposAsDirtyInMoc(moc: NSManagedObjectContext) {
		let f = NSFetchRequest(entityName: "Repo")
		f.predicate = NSPredicate(format: "dirty != YES and lastDirtied < %@", NSDate(timeInterval: -3600, sinceDate: NSDate()))
		f.includesPropertyValues = false
		f.returnsObjectsAsFaults = false
		let reposNotFetchedRecently = moc.executeFetchRequest(f, error: nil) as! [Repo]
		for r in reposNotFetchedRecently {
			r.resetSyncState()
		}

		if reposNotFetchedRecently.count>0 {
			DLog("Marked dirty %d repos which haven't been refreshed in over an hour", reposNotFetchedRecently.count)
		}
	}

	///////////////////////////////////////////////////////// Images

	private func clearImageCache() {
		let fileManager = NSFileManager.defaultManager()
		let files = fileManager.contentsOfDirectoryAtPath(cacheDirectory, error:nil) as? [String]
		for f in files! {
			if startsWith(f, "imgcache-") {
				let path = cacheDirectory.stringByAppendingPathComponent(f)
				fileManager.removeItemAtPath(path, error:nil)
			}
		}
	}

	func expireOldImageCacheEntries() {
		let fileManager = NSFileManager.defaultManager()
		let files = fileManager.contentsOfDirectoryAtPath(cacheDirectory, error:nil) as? [String]
		let now = NSDate()
		for f in files ?? [] {
			if startsWith(f, "imgcache-") {
				let path = cacheDirectory.stringByAppendingPathComponent(f)
				let attributes = fileManager.attributesOfItemAtPath(path, error: nil)!
				let date = attributes[NSFileCreationDate] as! NSDate
				if now.timeIntervalSinceDate(date) > (3600.0*24) {
					fileManager.removeItemAtPath(path, error:nil)
				}
			}
		}
	}

	// warning: now calls back on thread!!
	func getImage(url: NSURL, completion:(response: NSHTTPURLResponse?, data: NSData?, error: NSError?) -> Void) {

            let task = urlSession.dataTaskWithURL(url) { [weak self] data, res, e in

				let response = res as? NSHTTPURLResponse
				var error = e
				if error == nil && response?.statusCode>399 {
					error = NSError(domain: "Error response received", code: response!.statusCode, userInfo: nil)
				}
                completion(response: response, data: data, error: error)
				#if os(iOS)
					self!.networkIndicationEnd()
				#endif
			}

            #if os(iOS)
                task.priority = NSURLSessionTaskPriorityHigh
                atNextEvent { [weak self] in
					self!.networkIndicationStart()
                }
            #endif

            task.resume()
	}

	func haveCachedAvatar(path: String, tryLoadAndCallback: (IMAGE_CLASS?) -> Void) -> Bool {

		#if os(iOS)
			let absolutePath = path + (contains(path, "?") ? "&" : "?") + "s=\(40.0*GLOBAL_SCREEN_SCALE)"
        #else
			let absolutePath = path + (contains(path, "?") ? "&" : "?") + "s=88"
		#endif

		let imageKey = absolutePath + " " + currentAppVersion
		let cachePath = cacheDirectory.stringByAppendingPathComponent("imgcache-" + md5hash(imageKey))

		let fileManager = NSFileManager.defaultManager()
		if fileManager.fileExistsAtPath(cachePath) {
			#if os(iOS)
				let imgData = NSData(contentsOfFile: cachePath)
				let imgDataProvider = CGDataProviderCreateWithCFData(imgData)
				let cfImage = CGImageCreateWithJPEGDataProvider(imgDataProvider, nil, false, kCGRenderingIntentDefault)
				let ret = UIImage(CGImage: cfImage, scale: GLOBAL_SCREEN_SCALE, orientation:UIImageOrientation.Up)
            #else
				let ret = NSImage(contentsOfFile: cachePath)
			#endif
			if let r = ret {
				tryLoadAndCallback(r)
				return true
			} else {
				fileManager.removeItemAtPath(cachePath, error: nil)
			}
		}

        getImage(NSURL(string: absolutePath)!) { response, data, error in

            #if os(iOS)
                if let d = data, i = IMAGE_CLASS(data: d, scale:GLOBAL_SCREEN_SCALE) {
                    UIImageJPEGRepresentation(i, 1.0).writeToFile(cachePath, atomically: true)
                    dispatch_sync(dispatch_get_main_queue()) { tryLoadAndCallback(i) }
                }
            #else
                if let d = data, i = IMAGE_CLASS(data: d) {
                    i.TIFFRepresentation?.writeToFile(cachePath, atomically: true)
                    dispatch_sync(dispatch_get_main_queue()) { tryLoadAndCallback(i) }
                }
            #endif
        }
		return false
	}

	////////////////////////////////////// API interface

	func syncItemsForActiveReposAndCallback(callback: Completion) {
		let syncContext = DataManager.tempContext()

		let shouldRefreshReposToo = (app.lastRepoCheck.isEqualToDate(never())
			|| (NSDate().timeIntervalSinceDate(app.lastRepoCheck) > NSTimeInterval(Settings.newRepoCheckPeriod*3600.0))
			|| (Repo.countVisibleReposInMoc(syncContext)==0))

		if shouldRefreshReposToo {
			fetchRepositoriesToMoc(syncContext) { [weak self] in
				self!.syncToMoc(syncContext, callback: callback)
			}
		} else {
{'fraction_non_alphanumeric': 0.11581801747514311, 'fraction_numerical': 0.0039771015366074115, 'mean_word_length': 3.4103109221365933, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2174072', 'n_tokens_mistral': 5575, 'n_tokens_neox': 4945, 'n_words': 1036}