<commit_before>//
//  Filter.swift
//  FireAlarm
//
//  Created by NobodyNada on 9/24/16.
//  Copyright © 2016 NobodyNada. All rights reserved.
//

import Foundation

class Word {
	let text: String
	let trueProbability: Double
	let falseProbability: Double
	
	init(_ text: String, _ pTrue: Double, _ pFalse: Double) {
		self.text = text
		trueProbability = pTrue
		falseProbability = pFalse
	}
}

class Filter: WebSocketDelegate {
	let bot: ChatBot
	let client: Client
	
	let initialProbability: Double
	let words: [String:Word]
	
	var recentlyReportedPosts = [(id: Int, when: Date)]()
	
	init(_ bot: ChatBot) {
		self.bot = bot
		client = bot.room.client
		
		print("Loading filter...")
		
		let data = try! Data(contentsOf: saveDirURL.appendingPathComponent("filter.json"))
		let db = try! JSONSerialization.jsonObject(with: data, options: []) as! [String:Any]
		initialProbability = db["initialProbability"] as! Double
		var words = [String:Word]()
		for (word, probabilites) in db["wordProbabilities"] as! [String:[Double]] {
			words[word] = Word(word, probabilites.first!, probabilites.last!)
		}
		
		self.words = words
	}
	
	var ws: WebSocket!
	
	fileprivate var wsRetries = 0
	fileprivate let wsMaxRetries = 10
	
	private var _running = false
	
	
	var running: Bool {
		return _running
	}
	
	func start() throws {
		_running = true
		
		let request = URLRequest(url: URL(string: "wss://qa.sockets.stackexchange.com/")!)
		ws = WebSocket(request: request)
		ws.eventQueue = bot.room.client.queue
		ws.delegate = self
		ws.open()
	}
	
	func stop() {
		_running = false
		ws?.close()
	}
	
	func webSocketOpen() {
		print("Listening to active questions!")
		ws.send("155-questions-active")
	}
	
	func webSocketClose(_ code: Int, reason: String, wasClean: Bool) {
		//do nothing -- we'll handle this in webSocketEnd
	}
	
	func webSocketError(_ error: NSError) {
		//do nothing -- we'll handle this in webSocketEnd
	}
	
	enum QuestionProcessingError: Error {
		case textNotUTF8(text: String)
		
		case jsonNotDictionary(json: String)
		case jsonParsingError(json: String, error: NSError)
		case noDataObject(json: String)
		case noQuestionID(json: String)
		case noSite(json: String)
	}
	
	func checkPost(_ post: Post) -> Bool {
		var trueProbability = Double(0.263)
		var falseProbability = Double(1 - trueProbability)
		var checkedWords = [String]()
		
		let body = post.body
		
		for postWord in body.lowercased().components(separatedBy: CharacterSet.alphanumerics.inverted) {
			if postWord.isEmpty {
				continue
			}
			guard let word = words[postWord] else {
				continue
			}
			checkedWords.append(postWord)
			
			let pTrue = word.trueProbability
			let pFalse = word.falseProbability
			
			
			let newTrue = trueProbability * Double(pTrue)
			let newFalse = falseProbability * Double(pFalse)
			if newTrue != 0.0 && newFalse != 0.0 {
				trueProbability = newTrue
				falseProbability = newFalse
			}
		}
		
		return trueProbability * 1e45 > falseProbability
	}
	
	func checkAndReportPost(_ post: Post) throws {
		let bad = checkPost(post)
		if bad {
			print("Reporting question \(post.id).")
			reportPost(post)
		}
	}
	
	func reportPost(_ post: Post) {
		if let minDate: Date = Calendar(identifier: .gregorian).date(byAdding: DateComponents(hour: -6), to: Date()) {
			recentlyReportedPosts = recentlyReportedPosts.filter {
				$0.when > minDate
			}
		}
		else {
			bot.room.postMessage("Failed to calculate minimum report date!")
		}
		
		if recentlyReportedPosts.contains(where: { $0.id == post.id }) {
			return
		}
		
		recentlyReportedPosts.append((id: post.id, when: Date()))
		bot.room.postMessage("[ [FireAlarm-Swift](\(githubLink)) ] " +
			"[tag:\(post.tags.first ?? "tagless")] Potentially bad question: [\(post.title)](//stackoverflow.com/q/\(post.id)) " +
			bot.room.notificationString(tags: post.tags)
		)
	}
	
	func webSocketMessageText(_ text: String) {
		do {
			guard let data = text.data(using: .utf8) else {
				throw QuestionProcessingError.textNotUTF8(text: text)
			}
			webSocketMessageData(data)
		} catch {
			handleError(error, "while processing an active question")
		}
	}
	
	func webSocketMessageData(_ data: Data) {
		let string = String(data: data, encoding: .utf8) ?? "<not UTF-8: \(data.base64EncodedString())>"
		do {
			
			do {
				guard let json = try JSONSerialization.jsonObject(with: data, options: []) as? [String:String] else {
					throw QuestionProcessingError.jsonNotDictionary(json: string)
				}
				
				guard let dataObject = json["data"]?.data(using: .utf8) else {
					throw QuestionProcessingError.noDataObject(json: string)
				}<commit_msg>added a print when a duplicate report is skipped so the logs don't confuse me<commit_after>//
//  Filter.swift
//  FireAlarm
//
//  Created by NobodyNada on 9/24/16.
//  Copyright © 2016 NobodyNada. All rights reserved.
//

import Foundation

class Word {
	let text: String
	let trueProbability: Double
	let falseProbability: Double
	
	init(_ text: String, _ pTrue: Double, _ pFalse: Double) {
		self.text = text
		trueProbability = pTrue
		falseProbability = pFalse
	}
}

class Filter: WebSocketDelegate {
	let bot: ChatBot
	let client: Client
	
	let initialProbability: Double
	let words: [String:Word]
	
	var recentlyReportedPosts = [(id: Int, when: Date)]()
	
	init(_ bot: ChatBot) {
		self.bot = bot
		client = bot.room.client
		
		print("Loading filter...")
		
		let data = try! Data(contentsOf: saveDirURL.appendingPathComponent("filter.json"))
		let db = try! JSONSerialization.jsonObject(with: data, options: []) as! [String:Any]
		initialProbability = db["initialProbability"] as! Double
		var words = [String:Word]()
		for (word, probabilites) in db["wordProbabilities"] as! [String:[Double]] {
			words[word] = Word(word, probabilites.first!, probabilites.last!)
		}
		
		self.words = words
	}
	
	var ws: WebSocket!
	
	fileprivate var wsRetries = 0
	fileprivate let wsMaxRetries = 10
	
	private var _running = false
	
	
	var running: Bool {
		return _running
	}
	
	func start() throws {
		_running = true
		
		let request = URLRequest(url: URL(string: "wss://qa.sockets.stackexchange.com/")!)
		ws = WebSocket(request: request)
		ws.eventQueue = bot.room.client.queue
		ws.delegate = self
		ws.open()
	}
	
	func stop() {
		_running = false
		ws?.close()
	}
	
	func webSocketOpen() {
		print("Listening to active questions!")
		ws.send("155-questions-active")
	}
	
	func webSocketClose(_ code: Int, reason: String, wasClean: Bool) {
		//do nothing -- we'll handle this in webSocketEnd
	}
	
	func webSocketError(_ error: NSError) {
		//do nothing -- we'll handle this in webSocketEnd
	}
	
	enum QuestionProcessingError: Error {
		case textNotUTF8(text: String)
		
		case jsonNotDictionary(json: String)
		case jsonParsingError(json: String, error: NSError)
		case noDataObject(json: String)
		case noQuestionID(json: String)
		case noSite(json: String)
	}
	
	func checkPost(_ post: Post) -> Bool {
		var trueProbability = Double(0.263)
		var falseProbability = Double(1 - trueProbability)
		var checkedWords = [String]()
		
		let body = post.body
		
		for postWord in body.lowercased().components(separatedBy: CharacterSet.alphanumerics.inverted) {
			if postWord.isEmpty {
				continue
			}
			guard let word = words[postWord] else {
				continue
			}
			checkedWords.append(postWord)
			
			let pTrue = word.trueProbability
			let pFalse = word.falseProbability
			
			
			let newTrue = trueProbability * Double(pTrue)
			let newFalse = falseProbability * Double(pFalse)
			if newTrue != 0.0 && newFalse != 0.0 {
				trueProbability = newTrue
				falseProbability = newFalse
			}
		}
		
		return trueProbability * 1e45 > falseProbability
	}
	
	func checkAndReportPost(_ post: Post) throws {
		let bad = checkPost(post)
		if bad {
			reportPost(post)
		}
	}
	
	func reportPost(_ post: Post) {
		if let minDate: Date = Calendar(identifier: .gregorian).date(byAdding: DateComponents(hour: -6), to: Date()) {
			recentlyReportedPosts = recentlyReportedPosts.filter {
				$0.when > minDate
			}
		}
		else {
			bot.room.postMessage("Failed to calculate minimum report date!")
		}
		
		if recentlyReportedPosts.contains(where: { $0.id == post.id }) {
			print("Not reporting \(post.id) because it was recently reported.")
			return
		}
		print("Reporting question \(post.id).")
		
		recentlyReportedPosts.append((id: post.id, when: Date()))
		bot.room.postMessage("[ [FireAlarm-Swift](\(githubLink)) ] " +
			"[tag:\(post.tags.first ?? "tagless")] Potentially bad question: [\(post.title)](//stackoverflow.com/q/\(post.id)) " +
			bot.room.notificationString(tags: post.tags)
		)
	}
	
	func webSocketMessageText(_ text: String) {
		do {
			guard let data = text.data(using: .utf8) else {
				throw QuestionProcessingError.textNotUTF8(text: text)
			}
			webSocketMessageData(data)
		} catch {
			handleError(error, "while processing an active question")
		}
	}
	
	func webSocketMessageData(_ data: Data) {
		let string = String(data: data, encoding: .utf8) ?? "<not UTF-8: \(data.base64EncodedString())>"
		do {
			
			do {
				guard let json = try JSONSerialization.jsonObject(with: data, options: []) as? [String:String] else {
					throw QuestionProcessingError.jsonNotDictionary(json: string)
				}
				
				guard let dataObject = json["data"]?.data(using: .utf8) else {
					throw QuestionProcessingError.noDataObject(json: string)
				}
{'fraction_non_alphanumeric': 0.12112796411023286, 'fraction_numerical': 0.008117923520615253, 'mean_word_length': 3.598722986247544, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '475201', 'n_tokens_mistral': 3597, 'n_tokens_neox': 2991, 'n_words': 782}