<commit_before>import pymongo
import json
import traceback
import os
import datetime
import hashlib
import time
client = pymongo.MongoClient()

try:
	dbConf = json.load(open("./model/db.json","r"))
	nicknames = json.load(open("./model/nicknames.json","r"))
except:
	try:
		dbConf = json.load(open("./db.json","r"))
	except:
		dbConf = {'dbname':'voteview'}
		nicknames = []

db = client[dbConf["dbname"]]

def getIP(request):
	return request.environ.get('HTTP_X_FORWARDED_FOR') or request.environ.get('REMOTE_ADDR')

def getUserAgent(request):
	return request.get_header("User-Agent")

def generateSessionID(request):
	ipAddress = getIP(request)
	userAgent = getUserAgent(request)
	staticSalt = authData = json.load(open("./model/auth.json","r"))["quotaLogSalt"]
	dynamicSalt = str(datetime.datetime.now().timetuple().tm_yday)
	finalHash = str(hashlib.sha256(ipAddress+"/"+userAgent+"/"+staticSalt+"/"+dynamicSalt).hexdigest())[0:16]
	return finalHash

def logSearch(request, search):
	doc = {"session": generateSessionID(request), "time": int(time.time())}
	doc.update(search)

	db.search_log.insert(doc)
<commit_msg>Build-out on logging/quota to include logging and quota. Untested so far.<commit_after>import pymongo
import json
import traceback
import os
import datetime
import hashlib
import time
client = pymongo.MongoClient()

try:
	dbConf = json.load(open("./model/db.json","r"))
	nicknames = json.load(open("./model/nicknames.json","r"))
except:
	try:
		dbConf = json.load(open("./db.json","r"))
	except:
		dbConf = {'dbname':'voteview'}
		nicknames = []

db = client[dbConf["dbname"]]
authFile = json.load(open("./model/auth.json","r"))

# Wrapper for getting user IP
def getIP(request):
	return request.environ.get('HTTP_X_FORWARDED_FOR') or request.environ.get('REMOTE_ADDR')

# Wrapper for getting user agent
def getUserAgent(request):
	return request.get_header("User-Agent")

# Wrapper to load the salt and generate the user session ID
def generateSessionID(request):
	ipAddress = getIP(request)
	userAgent = getUserAgent(request)
	staticSalt = authFile["quotaLogSalt"]
	dynamicSalt = str(datetime.datetime.now().timetuple().tm_yday)
	finalHash = str(hashlib.sha256(ipAddress+"/"+userAgent+"/"+staticSalt+"/"+dynamicSalt).hexdigest())[0:16]
	return finalHash

# Given a search query and number of results, log the search
def logSearch(request, search):
	doc = {"session": generateSessionID(request), "time": int(time.time())}
	doc.update(search)

	db.search_log.insert(doc)

# Check user's quota status
def checkQuota(request):
	session = generateSessionID(request)

	r = db.api_quota.findOne({"session": session}, {"score": 1, "_id": 0})
	if r["score"] <= authFile["quotaLimit"]:
		return {"status": 0}
	else:
		whitelisted = 0
		blacklisted = 0
		ipAddress = getIP(request)
		for ip in authFile["blacklistIPs"]:
			if ipAddress.startswith(ip):
				blacklisted=1
				break

		if blacklisted:
			return {"status": 1, "error_message": "You have been blocked from completing this request due to abuse. For more information, go to https://voteview.com/quota"}

		for ip in authFile["whitelistIPs"]:
			if ipAddress.startswith(ip):
				whitelisted=1
				break
		if not whitelisted:
			return {"status": 1, "error_message": "You have been blocked from completing this request because you are over quota. To prevent abuse of our systems, users are limited from excessive numbers of search queries. For more information, go to https://voteview.com/quota"}
		else:
			return {"status": 0}

# Add a cost to user's quota
def addQuota(request, score):
	session = generateSessionID(request)
	r = db.api_quota.findOne({"session": session}, {"score": 1, "_id": 0})
	if r:
		db.api_quota.update({"session": session}, {"$set": {"score": r["score"]+score}})
	else:
		db.api_quota.insert({"session": session, "score": score})

{'fraction_non_alphanumeric': 0.15145173947161913, 'fraction_numerical': 0.006277792309704421, 'mean_word_length': 5.514480408858603, 'pattern_counts': {'":': 21, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '719325', 'n_tokens_mistral': 1360, 'n_tokens_neox': 1278, 'n_words': 317}