<commit_before>import falcon
import json
import leveldb
import zmq
import threading
from voluptuous import Schema, Required, All, Length, MultipleInvalid, Extra, Range
import uuid
import logging
import asyncio
import aiodns
import python_hosts

kind_key = b"_kinds"
self_key = b"_self"
stores_key = b"_stores"

log = logging.getLogger(__name__)

JSONEncoder_olddefault = json.JSONEncoder.default
def JSONEncoder_newdefault(self, o):
	if isinstance(o, uuid.UUID): return str(o)
	return JSONEncoder_olddefault(self, o)
json.JSONEncoder.default = JSONEncoder_newdefault

def UUID(value):
	return uuid.UUID(value)

class Client:
	def __init__(self, zmq, host, port):
		self.zmq = zmq
		self.host = host
		self.port = port
		self.stores = {}

	def __del__(self):
		self.zmq.close()

class JSONResource:
	def get_key(self, key):
		return self.db.Get(key).decode("utf-8")

	def get_json_key(self, key):
		return json.loads(self.get_key(key))

	def put_key(self, key, value):
		self.db.Put(key, json.dumps(value).encode("utf-8"))

	def check_req(self, req):
		try:
			data = json.loads(req.stream.read().decode("utf-8"))
			if not isinstance(data, dict):
				raise falcon.HTTPBadRequest("Invalid JSON", "Request was not a JSON dictionary")
			self.schema(data)
			return data
		except ValueError:
			raise falcon.HTTPBadRequest("Invalid JSON", "Request was not valid JSON")
		except MultipleInvalid as e:
			raise falcon.HTTPBadRequest("Invalid keys", str(e))

	def check_noargs(self, req):
		if len(req.stream.read()) > 0:
			raise falcon.HTTPBadRequest("accepts no arguments", "Gave a body to a method that doesn't accept one")


class ClientResource(JSONResource):
	def __init__(self, context, poller, clients):
		self.clients = clients
		self.context = context
		self.poller = poller

	def on_get(self, req, resp):
		self.check_noargs(req)
		resp.body = json.dumps(dict([(k, self.clients[k].stores) for k in self.clients.keys()]))

	schema = Schema({
		Required("host"): All(str, Length(min=1)),
		Required("port"): All(int, Range(min=1))
	})

	def on_put(self, req, resp):
		data = self.check_req(req)
		conn = self.context.socket(zmq.PAIR)
		conn.linger = 0
		host, port = data["host"], data["port"]
		ip = None
		if python_hosts.utils.is_ipv4(host):
			ip = host
		else:
			hosts = python_hosts.hosts.Hosts()
			for x in hosts.entries:
				if x.entry_type == "comment":
					continue
				if host in x.names:
					ip = x.address
					break
		if ip == None:
			res = err = None
			loop = asyncio.get_event_loop()
			resolver = aiodns.DNSResolver(loop=loop)
			try:
				f = resolver.query(host, 'A')
				result = loop.run_until_complete(f)
				raise Exception(result)
			except UnicodeError as e: # hostname too long for IDNA
				raise falcon.HTTPInvalidParam("Too long hostname", "host")
			except aiodns.error.DNSError:
				raise falcon.HTTPInvalidParam("DNS can't find host", "host")
		conn.connect("tcp://{host}:{port}".format(host = ip, port = port))
		self.clients["%s:%d" %(host,port)] = Client(conn, host, port)
		resp.status = falcon.HTTP_CREATED

class StoreResource(JSONResource):
	def __init__(self, db, clients):
		self.db = db<commit_msg>If we actually find a good host result, return that!<commit_after>import falcon
import json
import leveldb
import zmq
import threading
from voluptuous import Schema, Required, All, Length, MultipleInvalid, Extra, Range
import uuid
import logging
import asyncio
import aiodns
import python_hosts

kind_key = b"_kinds"
self_key = b"_self"
stores_key = b"_stores"

log = logging.getLogger(__name__)

JSONEncoder_olddefault = json.JSONEncoder.default
def JSONEncoder_newdefault(self, o):
	if isinstance(o, uuid.UUID): return str(o)
	return JSONEncoder_olddefault(self, o)
json.JSONEncoder.default = JSONEncoder_newdefault

def UUID(value):
	return uuid.UUID(value)

class Client:
	def __init__(self, zmq, host, port):
		self.zmq = zmq
		self.host = host
		self.port = port
		self.stores = {}

	def __del__(self):
		self.zmq.close()

class JSONResource:
	def get_key(self, key):
		return self.db.Get(key).decode("utf-8")

	def get_json_key(self, key):
		return json.loads(self.get_key(key))

	def put_key(self, key, value):
		self.db.Put(key, json.dumps(value).encode("utf-8"))

	def check_req(self, req):
		try:
			data = json.loads(req.stream.read().decode("utf-8"))
			if not isinstance(data, dict):
				raise falcon.HTTPBadRequest("Invalid JSON", "Request was not a JSON dictionary")
			self.schema(data)
			return data
		except ValueError:
			raise falcon.HTTPBadRequest("Invalid JSON", "Request was not valid JSON")
		except MultipleInvalid as e:
			raise falcon.HTTPBadRequest("Invalid keys", str(e))

	def check_noargs(self, req):
		if len(req.stream.read()) > 0:
			raise falcon.HTTPBadRequest("accepts no arguments", "Gave a body to a method that doesn't accept one")


class ClientResource(JSONResource):
	def __init__(self, context, poller, clients):
		self.clients = clients
		self.context = context
		self.poller = poller

	def on_get(self, req, resp):
		self.check_noargs(req)
		resp.body = json.dumps(dict([(k, self.clients[k].stores) for k in self.clients.keys()]))

	schema = Schema({
		Required("host"): All(str, Length(min=1)),
		Required("port"): All(int, Range(min=1))
	})

	def on_put(self, req, resp):
		data = self.check_req(req)
		conn = self.context.socket(zmq.PAIR)
		conn.linger = 0
		host, port = data["host"], data["port"]
		ip = None
		if python_hosts.utils.is_ipv4(host):
			ip = host
		else:
			hosts = python_hosts.hosts.Hosts()
			for x in hosts.entries:
				if x.entry_type == "comment":
					continue
				if host in x.names:
					ip = x.address
					break
		if ip == None:
			res = err = None
			loop = asyncio.get_event_loop()
			resolver = aiodns.DNSResolver(loop=loop)
			try:
				f = resolver.query(host, 'A')
				result = loop.run_until_complete(f)
				ip = result[0].host
			except UnicodeError as e: # hostname too long for IDNA
				raise falcon.HTTPInvalidParam("Too long hostname", "host")
			except aiodns.error.DNSError:
				raise falcon.HTTPInvalidParam("DNS can't find host", "host")
		conn.connect("tcp://{host}:{port}".format(host = ip, port = port))
		self.clients["%s:%d" %(host,port)] = Client(conn, host, port)
		resp.status = falcon.HTTP_CREATED

class StoreResource(JSONResource):
	def __init__(self, db, clients):
		self.db = db
{'fraction_non_alphanumeric': 0.13078502606223347, 'fraction_numerical': 0.002685199810456484, 'mean_word_length': 4.338954468802698, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6618444', 'n_tokens_mistral': 2430, 'n_tokens_neox': 2201, 'n_words': 572}