<commit_before>
#
# dht.py
#
# Distributed under the MIT/X11 software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
#
# Messages:
#	ping -> pong
#	store -> ok, err
#	find-nodes -> nodes, err
#	find-value -> nodes, data, err
#

import socket
import datetime
import asyncore
import hashlib
import random

import codec_pb2
import LRU

debugdht = True

def verbose_sendmsg(command):
	if debugdht:
		return True
	return True

def valid_key_len(n):
	if n == 20 or n == 32 or n == 64:
		return True
	return False

def matching_bits(v1, v2):
	for i in xrange(v1.bit_length):
		if ((v1 & (1 << i)) != (v2 & (1 << i))):
			return i
	return v1.bit_length

def bn2bin(v):
	s = bytearray()
	i = bn_bytes(v)
	while i > 0:
		s.append((v >> ((i-1) * 8)) & 0xff)
		i -= 1
	return s

def bin2bn(s):
	l = 0L
	for ch in s:
		l = (l << 8) | ch
	return l

def hash_node_id(node_id):
	node_id_bin = bn2bin(node_id)
	hash = hashlib.sha256(node_id_bin).digest()
	return hash

class NodeDist(object):
	def __init__(self, node_id=0L, distance=0L):
		self.node_id = node_id
		self.distance = distance

class DHTTask(object):
	def __init__(self):
		self.addr = None
		self.msg_start = None
		self.time_start = datetime.datetime.utcnow()
		self.timeout = None
		self.timeout_actor = None
		self.timeout_args = []
		self.task_id = None

class DHTTaskManager(object):
	def __init__(self):
		self.tasks = {}

	def new_task_id(self):
		while True:
			rand_id = random.getrandbits(64)
			if rand_id not in self.tasks:
				return rand_id

	def new_task(self):
		task = DHTTask()
		task.task_id = self.new_task_id()

		self.tasks[task.task_id] = task

		return task

	def delete(self, task_id, addr):
		if task_id not in self.tasks:
			return None

		task = self.tasks[task_id]

		if task.addr != addr:
			return None

		del self.tasks[task_id]

		return task

class DHTNode(object):
	def __init__(self, node_id=0L, ip=None, port=0, flags=0):
		self.node_id = node_id
		self.ip = ip
		self.port = port
		self.flags = flags

		self.first_seen = datetime.datetime.utcnow()
		self.last_seen = datetime.datetime(1980, 1, 1)
		self.rtt = None
		self.bucket_idx = -1
		self.active = False
		self.demoted = 0

class DHTBucket(object):
	def __init__(self):
		self.active_max = 20
		self.demote_max = 3
		self.active = {}
		self.candidates = []

	def full(self):
		return (len(self.active) >= self.active_max)

	def oldest_active(self):
		oldest = datetime.datetime.utcnow()
		old_node = None
		for node in self.active.itervalues():
			if node.last_seen < oldest:
				oldest = node.last_seen
				old_node = node

		return old_node

	def newest_candidate(self):
		tmp_seen = datetime.datetime(1979, 1, 1)
		new_node = None
		for node in self.candidates:
			if node.last_seen > tmp_seen:
				tmp_seen = node.last_seen
				new_node = node

		return new_node

	def refill(self):
		while not bucket.full() and len(self.candidates) > 0:
			new_node = bucket.newest_candidate()
			self.promote_node(bucket, new_node)

	def demote_node(self, node):
		if (not node.active or
		    node.node_id == 0):
			return False

		# demote node to candidate: active -> candidate transition
		del self.active[node.node_id]
		node.active = False
		node.demoted += 1<commit_msg>DHT: fix obvious brokenness in DHTBucket.refill()<commit_after>
#
# dht.py
#
# Distributed under the MIT/X11 software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
#
# Messages:
#	ping -> pong
#	store -> ok, err
#	find-nodes -> nodes, err
#	find-value -> nodes, data, err
#

import socket
import datetime
import asyncore
import hashlib
import random

import codec_pb2
import LRU

debugdht = True

def verbose_sendmsg(command):
	if debugdht:
		return True
	return True

def valid_key_len(n):
	if n == 20 or n == 32 or n == 64:
		return True
	return False

def matching_bits(v1, v2):
	for i in xrange(v1.bit_length):
		if ((v1 & (1 << i)) != (v2 & (1 << i))):
			return i
	return v1.bit_length

def bn2bin(v):
	s = bytearray()
	i = bn_bytes(v)
	while i > 0:
		s.append((v >> ((i-1) * 8)) & 0xff)
		i -= 1
	return s

def bin2bn(s):
	l = 0L
	for ch in s:
		l = (l << 8) | ch
	return l

def hash_node_id(node_id):
	node_id_bin = bn2bin(node_id)
	hash = hashlib.sha256(node_id_bin).digest()
	return hash

class NodeDist(object):
	def __init__(self, node_id=0L, distance=0L):
		self.node_id = node_id
		self.distance = distance

class DHTTask(object):
	def __init__(self):
		self.addr = None
		self.msg_start = None
		self.time_start = datetime.datetime.utcnow()
		self.timeout = None
		self.timeout_actor = None
		self.timeout_args = []
		self.task_id = None

class DHTTaskManager(object):
	def __init__(self):
		self.tasks = {}

	def new_task_id(self):
		while True:
			rand_id = random.getrandbits(64)
			if rand_id not in self.tasks:
				return rand_id

	def new_task(self):
		task = DHTTask()
		task.task_id = self.new_task_id()

		self.tasks[task.task_id] = task

		return task

	def delete(self, task_id, addr):
		if task_id not in self.tasks:
			return None

		task = self.tasks[task_id]

		if task.addr != addr:
			return None

		del self.tasks[task_id]

		return task

class DHTNode(object):
	def __init__(self, node_id=0L, ip=None, port=0, flags=0):
		self.node_id = node_id
		self.ip = ip
		self.port = port
		self.flags = flags

		self.first_seen = datetime.datetime.utcnow()
		self.last_seen = datetime.datetime(1980, 1, 1)
		self.rtt = None
		self.bucket_idx = -1
		self.active = False
		self.demoted = 0

class DHTBucket(object):
	def __init__(self):
		self.active_max = 20
		self.demote_max = 3
		self.active = {}
		self.candidates = []

	def full(self):
		return (len(self.active) >= self.active_max)

	def oldest_active(self):
		oldest = datetime.datetime.utcnow()
		old_node = None
		for node in self.active.itervalues():
			if node.last_seen < oldest:
				oldest = node.last_seen
				old_node = node

		return old_node

	def newest_candidate(self):
		tmp_seen = datetime.datetime(1979, 1, 1)
		new_node = None
		for node in self.candidates:
			if node.last_seen > tmp_seen:
				tmp_seen = node.last_seen
				new_node = node

		return new_node

	def refill(self):
		while not self.full() and len(self.candidates) > 0:
			new_node = self.newest_candidate()
			self.promote_node(new_node)

	def demote_node(self, node):
		if (not node.active or
		    node.node_id == 0):
			return False

		# demote node to candidate: active -> candidate transition
		del self.active[node.node_id]
		node.active = False
		node.demoted += 1
{'fraction_non_alphanumeric': 0.11824839792493134, 'fraction_numerical': 0.01739395788831248, 'mean_word_length': 3.4958847736625516, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3963648', 'n_tokens_mistral': 2789, 'n_tokens_neox': 2488, 'n_words': 647}