<commit_before># estimates clade frequencies using SMC

import time
from io_util import *
from tree_util import *
from date_util import *
import numpy as np

START_DATE = "2011-01-01"
DAY_STEP = 10

def observations_for_clade(node, dates):
	"""Takes a node and a list of dates and returns an observation list"""
	node_dates = get_dates(node)
	obs = []
	for d in dates:
		if d in node_dates:
			obs.append(1)
			node_dates.remove(d)
		else:
			obs.append(0)
	return obs

def obs_likelihood(obs, freq):
	"""Bernoulli probability of observing binary presence / absence given circulating frequency"""
	"""obs is 0 or 1 binary indicator for absence / presence"""
	return (1-freq)**(1-obs) * freq**obs

class Particle:
	"""An individual SMC particle.  Stores value and weight."""
	min_value = 0.00001
	max_value = 0.99999

	def __init__(self, obs):
		self.value = np.random.random_sample()	# between [0,1]
		self.reweight(obs)

	def __str__(self):
		return "{value: " + str(self.value) + ", weight: " + str(self.weight) + "}"

	def simulate_step(self, dt, sigma):
		"""Single time step forward"""
		w = np.random.normal()
		x0 = self.value
		x1 = x0 + x0 * (1-x0) * sigma * np.sqrt(dt) * w
		if x1 < self.min_value:
			x1 = self.min_value
		if x1 > self.max_value:
			x1 = self.max_value
		self.value = x1

	def simulate(self, t, dt, sigma):
		"""Simulate forward t years with timesteps dt"""
		"""Implements Euler-Maruyama method"""
		steps = int(np.ceil(t / dt))
		if steps > 0:
			dt = t / steps
			sqrtdt_sigma = np.sqrt(dt) * sigma
			for i in range(steps):
				x0 = self.value
				x1 = x0 + x0 * (1-x0) * sqrtdt_sigma * np.random.normal()
				if x1 < self.min_value:
					x1 = self.min_value
				if x1 > self.max_value:
					x1 = self.max_value
				self.value = x1

	def reweight(self, obs):
		self.weight = obs_likelihood(obs, self.value) + 0.000000001

	def likelihood(self, obs):
		return obs_likelihood(obs, self.value)

class Filter:
	"""An SMC particle filter, comprising n particles."""
	"""Takes a list of dates and a list of (0,1) observations."""
	"""Time in encoded in continuous units of years."""
	particle_count = 500
	timestep = 0.01
	sigma = 5
<commit_msg>inherited from object in classes. this provides a bunch of useful methods<commit_after># estimates clade frequencies using SMC

import time
from io_util import *
from tree_util import *
from date_util import *
import numpy as np

START_DATE = "2011-01-01"
DAY_STEP = 10

def observations_for_clade(node, dates):
	"""Takes a node and a list of dates and returns an observation list"""
	node_dates = get_dates(node)
	obs = []
	for d in dates:
		if d in node_dates:
			obs.append(1)
			node_dates.remove(d)
		else:
			obs.append(0)
	return obs

def obs_likelihood(obs, freq):
	"""Bernoulli probability of observing binary presence / absence given circulating frequency"""
	"""obs is 0 or 1 binary indicator for absence / presence"""
	return (1-freq)**(1-obs) * freq**obs

class Particle(object):
	"""An individual SMC particle.  Stores value and weight."""
	min_value = 0.00001
	max_value = 0.99999

	def __init__(self, obs):
		self.value = np.random.random_sample()	# between [0,1]
		self.reweight(obs)

	def __str__(self):
		return "{value: " + str(self.value) + ", weight: " + str(self.weight) + "}"

	def simulate_step(self, dt, sigma):
		"""Single time step forward"""
		w = np.random.normal()
		x0 = self.value
		x1 = x0 + x0 * (1-x0) * sigma * np.sqrt(dt) * w
		if x1 < self.min_value:
			x1 = self.min_value
		if x1 > self.max_value:
			x1 = self.max_value
		self.value = x1

	def simulate(self, t, dt, sigma):
		"""Simulate forward t years with timesteps dt"""
		"""Implements Euler-Maruyama method"""
		steps = int(np.ceil(t / dt))
		if steps > 0:
			dt = t / steps
			sqrtdt_sigma = np.sqrt(dt) * sigma
			for i in range(steps):
				x0 = self.value
				x1 = x0 + x0 * (1-x0) * sqrtdt_sigma * np.random.normal()
				if x1 < self.min_value:
					x1 = self.min_value
				if x1 > self.max_value:
					x1 = self.max_value
				self.value = x1

	def reweight(self, obs):
		self.weight = obs_likelihood(obs, self.value) + 0.000000001

	def likelihood(self, obs):
		return obs_likelihood(obs, self.value)

class Filter(object):
	"""An SMC particle filter, comprising n particles."""
	"""Takes a list of dates and a list of (0,1) observations."""
	"""Time in encoded in continuous units of years."""
	particle_count = 500
	timestep = 0.01
	sigma = 5

{'fraction_non_alphanumeric': 0.12782805429864252, 'fraction_numerical': 0.03257918552036199, 'mean_word_length': 3.562435500515996, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4091988', 'n_tokens_mistral': 1788, 'n_tokens_neox': 1553, 'n_words': 495}