<commit_before>use std::cell::RefCell;
use std::cmp::Ordering;
use rand::{ChaChaRng, SeedableRng};
use {DeterministicRandomId, PredictorId, TimeId, RowId, FieldId, SiphashIdGenerator,
     IterationType, Basics, ExtendedTime, GenericExtendedTime};

// https://github.com/rust-lang/rfcs/issues/1485
pub trait Filter<T> {
  fn filter<P: FnOnce(&T) -> bool>(self, predicate: P) -> Self;
}
impl<T> Filter<T> for Option<T> {
  fn filter<P: FnOnce(&T) -> bool>(self, predicate: P) -> Self {
    self.and_then(|x| { if predicate(&x) { Some(x) } else { None } })
  }
}

type EventRng = ChaChaRng;
fn generator_for_event(id: TimeId) -> EventRng {
  EventRng::from_seed(&[(id.data()[0] >> 32) as u32,
                        (id.data()[0] & 0xffffffff) as u32,
                        (id.data()[1] >> 32) as u32,
                        (id.data()[1] & 0xffffffff) as u32])
}

#[macro_export]
macro_rules! time_steward_common_dynamic_callback_structs {

($M: ident, $PA: ident, $DynamicEventFn: ident, $DynamicPredictorFn: ident, $DynamicPredictor: ident, $StandardSettings: ident) => {

mod __time_steward_make_dynamic_callbacks_impl {

use $crate::{Basics, Event, Predictor, RowId, ColumnId, PredictorId, StewardRc, PredictorList, predictor_list};
use std::collections::HashMap;
use std::marker::PhantomData;

pub struct DynamicEventFn <E: Event> (E);
pub struct DynamicPredictorFn <P: Predictor> (PhantomData <P>);

impl<E: Event> DynamicEventFn <E> {
  pub fn new (event: E)->Self {DynamicEventFn (event)}
}
impl<P: Predictor> DynamicPredictorFn <P> {
  pub fn new ()->Self {DynamicPredictorFn (PhantomData)}
}

impl<'a, 'b, E: Event> Fn <(& 'a mut super::$M <'b, E::Basics>,)> for DynamicEventFn <E> {
  extern "rust-call" fn call (&self, arguments: (& 'a mut super::$M <'b, E::Basics>,)) {
    self.0.call (arguments.0)
  }
}
impl<'a, 'b, P: Predictor> Fn <(& 'a mut super::$PA <'b, P::Basics>, RowId)> for DynamicPredictorFn <P> {
  extern "rust-call" fn call (&self, arguments: (& 'a mut super::$PA <'b, P::Basics>, RowId)) {
    P::call (arguments.0, arguments.1)
  }
}

impl<'a, 'b, E: Event> FnMut <(& 'a mut super::$M <'b, E::Basics>,)> for DynamicEventFn <E> {
  extern "rust-call" fn call_mut (&mut self, arguments: (& 'a mut super::$M <'b, E::Basics>,)) {
    self.call (arguments)
  }
}
impl<'a, 'b, E: Event> FnOnce <(& 'a mut super::$M <'b, E::Basics>,)> for DynamicEventFn <E> {
  type Output = ();
  extern "rust-call" fn call_once (self, arguments: (& 'a mut super::$M <'b, E::Basics>,)) {
    self.call (arguments)
  }
}


impl<'a, 'b, P: Predictor> FnMut <(& 'a mut super::$PA <'b, P::Basics>, RowId)> for DynamicPredictorFn <P> {
  extern "rust-call" fn call_mut (&mut self, arguments: (& 'a mut super::$PA <'b, P::Basics>, RowId)) {
    self.call (arguments)
  }
}
impl<'a, 'b, P: Predictor> FnOnce <(& 'a mut super::$PA <'b, P::Basics>, RowId)> for DynamicPredictorFn <P> {
  type Output = ();
  extern "rust-call" fn call_once (self, arguments: (& 'a mut super::$PA <'b, P::Basics>, RowId)) {
    self.call (arguments)
  }
}

//trait DynamicPredictorTrait: for <'a, 'b> Fn(& 'a mut super::$PA <'b, B>, RowId) {
//  
//}

// #[derive (Clone)]
pub struct DynamicPredictor <B: Basics> {
  pub predictor_id: PredictorId,
  pub column_id: ColumnId,
  pub function: StewardRc <for <'a, 'b> Fn(& 'a mut super::$PA <'b, B>, RowId)>,
  _marker: PhantomData <B>,
}
// explicitly implement Clone to work around [a compiler weakness](https://github.com/rust-lang/rust/issues/26925).
impl<B: Basics> Clone for DynamicPredictor<B> {
  fn clone(&self) -> Self {
    DynamicPredictor {
      predictor_id: self.predictor_id,
      column_id: self.column_id,
      function: self.function.clone(),
      _marker: PhantomData,
    }
  }
}

#[derive (Clone)]
pub struct StandardSettings <B: Basics> {
  pub predictors_by_column: HashMap<ColumnId, Vec<DynamicPredictor <B>>>,
  pub predictors_by_id: HashMap<PredictorId, DynamicPredictor <B>>,
}
impl<B: Basics> predictor_list::User <B> for StandardSettings <B> {
  fn apply <P: Predictor <Basics = B>> (&mut self) {
    let predictor = DynamicPredictor {
      predictor_id: P::predictor_id(),
      column_id: P::column_id(),
      function: StewardRc::new (DynamicPredictorFn::<P>::new ()),
      _marker: PhantomData,
    };
    self.predictors_by_id.insert(predictor.predictor_id, predictor.clone());
    self.predictors_by_column.entry(predictor.column_id).or_insert(Vec::new()).push(predictor);
  }
}
impl<B: Basics> StandardSettings <B> {
  pub fn new()->Self {
    let mut result = StandardSettings {
      predictors_by_id: HashMap::new(),
      predictors_by_column: HashMap::new(),
    };
    B::IncludedTypes::apply (&mut result);
    result
  }
}

}

#[allow (unused_imports)]
use self::__time_steward_make_dynamic_callbacks_impl::DynamicEventFn as $DynamicEventFn;
#[allow (unused_imports)]
use self::__time_steward_make_dynamic_callbacks_impl::DynamicPredictor as $DynamicPredictor;
#[allow (unused_imports)]
use self::__time_steward_make_dynamic_callbacks_impl::DynamicPredictorFn as $DynamicPredictorFn;
pub use self::__time_steward_make_dynamic_callbacks_impl::StandardSettings as $StandardSettings;

}}

pub struct GenericPredictorAccessor<B: Basics, E> {
  pub soonest_prediction: Option<(B::Time, E)>,
  pub dependencies: RefCell<(Vec<FieldId>, SiphashIdGenerator)>,
}
impl<B: Basics, E> GenericPredictorAccessor<B, E> {
  pub fn new() -> Self {
    GenericPredictorAccessor {
      soonest_prediction: None,
      dependencies: RefCell::new((Vec::new(), SiphashIdGenerator::new())),
    }
  }
}
#[macro_export]
macro_rules! time_steward_common_accessor_methods_for_predictor_accessor {
  ($B: ty, $get: ident) => {
    fn generic_data_and_extended_last_change(&self,
                                             id: FieldId)
                                             -> Option<(&FieldRc, &ExtendedTime<$B>)> {
      let dependencies: &mut (Vec<FieldId>, SiphashIdGenerator) = &mut*self.generic.dependencies.borrow_mut();
      dependencies.0.push(id);
      self.$get(id).map(|p| {
        {
          use bincode;
          use serde::Serialize;
          let mut serializer = bincode::serde::Serializer::new (&mut dependencies.1);
          p.1.id.serialize (&mut serializer).unwrap();
        }
        p
      })
    }
  }
}
#[macro_export]
macro_rules! time_steward_common_predictor_accessor_methods_for_predictor_accessor {
  ($B: ty, $DynamicEventFn: ident) => {
    fn predict_at_time <E: $crate::Event <Basics = $B>> (&mut self, time: <$B as $crate::Basics>::Time, event: E) {
      if time < *self.unsafe_now() {
        return;
      }
      if let Some((ref old_time, _)) = self.generic.soonest_prediction {
        if old_time <= &time {
          return;
        }
      }
      self.generic.soonest_prediction = Some((time, StewardRc::new ($DynamicEventFn ::new (event))));
    }
  }
}


pub struct GenericMutator<B: Basics> {
  pub now: ExtendedTime<B>,
  pub generator: ChaChaRng,
}
impl<B: Basics> GenericMutator<B> {
  pub fn new(now: ExtendedTime<B>) -> Self {
    let generator = generator_for_event(now.id);
    GenericMutator {
      now: now,
      generator: generator,
    }
  }
}
#[macro_export]
macro_rules! time_steward_common_accessor_methods_for_mutator {
  ($B: ty) => {
    fn unsafe_now(& self)->& <$B as $crate::Basics>::Time {& self.generic.now.base}
  }
}
#[macro_export]
macro_rules! time_steward_common_mutator_methods_for_mutator {
  ($B: ty) => {
    fn extended_now(& self)->& ExtendedTime <$B> {& self.generic.now}
    fn gen_id(&mut self) -> RowId {RowId::from_rng (&mut self.generic.generator)}
  }
}
#[macro_export]
macro_rules! time_steward_common_rng_methods_for_mutator {
  ($B: ty) => {
    fn next_u32(&mut self) -> u32 {self.generic.generator.next_u32()}

    fn next_f32(&mut self) -> f32 {
      assert!(<$B as $crate::Basics>::allow_floats_unsafe(), "Using floating point numbers in TimeSteward events is forbidden by default because it is nondeterministic across platforms. If you know you don't need to be consistent between different computers, or you otherwise know EXACTLY what you're doing, you may add fn allow_floats_unsafe() {true} to your Basics.");
      self.generic.generator.next_f32()
    }
    fn next_f64(&mut self) -> f64 {
      assert!(<$B as $crate::Basics>::allow_floats_unsafe(), "Using floating point numbers in TimeSteward events is forbidden by default because it is nondeterministic across platforms. If you know you don't need to be consistent between different computers, or you otherwise know EXACTLY what you're doing, you may add fn allow_floats_unsafe() {true} to your Basics.");
      self.generic.generator.next_f64()
    }
  }
}





pub fn extended_time_of_fiat_event<BaseTime: Ord>(time: BaseTime,
                                                  id: TimeId)
                                                  -> GenericExtendedTime<BaseTime> {
  GenericExtendedTime {
    base: time,
    iteration: 0,
    id: id.for_fiat_event_internal(),
  }
}

pub fn time_id_for_predicted_event(predictor_id: PredictorId,
                                   row_id: RowId,
                                   iteration: IterationType,
                                   dependencies_hash: DeterministicRandomId)
                                   -> TimeId {
  TimeId::new(&(predictor_id, row_id, iteration, dependencies_hash))
}
pub fn next_extended_time_of_predicted_event<BaseTime: Ord>
  (predictor_id: PredictorId,
   row_id: RowId,
   dependencies_hash: DeterministicRandomId,
   event_base_time: BaseTime,
   from: &GenericExtendedTime<BaseTime>)
   -> Option<GenericExtendedTime<BaseTime>> {
  let (iteration, id) = match event_base_time.cmp(&from.base) {
    Ordering::Less => return None, // short-circuit
    Ordering::Greater => {
      (0, time_id_for_predicted_event(predictor_id, row_id, 0, dependencies_hash))
    }
    Ordering::Equal => {
      let id = time_id_for_predicted_event(predictor_id, row_id, from.iteration, dependencies_hash);
      if id > from.id {
        (from.iteration, id)
      } else {
        if from.iteration == IterationType::max_value() {
          panic!("Too many iterations at the same base time; probably an infinite loop")
        }
        (from.iteration + 1,
         time_id_for_predicted_event(predictor_id, row_id, from.iteration + 1, dependencies_hash))
      }
    }
  };
  Some(GenericExtendedTime {
    base: event_base_time,
    iteration: iteration,
    id: id,
  })
}
<commit_msg>Generalized DynamicEventFn etc. – they didn't actually need to be inside a macro<commit_after>use std::cell::RefCell;
use std::cmp::Ordering;
use rand::{ChaChaRng, SeedableRng};
use {DeterministicRandomId, PredictorId, TimeId, RowId, FieldId, SiphashIdGenerator,
     IterationType, Basics, ExtendedTime, GenericExtendedTime, Predictor, Event, PredictorAccessor, Mutator};
use std::marker::PhantomData;

// https://github.com/rust-lang/rfcs/issues/1485
pub trait Filter<T> {
  fn filter<P: FnOnce(&T) -> bool>(self, predicate: P) -> Self;
}
impl<T> Filter<T> for Option<T> {
  fn filter<P: FnOnce(&T) -> bool>(self, predicate: P) -> Self {
    self.and_then(|x| { if predicate(&x) { Some(x) } else { None } })
  }
}

type EventRng = ChaChaRng;
fn generator_for_event(id: TimeId) -> EventRng {
  EventRng::from_seed(&[(id.data()[0] >> 32) as u32,
                        (id.data()[0] & 0xffffffff) as u32,
                        (id.data()[1] >> 32) as u32,
                        (id.data()[1] & 0xffffffff) as u32])
}


pub struct DynamicEventFn <E: Event> (E);
pub struct DynamicPredictorFn <P: Predictor> (PhantomData <P>);

impl<E: Event> DynamicEventFn <E> {
  pub fn new (event: E)->Self {DynamicEventFn (event)}
}
impl<P: Predictor> DynamicPredictorFn <P> {
  pub fn new ()->Self {DynamicPredictorFn (PhantomData)}
}

impl<'a, 'b, E: Event, M: Mutator <E::Basics>> Fn <(& 'a mut M,)> for DynamicEventFn <E> {
  extern "rust-call" fn call (&self, arguments: (& 'a mut M,)) {
    self.0.call (arguments.0)
  }
}
impl<'a, 'b, P: Predictor, PA: PredictorAccessor <P::Basics>> Fn <(& 'a mut PA, RowId)> for DynamicPredictorFn <P> {
  extern "rust-call" fn call (&self, arguments: (& 'a mut PA, RowId)) {
    P::call (arguments.0, arguments.1)
  }
}

impl<'a, 'b, E: Event, M: Mutator <E::Basics>> FnMut <(& 'a mut M,)> for DynamicEventFn <E> {
  extern "rust-call" fn call_mut (&mut self, arguments: (& 'a mut M,)) {
    self.call (arguments)
  }
}
impl<'a, 'b, E: Event, M: Mutator <E::Basics>> FnOnce <(& 'a mut M,)> for DynamicEventFn <E> {
  type Output = ();
  extern "rust-call" fn call_once (self, arguments: (& 'a mut M,)) {
    self.call (arguments)
  }
}


impl<'a, 'b, P: Predictor, PA: PredictorAccessor <P::Basics>> FnMut <(& 'a mut PA,RowId)> for DynamicPredictorFn <P> {
  extern "rust-call" fn call_mut (&mut self, arguments: (& 'a mut PA, RowId)) {
    self.call (arguments)
  }
}
impl<'a, 'b, P: Predictor, PA: PredictorAccessor <P::Basics>> FnOnce <(& 'a mut PA,RowId)> for DynamicPredictorFn <P> {
  type Output = ();
  extern "rust-call" fn call_once (self, arguments: (& 'a mut PA,RowId)) {
    self.call (arguments)
  }
}

//trait DynamicPredictorTrait <PA: PredictorAccessor>: for <'a, 'b> Fn(& 'a mut PA, RowId) {
 // 
//}


#[macro_export]
macro_rules! time_steward_common_dynamic_callback_structs {

($M: ident, $PA: ident, $DynamicEventFn: ident, $DynamicPredictorFn: ident, $DynamicPredictor: ident, $StandardSettings: ident) => {

mod __time_steward_make_dynamic_callbacks_impl {

use $crate::{Basics, Event, Predictor, RowId, ColumnId, PredictorId, StewardRc, PredictorList, predictor_list};
use std::collections::HashMap;
use std::marker::PhantomData;

use ::stewards::common::*;


// #[derive (Clone)]
pub struct DynamicPredictor <B: Basics> {
  pub predictor_id: PredictorId,
  pub column_id: ColumnId,
  pub function: StewardRc <for <'a, 'b> Fn(& 'a mut super::$PA <'b, B>, RowId)>,
  _marker: PhantomData <B>,
}
// explicitly implement Clone to work around [a compiler weakness](https://github.com/rust-lang/rust/issues/26925).
impl<B: Basics> Clone for DynamicPredictor<B> {
  fn clone(&self) -> Self {
    DynamicPredictor {
      predictor_id: self.predictor_id,
      column_id: self.column_id,
      function: self.function.clone(),
      _marker: PhantomData,
    }
  }
}

#[derive (Clone)]
pub struct StandardSettings <B: Basics> {
  pub predictors_by_column: HashMap<ColumnId, Vec<DynamicPredictor <B>>>,
  pub predictors_by_id: HashMap<PredictorId, DynamicPredictor <B>>,
}
impl<B: Basics> predictor_list::User <B> for StandardSettings <B> {
  fn apply <P: Predictor <Basics = B>> (&mut self) {
    let predictor = DynamicPredictor {
      predictor_id: P::predictor_id(),
      column_id: P::column_id(),
      function: StewardRc::new (DynamicPredictorFn::<P>::new ()),
      _marker: PhantomData,
    };
    self.predictors_by_id.insert(predictor.predictor_id, predictor.clone());
    self.predictors_by_column.entry(predictor.column_id).or_insert(Vec::new()).push(predictor);
  }
}
impl<B: Basics> StandardSettings <B> {
  pub fn new()->Self {
    let mut result = StandardSettings {
      predictors_by_id: HashMap::new(),
      predictors_by_column: HashMap::new(),
    };
    B::IncludedTypes::apply (&mut result);
    result
  }
}

}

#[allow (unused_imports)]
use ::stewards::common::DynamicEventFn as $DynamicEventFn;
#[allow (unused_imports)]
use self::__time_steward_make_dynamic_callbacks_impl::DynamicPredictor as $DynamicPredictor;
#[allow (unused_imports)]
use ::stewards::common::DynamicPredictorFn as $DynamicPredictorFn;
pub use self::__time_steward_make_dynamic_callbacks_impl::StandardSettings as $StandardSettings;

}}

pub struct GenericPredictorAccessor<B: Basics, E> {
  pub soonest_prediction: Option<(B::Time, E)>,
  pub dependencies: RefCell<(Vec<FieldId>, SiphashIdGenerator)>,
}
impl<B: Basics, E> GenericPredictorAccessor<B, E> {
  pub fn new() -> Self {
    GenericPredictorAccessor {
      soonest_prediction: None,
      dependencies: RefCell::new((Vec::new(), SiphashIdGenerator::new())),
    }
  }
}
#[macro_export]
macro_rules! time_steward_common_accessor_methods_for_predictor_accessor {
  ($B: ty, $get: ident) => {
    fn generic_data_and_extended_last_change(&self,
                                             id: FieldId)
                                             -> Option<(&FieldRc, &ExtendedTime<$B>)> {
      let dependencies: &mut (Vec<FieldId>, SiphashIdGenerator) = &mut*self.generic.dependencies.borrow_mut();
      dependencies.0.push(id);
      self.$get(id).map(|p| {
        {
          use bincode;
          use serde::Serialize;
          let mut serializer = bincode::serde::Serializer::new (&mut dependencies.1);
          p.1.id.serialize (&mut serializer).unwrap();
        }
        p
      })
    }
  }
}
#[macro_export]
macro_rules! time_steward_common_predictor_accessor_methods_for_predictor_accessor {
  ($B: ty, $DynamicEventFn: ident) => {
    fn predict_at_time <E: $crate::Event <Basics = $B>> (&mut self, time: <$B as $crate::Basics>::Time, event: E) {
      if time < *self.unsafe_now() {
        return;
      }
      if let Some((ref old_time, _)) = self.generic.soonest_prediction {
        if old_time <= &time {
          return;
        }
      }
      self.generic.soonest_prediction = Some((time, StewardRc::new ($DynamicEventFn ::new (event))));
    }
  }
}


pub struct GenericMutator<B: Basics> {
  pub now: ExtendedTime<B>,
  pub generator: ChaChaRng,
}
impl<B: Basics> GenericMutator<B> {
  pub fn new(now: ExtendedTime<B>) -> Self {
    let generator = generator_for_event(now.id);
    GenericMutator {
      now: now,
      generator: generator,
    }
  }
}
#[macro_export]
macro_rules! time_steward_common_accessor_methods_for_mutator {
  ($B: ty) => {
    fn unsafe_now(& self)->& <$B as $crate::Basics>::Time {& self.generic.now.base}
  }
}
#[macro_export]
macro_rules! time_steward_common_mutator_methods_for_mutator {
  ($B: ty) => {
    fn extended_now(& self)->& ExtendedTime <$B> {& self.generic.now}
    fn gen_id(&mut self) -> RowId {RowId::from_rng (&mut self.generic.generator)}
  }
}
#[macro_export]
macro_rules! time_steward_common_rng_methods_for_mutator {
  ($B: ty) => {
    fn next_u32(&mut self) -> u32 {self.generic.generator.next_u32()}

    fn next_f32(&mut self) -> f32 {
      assert!(<$B as $crate::Basics>::allow_floats_unsafe(), "Using floating point numbers in TimeSteward events is forbidden by default because it is nondeterministic across platforms. If you know you don't need to be consistent between different computers, or you otherwise know EXACTLY what you're doing, you may add fn allow_floats_unsafe() {true} to your Basics.");
      self.generic.generator.next_f32()
    }
    fn next_f64(&mut self) -> f64 {
      assert!(<$B as $crate::Basics>::allow_floats_unsafe(), "Using floating point numbers in TimeSteward events is forbidden by default because it is nondeterministic across platforms. If you know you don't need to be consistent between different computers, or you otherwise know EXACTLY what you're doing, you may add fn allow_floats_unsafe() {true} to your Basics.");
      self.generic.generator.next_f64()
    }
  }
}





pub fn extended_time_of_fiat_event<BaseTime: Ord>(time: BaseTime,
                                                  id: TimeId)
                                                  -> GenericExtendedTime<BaseTime> {
  GenericExtendedTime {
    base: time,
    iteration: 0,
    id: id.for_fiat_event_internal(),
  }
}

pub fn time_id_for_predicted_event(predictor_id: PredictorId,
                                   row_id: RowId,
                                   iteration: IterationType,
                                   dependencies_hash: DeterministicRandomId)
                                   -> TimeId {
  TimeId::new(&(predictor_id, row_id, iteration, dependencies_hash))
}
pub fn next_extended_time_of_predicted_event<BaseTime: Ord>
  (predictor_id: PredictorId,
   row_id: RowId,
   dependencies_hash: DeterministicRandomId,
   event_base_time: BaseTime,
   from: &GenericExtendedTime<BaseTime>)
   -> Option<GenericExtendedTime<BaseTime>> {
  let (iteration, id) = match event_base_time.cmp(&from.base) {
    Ordering::Less => return None, // short-circuit
    Ordering::Greater => {
      (0, time_id_for_predicted_event(predictor_id, row_id, 0, dependencies_hash))
    }
    Ordering::Equal => {
      let id = time_id_for_predicted_event(predictor_id, row_id, from.iteration, dependencies_hash);
      if id > from.id {
        (from.iteration, id)
      } else {
        if from.iteration == IterationType::max_value() {
          panic!("Too many iterations at the same base time; probably an infinite loop")
        }
        (from.iteration + 1,
         time_id_for_predicted_event(predictor_id, row_id, from.iteration + 1, dependencies_hash))
      }
    }
  };
  Some(GenericExtendedTime {
    base: event_base_time,
    iteration: iteration,
    id: id,
  })
}

{'fraction_non_alphanumeric': 0.14161904761904762, 'fraction_numerical': 0.0054285714285714284, 'mean_word_length': 3.590382513661202, 'pattern_counts': {'":': 0, '<': 187, '<?xml version=': 0, '>': 253, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7521017', 'n_tokens_mistral': 7375, 'n_tokens_neox': 7143, 'n_words': 1687}