<commit_before>use std::collections::{HashMap, HashSet};
use std::ops::{Deref, DerefMut};

use ndarray::Array2;
use jagged_array::Jagged2;

use render::Renderer;
use resman::AudioBuffer;
use routing::{Edge, GraphWatcher, NodeData, NodeHandle};
use routing::effect::{PrimitiveEffect, EffectData};
use streaming_iterator::StreamingIterator;
use util::unpack_f32;


#[derive(Debug)]
struct NodeMap {
    nodes: HashMap<NodeHandle, Node>,
}

#[derive(Default, Debug)]
pub struct RefRenderer {
    nodes: NodeMap,
    /// inputs[slot][sample] represents the value of the external input at
    /// the given slot and time. On OOB, it is assumed to be 0.
    inputs: Vec<Vec<f32>>,
    /// Next expected sample to be queried.
    /// This is tracked because if we do a seek, the inputs need to be zero'd.
    head: u64,
}

#[derive(Default, Debug)]
struct Node {
    data: Option<MyNodeData>,
    inbound: HashSet<Edge>
}

#[derive(Debug)]
enum MyNodeData {
    /// This node is a non-primitive effect.
    UserNode(NodeMap),
    /// Primitive effect (delay, constant, etc).
    Primitive(PrimitiveEffect),
    /// External audio.
    Buffer(AudioBuffer),
}

impl Renderer for RefRenderer {
    fn fill_buffer(&mut self, buff: &mut Array2<f32>, idx: u64, inputs: Jagged2<f32>) {
        let (n_slots, n_times) = buff.dim().into();
        // Store inputs for future use
        {
            // If this is a seek operation, forget historical input values.
            if idx != self.head {
                for slot in &mut self.inputs {
                    *slot = Vec::new();
                    // NB: Improper indexing on 32-bit OS, but will OOM first.
                    slot.resize(idx as usize, 0f32);
                }
            }
            // Make sure we have storage space for all inputs.
            while self.inputs.len() < buff.len() {
                // NB: Improper indexing on 32-bit OS, but will OOM first.
                let mut v = Vec::with_capacity(idx as usize);
                v.resize(idx as usize, 0f32);
                self.inputs.push(v);
            }
            let mut stream = inputs.stream();
            let mut self_it = self.inputs.iter_mut();
            while let (Some(row), Some(vec_dest)) = (stream.next(), self_it.next()) {
                assert_eq!(vec_dest.len(), idx as usize);
                vec_dest.extend(row);
                assert!(vec_dest.len() <= idx as usize + n_times); // cannot send inputs ahead of outputs.
                let pad_val = vec_dest.last().cloned().unwrap_or(0f32);
                vec_dest.resize(idx as usize +n_times, pad_val);
            }
        }

        // Calculate outputs
        for slot in 0..n_slots as u32 {
            for time in idx..idx+n_times as u64 {
                buff[[slot as usize, (time - idx) as usize]] = self.get_sample(time, slot);
            }
        }
        // Keep track of the playhead
        self.head = idx + n_times as u64;
    }
}
impl RefRenderer {
    fn get_sample(&mut self, time: u64, slot: u32) -> f32 {
        let node = &self.nodes[&NodeHandle::toplevel()];
        self.sum_input_to_slot(&self.nodes, &node, time, slot, &Vec::new()) as f32
    }
    /// Get the value on an edge at a particular time
    /// When backtracking from the output, we push each Node onto the context if we enter inside of
    ///   it (i.e. if it's a nested DAG) & pop when exiting.
    fn get_value(&self, nodes: &NodeMap, edge: &Edge, time: u64, context: &Vec<(&NodeMap, NodeHandle)>) -> f64 {
        let from = edge.from_full();
        if *from.node_handle() == None {
            if context.is_empty() {
                // toplevel input (i.e. external input)
                let slot = edge.from_slot() as usize;
                *self.inputs.get(slot).and_then(|v| v.get(time as usize)).unwrap_or(&0f32) as f64
            } else {
                // Reading from one of the inputs to the top of `context`
                // TODO: we can avoid cloning by reversing the pop after recursing.
                let mut new_context = context.clone();
                let (new_nodes, head) = new_context.pop().unwrap();
                // Sum the inputs to the matching slot
                self.sum_input_to_slot(new_nodes, &new_nodes[&head], time, edge.from_slot(), &new_context)
            }
        } else {
            // Reading from another node within the DAG
            let node = &nodes[&from];
            match *node.data.as_ref().expect("Expected node to have associated data") {
                MyNodeData::UserNode(ref new_nodes) => {
                    let mut new_context = context.clone();
                    new_context.push((nodes, from));
                    // Now find the *output* of the sub dag (or 0 if the sub dag has no outputs)
                    new_nodes.get(&NodeHandle::toplevel()).map_or(0f64, |root_node| {
                        self.sum_input_to_slot(new_nodes, root_node, time, edge.from_slot(), &new_context)
                    })
                },
                MyNodeData::Primitive(prim) => match prim {
                    // Output = sum of all inputs to slot 0.
                    PrimitiveEffect::Delay => {
                        // The only nonzero output is slot=0.
                        if edge.from_slot() != 0 {
                            warn!("Attempt to read from Delay slot != 0");
                            0f64
                        } else {
                            let delay_frames = self.sum_input_to_slot(nodes, node, time, 1, context);
                            // Clamp delay value to [0, u64::max]
                            let delay_int = if delay_frames < 0f64 {
                                0u64
                            } else if delay_frames > u64::max_value() as f64 {
                                // TODO: u64::max isn't precisely representable in f64;
                                // will this cause issues?
                                u64::max_value()
                            } else {
                                // Note: this conversion is flooring.
                                delay_frames as u64
                            };
                            // t<0 -> value is 0.
                            time.checked_sub(delay_int).map_or(0f64, |origin_time| {
                                self.sum_input_to_slot(nodes, node, origin_time, 0, context)
                            })
                        }
                    },
                    PrimitiveEffect::F32Constant => {
                        // Float value is encoded via the slot.
                        let value = edge.from_slot();
                        unpack_f32(value) as f64
                    },
                    PrimitiveEffect::Multiply => {
                        // The only nonzero output is slot=0.
                        if edge.from_slot() != 0 {
                            warn!("Attempt to read from Multiply slot != 0");
                            0f64
                        } else {
                            // Sum all inputs from slot=0 and slot=2 into two separate
                            // variables, then multiply them.
                            let val_a = self.sum_input_to_slot(nodes, node, time, 0, context);
                            let val_b = self.sum_input_to_slot(nodes, node, time, 1, context);
                            val_a * val_b
                        }
                    },
                    PrimitiveEffect::Divide => {
                        // The only nonzero output is slot=0.
                        if edge.from_slot() != 0 {
                            warn!("Attempt to read from MultInv slot != 0");
                            0f64
                        } else {
                            // Sum all inputs
                            let dividend = self.sum_input_to_slot(nodes, node, time, 0, context);
                            let divisor = self.sum_input_to_slot(nodes, node, time, 1, context);
                            dividend / divisor
                        }
                    },
                    PrimitiveEffect::Minimum => {
                        // The only nonzero output is slot=0.
                        if edge.from_slot() != 0 {
                            warn!("Attempt to read from Modulo slot != 0");
                            0f64
                        } else {
                            let input_a = self.sum_input_to_slot(nodes, node, time, 0, context);
                            let input_b = self.sum_input_to_slot(nodes, node, time, 1, context);
                            input_a.min(input_b)
                        }
                    }
                    PrimitiveEffect::Modulo => {
                        // The only nonzero output is slot=0.
                        if edge.from_slot() != 0 {
                            warn!("Attempt to read from Modulo slot != 0");
                            0f64
                        } else {
                            // Sum all dividends
                            let dividend = self.sum_input_to_slot(nodes, node, time, 0, context);
                            // Sum all divisors
                            let divisor = self.sum_input_to_slot(nodes, node, time, 1, context);
                            let rem = dividend % divisor;
                            if rem < 0f64 {
                                // TODO: We may be losing precision here, if rem is small.
                                // We should find a way to do true modulus.
                                rem + divisor
                            } else {
                                rem
                            }
                        }
                    },
                },
                MyNodeData::Buffer(ref buf) => buf.get(time, edge.from_slot()) as f64,
            }
        }
    }
    /// Return the sum of all inputs into a specific slot of the given
    /// node at the given time.
    fn sum_input_to_slot(&self, nodes: &NodeMap, node: &Node, time: u64, slot: u32, context: &Vec<(&NodeMap, NodeHandle)>) -> f64 {
        let edges_in = node.inbound.iter().filter(|in_edge| {
            in_edge.to_slot() == slot
        });
        edges_in.map(|edge| self.get_value(nodes, edge, time, context)).sum()
    }

    fn make_node(&self, effect: &NodeData) -> MyNodeData {
        match *effect.data() {
            EffectData::Primitive(e) => MyNodeData::Primitive(e),
            EffectData::Buffer(ref buff) => MyNodeData::Buffer(buff.clone()),
            EffectData::RouteGraph(ref graph) => {
                let mut nodes: NodeMap = Default::default();

                for (node, data) in graph.iter_nodes() {
                    nodes.insert(*node, Node::new(Some(self.make_node(data))));
                }
                for edge in graph.iter_edges() {
                    nodes.get_mut(&edge.to_full()).unwrap().inbound.insert(edge.clone());
                }
                MyNodeData::UserNode(nodes)
            }
        }
    }
}

impl GraphWatcher for RefRenderer {
    fn on_add_node(&mut self, handle: &NodeHandle, data: &NodeData) {
        let my_node_data = self.make_node(data);
        self.nodes.insert(*handle, Node::new(Some(my_node_data)));
    }
    fn on_del_node(&mut self, handle: &NodeHandle) {
        self.nodes.remove(handle);
    }
    fn on_add_edge(&mut self, edge: &Edge) {
        self.nodes.get_mut(&edge.to_full()).unwrap().inbound.insert(edge.clone());
    }
    fn on_del_edge(&mut self, edge: &Edge) {
        self.nodes.get_mut(&edge.to_full()).unwrap().inbound.remove(edge);
    }
}


impl Node {
    fn new(data: Option<MyNodeData>) -> Self {
        Node {
            data: data,
            inbound: HashSet::new(),
        }
    }
}


impl Default for NodeMap {
    fn default() -> Self {
        // Create a NodeMap that already has an entry for the toplevel so that it can receive
        // edges.
        let nodes = Some((NodeHandle::toplevel(), Default::default())).into_iter().collect();
        Self { nodes }
    }
}

impl Deref for NodeMap {
    type Target = HashMap<NodeHandle, Node>;
    fn deref(&self) -> &Self::Target {
        &self.nodes
    }
}

impl DerefMut for NodeMap {
    fn deref_mut(&mut self) -> &mut Self::Target {
        &mut self.nodes
    }
}
<commit_msg>More debug info<commit_after>use std::collections::{HashMap, HashSet};
use std::ops::{Deref, DerefMut};

use ndarray::Array2;
use jagged_array::Jagged2;

use render::Renderer;
use resman::AudioBuffer;
use routing::{Edge, GraphWatcher, NodeData, NodeHandle};
use routing::effect::{PrimitiveEffect, EffectData};
use streaming_iterator::StreamingIterator;
use util::unpack_f32;


#[derive(Debug)]
struct NodeMap {
    nodes: HashMap<NodeHandle, Node>,
}

#[derive(Default, Debug)]
pub struct RefRenderer {
    nodes: NodeMap,
    /// inputs[slot][sample] represents the value of the external input at
    /// the given slot and time. On OOB, it is assumed to be 0.
    inputs: Vec<Vec<f32>>,
    /// Next expected sample to be queried.
    /// This is tracked because if we do a seek, the inputs need to be zero'd.
    head: u64,
}

#[derive(Default, Debug)]
struct Node {
    data: Option<MyNodeData>,
    inbound: HashSet<Edge>
}

#[derive(Debug)]
enum MyNodeData {
    /// This node is a non-primitive effect.
    UserNode(NodeMap),
    /// Primitive effect (delay, constant, etc).
    Primitive(PrimitiveEffect),
    /// External audio.
    Buffer(AudioBuffer),
}

impl Renderer for RefRenderer {
    fn fill_buffer(&mut self, buff: &mut Array2<f32>, idx: u64, inputs: Jagged2<f32>) {
        let (n_slots, n_times) = buff.dim().into();
        // Store inputs for future use
        {
            // If this is a seek operation, forget historical input values.
            if idx != self.head {
                for slot in &mut self.inputs {
                    *slot = Vec::new();
                    // NB: Improper indexing on 32-bit OS, but will OOM first.
                    slot.resize(idx as usize, 0f32);
                }
            }
            // Make sure we have storage space for all inputs.
            while self.inputs.len() < buff.len() {
                // NB: Improper indexing on 32-bit OS, but will OOM first.
                let mut v = Vec::with_capacity(idx as usize);
                v.resize(idx as usize, 0f32);
                self.inputs.push(v);
            }
            let mut stream = inputs.stream();
            let mut self_it = self.inputs.iter_mut();
            while let (Some(row), Some(vec_dest)) = (stream.next(), self_it.next()) {
                assert_eq!(vec_dest.len(), idx as usize);
                vec_dest.extend(row);
                assert!(vec_dest.len() <= idx as usize + n_times); // cannot send inputs ahead of outputs.
                let pad_val = vec_dest.last().cloned().unwrap_or(0f32);
                vec_dest.resize(idx as usize +n_times, pad_val);
            }
        }

        // Calculate outputs
        for slot in 0..n_slots as u32 {
            for time in idx..idx+n_times as u64 {
                buff[[slot as usize, (time - idx) as usize]] = self.get_sample(time, slot);
            }
        }
        // Keep track of the playhead
        self.head = idx + n_times as u64;
    }
}
impl RefRenderer {
    fn get_sample(&mut self, time: u64, slot: u32) -> f32 {
        let node = &self.nodes[&NodeHandle::toplevel()];
        self.sum_input_to_slot(&self.nodes, &node, time, slot, &Vec::new()) as f32
    }
    /// Get the value on an edge at a particular time
    /// When backtracking from the output, we push each Node onto the context if we enter inside of
    ///   it (i.e. if it's a nested DAG) & pop when exiting.
    fn get_value(&self, nodes: &NodeMap, edge: &Edge, time: u64, context: &Vec<(&NodeMap, NodeHandle)>) -> f64 {
        let from = edge.from_full();
        if *from.node_handle() == None {
            if context.is_empty() {
                // toplevel input (i.e. external input)
                let slot = edge.from_slot() as usize;
                let value = *self.inputs.get(slot).and_then(|v| v.get(time as usize)).unwrap_or(&0f32) as f64;
                trace!("Renderer reading from ext input [t={}] [slot={}]: {}", time, slot, value);
                value
            } else {
                // Reading from one of the inputs to the top of `context`
                // TODO: we can avoid cloning by reversing the pop after recursing.
                let mut new_context = context.clone();
                let (new_nodes, head) = new_context.pop().unwrap();
                // Sum the inputs to the matching slot
                self.sum_input_to_slot(new_nodes, &new_nodes[&head], time, edge.from_slot(), &new_context)
            }
        } else {
            // Reading from another node within the DAG
            let node = &nodes[&from];
            match *node.data.as_ref().expect("Expected node to have associated data") {
                MyNodeData::UserNode(ref new_nodes) => {
                    let mut new_context = context.clone();
                    new_context.push((nodes, from));
                    // Now find the *output* of the sub dag (or 0 if the sub dag has no outputs)
                    new_nodes.get(&NodeHandle::toplevel()).map_or(0f64, |root_node| {
                        self.sum_input_to_slot(new_nodes, root_node, time, edge.from_slot(), &new_context)
                    })
                },
                MyNodeData::Primitive(prim) => match prim {
                    // Output = sum of all inputs to slot 0.
                    PrimitiveEffect::Delay => {
                        // The only nonzero output is slot=0.
                        if edge.from_slot() != 0 {
                            warn!("Attempt to read from Delay slot != 0");
                            0f64
                        } else {
                            let delay_frames = self.sum_input_to_slot(nodes, node, time, 1, context);
                            // Clamp delay value to [0, u64::max]
                            let delay_int = if delay_frames < 0f64 {
                                0u64
                            } else if delay_frames > u64::max_value() as f64 {
                                // TODO: u64::max isn't precisely representable in f64;
                                // will this cause issues?
                                u64::max_value()
                            } else {
                                // Note: this conversion is flooring.
                                delay_frames as u64
                            };
                            // t<0 -> value is 0.
                            time.checked_sub(delay_int).map_or(0f64, |origin_time| {
                                self.sum_input_to_slot(nodes, node, origin_time, 0, context)
                            })
                        }
                    },
                    PrimitiveEffect::F32Constant => {
                        // Float value is encoded via the slot.
                        let value = edge.from_slot();
                        unpack_f32(value) as f64
                    },
                    PrimitiveEffect::Multiply => {
                        // The only nonzero output is slot=0.
                        if edge.from_slot() != 0 {
                            warn!("Attempt to read from Multiply slot != 0");
                            0f64
                        } else {
                            // Sum all inputs from slot=0 and slot=2 into two separate
                            // variables, then multiply them.
                            let val_a = self.sum_input_to_slot(nodes, node, time, 0, context);
                            let val_b = self.sum_input_to_slot(nodes, node, time, 1, context);
                            val_a * val_b
                        }
                    },
                    PrimitiveEffect::Divide => {
                        // The only nonzero output is slot=0.
                        if edge.from_slot() != 0 {
                            warn!("Attempt to read from MultInv slot != 0");
                            0f64
                        } else {
                            // Sum all inputs
                            let dividend = self.sum_input_to_slot(nodes, node, time, 0, context);
                            let divisor = self.sum_input_to_slot(nodes, node, time, 1, context);
                            dividend / divisor
                        }
                    },
                    PrimitiveEffect::Minimum => {
                        // The only nonzero output is slot=0.
                        if edge.from_slot() != 0 {
                            warn!("Attempt to read from Modulo slot != 0");
                            0f64
                        } else {
                            let input_a = self.sum_input_to_slot(nodes, node, time, 0, context);
                            let input_b = self.sum_input_to_slot(nodes, node, time, 1, context);
                            input_a.min(input_b)
                        }
                    }
                    PrimitiveEffect::Modulo => {
                        // The only nonzero output is slot=0.
                        if edge.from_slot() != 0 {
                            warn!("Attempt to read from Modulo slot != 0");
                            0f64
                        } else {
                            // Sum all dividends
                            let dividend = self.sum_input_to_slot(nodes, node, time, 0, context);
                            // Sum all divisors
                            let divisor = self.sum_input_to_slot(nodes, node, time, 1, context);
                            let rem = dividend % divisor;
                            if rem < 0f64 {
                                // TODO: We may be losing precision here, if rem is small.
                                // We should find a way to do true modulus.
                                rem + divisor
                            } else {
                                rem
                            }
                        }
                    },
                },
                MyNodeData::Buffer(ref buf) => buf.get(time, edge.from_slot()) as f64,
            }
        }
    }
    /// Return the sum of all inputs into a specific slot of the given
    /// node at the given time.
    fn sum_input_to_slot(&self, nodes: &NodeMap, node: &Node, time: u64, slot: u32, context: &Vec<(&NodeMap, NodeHandle)>) -> f64 {
        let edges_in = node.inbound.iter().filter(|in_edge| {
            in_edge.to_slot() == slot
        });
        edges_in.map(|edge| self.get_value(nodes, edge, time, context)).sum()
    }

    fn make_node(&self, effect: &NodeData) -> MyNodeData {
        match *effect.data() {
            EffectData::Primitive(e) => MyNodeData::Primitive(e),
            EffectData::Buffer(ref buff) => MyNodeData::Buffer(buff.clone()),
            EffectData::RouteGraph(ref graph) => {
                let mut nodes: NodeMap = Default::default();

                for (node, data) in graph.iter_nodes() {
                    nodes.insert(*node, Node::new(Some(self.make_node(data))));
                }
                for edge in graph.iter_edges() {
                    nodes.get_mut(&edge.to_full()).unwrap().inbound.insert(edge.clone());
                }
                MyNodeData::UserNode(nodes)
            }
        }
    }
}

impl GraphWatcher for RefRenderer {
    fn on_add_node(&mut self, handle: &NodeHandle, data: &NodeData) {
        let my_node_data = self.make_node(data);
        self.nodes.insert(*handle, Node::new(Some(my_node_data)));
    }
    fn on_del_node(&mut self, handle: &NodeHandle) {
        self.nodes.remove(handle);
    }
    fn on_add_edge(&mut self, edge: &Edge) {
        self.nodes.get_mut(&edge.to_full()).unwrap().inbound.insert(edge.clone());
    }
    fn on_del_edge(&mut self, edge: &Edge) {
        self.nodes.get_mut(&edge.to_full()).unwrap().inbound.remove(edge);
    }
}


impl Node {
    fn new(data: Option<MyNodeData>) -> Self {
        Node {
            data: data,
            inbound: HashSet::new(),
        }
    }
}


impl Default for NodeMap {
    fn default() -> Self {
        // Create a NodeMap that already has an entry for the toplevel so that it can receive
        // edges.
        let nodes = Some((NodeHandle::toplevel(), Default::default())).into_iter().collect();
        Self { nodes }
    }
}

impl Deref for NodeMap {
    type Target = HashMap<NodeHandle, Node>;
    fn deref(&self) -> &Self::Target {
        &self.nodes
    }
}

impl DerefMut for NodeMap {
    fn deref_mut(&mut self) -> &mut Self::Target {
        &mut self.nodes
    }
}

{'fraction_non_alphanumeric': 0.10717734148696492, 'fraction_numerical': 0.011586739620212423, 'mean_word_length': 1.3592444950645406, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7114719', 'n_tokens_mistral': 6947, 'n_tokens_neox': 6592, 'n_words': 1877}