<commit_before>//! Provides a queue of block indices that the sampler positions can be initialized
//! from for the worker threads. The queue itself is not changed after creation
//! we simply work through it with an atomic counter to track the index of the next
//! block to work on

use std::vec::Vec;
use std::sync::atomic::{AtomicUint, AcqRel};
use sampler::morton;

/// The queue of blocks to be worked on shared immutably between worker threads.
pub struct BlockQueue {
    /// The block indices of blocks to work on for the image
    blocks: Vec<(u32, u32)>,
    /// Get the dimensions of an individual block
    dimensions: (u32, u32),
    /// Index of the next block to be worked on
    next: AtomicUint,
}

impl BlockQueue {
    /// Create a block queue for the image with dimensions `img`.
    /// Panics if the image is not evenly broken into blocks of dimension `dim`
    pub fn new(img: (u32, u32), dim: (u32, u32)) -> BlockQueue {
        if img.0 % dim.0 != 0 || img.1 % dim.1 != 0 {
            panic!("Image with dimension {} not evenly divided by dims of {}",
                   img, dim);
        }
        let num_blocks = (img.0 / dim.0, img.1 / dim.1);
        let mut blocks: Vec<(u32, u32)> = range(0, num_blocks.0 * num_blocks.1)
            .map(|i| (i % num_blocks.0, i / num_blocks.0)).collect();
        blocks.sort_by(|a, b| morton::morton2(a).cmp(&morton::morton2(b)));
        BlockQueue { blocks: blocks, dimensions: dim, next: AtomicUint::new(0) }
    }
    /// Get the dimensions of an individual block in the queue
    pub fn block_dim(&self) -> (u32, u32) { self.dimensions }
}

impl Iterator<(u32, u32)> for BlockQueue {
    fn next(&mut self) -> Option<(u32, u32)> {
        let i = self.next.fetch_add(1, AcqRel);
        if i >= self.blocks.len() {
            None
        } else {
            Some(self.blocks[i])
        }
    }
    fn size_hint(&self) -> (uint, Option<uint>) {
        let i = self.next.load(AcqRel);
        (i, Some(i))
    }
}
<commit_msg>Make it possible to iterate cleanly over the queue on multiple threads<commit_after>//! Provides a queue of block indices that the sampler positions can be initialized
//! from for the worker threads. The queue itself is not changed after creation
//! we simply work through it with an atomic counter to track the index of the next
//! block to work on

use std::vec::Vec;
use std::sync::atomic::{AtomicUint, AcqRel};
use sampler::morton;

/// The queue of blocks to be worked on shared immutably between worker threads.
pub struct BlockQueue {
    /// The block indices of blocks to work on for the image
    blocks: Vec<(u32, u32)>,
    /// Get the dimensions of an individual block
    dimensions: (u32, u32),
    /// Index of the next block to be worked on
    next: AtomicUint,
}

/// Iterator to work through the queue safely
pub struct BlockQueueIterator<'a> {
    queue: &'a BlockQueue,
}

impl BlockQueue {
    /// Create a block queue for the image with dimensions `img`.
    /// Panics if the image is not evenly broken into blocks of dimension `dim`
    pub fn new(img: (u32, u32), dim: (u32, u32)) -> BlockQueue {
        if img.0 % dim.0 != 0 || img.1 % dim.1 != 0 {
            panic!("Image with dimension {} not evenly divided by blocks of {}", img, dim);
        }
        let num_blocks = (img.0 / dim.0, img.1 / dim.1);
        let mut blocks: Vec<(u32, u32)> = range(0, num_blocks.0 * num_blocks.1)
            .map(|i| (i % num_blocks.0, i / num_blocks.0)).collect();
        blocks.sort_by(|a, b| morton::morton2(a).cmp(&morton::morton2(b)));
        BlockQueue { blocks: blocks, dimensions: dim, next: AtomicUint::new(0) }
    }
    /// Get the dimensions of an individual block in the queue
    pub fn block_dim(&self) -> (u32, u32) { self.dimensions }
    /// Get an iterator to work through the queue
    pub fn iter(&self) -> BlockQueueIterator { BlockQueueIterator { queue: self } }
    /// Get the next block in the queue or None if the queue is finished
    fn next(&self) -> Option<(u32, u32)> {
        let i = self.next.fetch_add(1, AcqRel);
        if i >= self.blocks.len() {
            None
        } else {
            Some(self.blocks[i])
        }
    }
    /// Get the length of the queue
    pub fn len(&self) -> uint { self.blocks.len() }
}

impl<'a> Iterator<(u32, u32)> for BlockQueueIterator<'a> {
    fn next(&mut self) -> Option<(u32, u32)> {
        self.queue.next()
    }
}

{'fraction_non_alphanumeric': 0.12886481606860753, 'fraction_numerical': 0.023922365154592642, 'mean_word_length': 2.562700964630225, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3071572', 'n_tokens_mistral': 1427, 'n_tokens_neox': 1373, 'n_words': 522}