<commit_before>use byteorder::{LittleEndian, WriteBytesExt};

#[derive(Copy, Clone, Debug)]
pub struct Label(usize);

struct ByteBuffer {
    data: Vec<u8>,
}

impl ByteBuffer {
    fn new() -> ByteBuffer {
        ByteBuffer {
            data: Vec::new()
        }
    }

    pub fn data(self) -> Vec<u8> {
        self.data
    }

    pub fn pos(&mut self) -> usize {
        self.data.len()
    }

    pub fn emit_u8(&mut self, value: u8) {
        self.data.write_u8(value).unwrap();
    }

    pub fn emit_u16(&mut self, value: u16) {
        self.data.write_u16::<LittleEndian>(value).unwrap();
    }

    pub fn emit_u32(&mut self, value: u32) {
        self.data.write_u32::<LittleEndian>(value).unwrap();
    }

    pub fn emit_u32_at(&mut self, value: u32, pos: usize) {
        let mut slice = &mut self.data[pos..];
        slice.write_u32::<LittleEndian>(value).unwrap();
    }

    pub fn emit_u64(&mut self, value: u64) {
        self.data.write_u64::<LittleEndian>(value).unwrap();
    }
}

pub struct Buffer {
    buf: ByteBuffer,
    labels: Vec<Option<usize>>,
    jumps: Vec<ForwardJump>,
}

impl Buffer {
    pub fn new() -> Buffer {
        Buffer {
            buf: ByteBuffer::new(),
            labels: Vec::new(),
            jumps: Vec::new(),
        }
    }

    pub fn finish(mut self) -> Vec<u8> {
        self.calculate_jumps();

        self.buf.data()
    }

    fn calculate_jumps(&mut self) {
        for jmp in &self.jumps {
            let target = self.labels[jmp.to.0].expect("label not defined");
            let diff = (target - jmp.at - 4) as i32;

            self.buf.emit_u32_at(diff as u32, jmp.at);
        }
    }

    pub fn create_label(&mut self) -> Label {
        let idx = self.buf.pos();
        self.labels.push(None);

        Label(idx)
    }

    pub fn define_label(&mut self, lbl: Label) {
        assert!(self.labels[lbl.0].is_none());
        self.labels[lbl.0] = Some(self.buf.pos());
    }

    pub fn emit_label(&mut self, lbl: Label) {
        let value = self.labels[lbl.0];

        match value {
            Some(idx) => {
                let current = self.buf.pos() + 4;
                let target = idx;

                let diff = -((current - target) as i32);
                self.emit_u32(diff as u32);
            }

            None => {
                let pos = self.buf.pos();
                self.emit_u32(0);
                self.jumps.push(ForwardJump {
                    at: pos,
                    to: lbl
                });
            }
        }
    }

    pub fn emit_u8(&mut self, value: u8) {
        self.buf.emit_u8(value);
    }

    pub fn emit_u32(&mut self, value: u32) {<commit_msg>codegen: renamed to fix_forward_jumps<commit_after>use byteorder::{LittleEndian, WriteBytesExt};

#[derive(Copy, Clone, Debug)]
pub struct Label(usize);

struct ByteBuffer {
    data: Vec<u8>,
}

impl ByteBuffer {
    fn new() -> ByteBuffer {
        ByteBuffer {
            data: Vec::new()
        }
    }

    pub fn data(self) -> Vec<u8> {
        self.data
    }

    pub fn pos(&mut self) -> usize {
        self.data.len()
    }

    pub fn emit_u8(&mut self, value: u8) {
        self.data.write_u8(value).unwrap();
    }

    pub fn emit_u16(&mut self, value: u16) {
        self.data.write_u16::<LittleEndian>(value).unwrap();
    }

    pub fn emit_u32(&mut self, value: u32) {
        self.data.write_u32::<LittleEndian>(value).unwrap();
    }

    pub fn emit_u32_at(&mut self, value: u32, pos: usize) {
        let mut slice = &mut self.data[pos..];
        slice.write_u32::<LittleEndian>(value).unwrap();
    }

    pub fn emit_u64(&mut self, value: u64) {
        self.data.write_u64::<LittleEndian>(value).unwrap();
    }
}

pub struct Buffer {
    buf: ByteBuffer,
    labels: Vec<Option<usize>>,
    jumps: Vec<ForwardJump>,
}

impl Buffer {
    pub fn new() -> Buffer {
        Buffer {
            buf: ByteBuffer::new(),
            labels: Vec::new(),
            jumps: Vec::new(),
        }
    }

    pub fn finish(mut self) -> Vec<u8> {
        self.fix_forward_jumps();

        self.buf.data()
    }

    fn fix_forward_jumps(&mut self) {
        for jmp in &self.jumps {
            let target = self.labels[jmp.to.0].expect("label not defined");
            let diff = (target - jmp.at - 4) as i32;

            self.buf.emit_u32_at(diff as u32, jmp.at);
        }
    }

    pub fn create_label(&mut self) -> Label {
        let idx = self.buf.pos();
        self.labels.push(None);

        Label(idx)
    }

    pub fn define_label(&mut self, lbl: Label) {
        assert!(self.labels[lbl.0].is_none());
        self.labels[lbl.0] = Some(self.buf.pos());
    }

    pub fn emit_label(&mut self, lbl: Label) {
        let value = self.labels[lbl.0];

        match value {
            // backward jumps already know their target
            Some(idx) => {
                let current = self.buf.pos() + 4;
                let target = idx;

                let diff = -((current - target) as i32);
                self.emit_u32(diff as u32);
            }

            // forward jumps do not know their target yet
            None => {
                let pos = self.buf.pos();
                self.emit_u32(0);
                self.jumps.push(ForwardJump {
                    at: pos,
                    to: lbl
                });
            }
        }
    }

    pub fn emit_u8(&mut self, value: u8) {
        self.buf.emit_u8(value);
    }

    pub fn emit_u32(&mut self, value: u32) {
{'fraction_non_alphanumeric': 0.14086294416243655, 'fraction_numerical': 0.02102973168963017, 'mean_word_length': 1.6964809384164223, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4854095', 'n_tokens_mistral': 1919, 'n_tokens_neox': 1835, 'n_words': 409}