<commit_before>use std::io::Read;
use error::{Result, Error, ErrorKind};
use reader::{FbxEvent, PropertyValue};
use super::CommonState;

#[derive(Debug, Clone)]
pub struct BinaryParser {
    version: u32,
    end_offset_stack: Vec<u32>,
}

impl BinaryParser {
    pub fn new(version: u32) -> Self {
        BinaryParser {
            version: version,
            end_offset_stack: vec![],
        }
    }

    pub fn next<R: Read>(&mut self, reader: &mut R, common: &mut CommonState) -> Result<FbxEvent> {
        // Check if the previously read node ends here.
        if let Some(&end_pos_top) = self.end_offset_stack.last() {
            if end_pos_top as u64 == common.pos {
                // Reached the end of previously read node.
                self.end_offset_stack.pop();
                return Ok(FbxEvent::EndNode);
            }
        }

        // Read a node record header.
        let node_record_header = try!(NodeRecordHeader::read(reader, &mut common.pos));
        if node_record_header.is_null_record() {
            // End of a node.
            return if let Some(expected_pos) = self.end_offset_stack.pop() {
                if common.pos == expected_pos as u64 {
                    Ok(FbxEvent::EndNode)
                } else {
                    // Data is collapsed (the node doesn't end at expected position).
                    Err(Error::new(
                            common.pos,
                            ErrorKind::DataError(format!("Node does not end at expected position (expected {}, now at {})", expected_pos, common.pos))))
                }
            } else {
                // Data is collapsed (an extra node end marker found).
                Err(Error::new(common.pos, ErrorKind::DataError("An extra node end marker found".to_string())))
            };
        } else {
            // Start of a node.
            self.end_offset_stack.push(node_record_header.end_offset);
        }

        // Read a node name.
        let name = try_read_fixstr!(common.pos, reader, node_record_header.name_len);

        // Read properties.
        let mut properties = Vec::<PropertyValue>::with_capacity(node_record_header.num_properties as usize);
        for _ in 0..node_record_header.num_properties {
            let prop = try!(self.read_property(reader, common));
            properties.push(prop);
        }

        Ok(FbxEvent::StartNode {
            name: name,
            properties: properties,
        })
    }

    fn read_property<R: Read>(&mut self, reader: &mut R, common: &mut CommonState) -> Result<PropertyValue> {
        let type_code = try_read_le_u8!(common.pos, reader);
        // type code must be ASCII.
        let type_code = if type_code > 0x80 {
            return Err(Error::new(common.pos-1, ErrorKind::DataError(format!("Expected property type code (ASCII) but got {:#x}", type_code))));
        } else {
            type_code as char
        };
        let value = match type_code {
            // 1 bit boolean (1: true, 0: false) encoded as the LSB of a 1 byte value.
            'C' => {
                let val = try_read_le_u8!(common.pos, reader);
                // It seems 'T' (0x54) is used as `false`, 'T' (0x59) is used as `true`.
                if (val != 'T' as u8) && (val != 'Y' as u8) {
                    // Should this treated as error?
                    // (I don't know whether other characters than 'T' and 'Y' are allowed...)
                    warn!("Expected 'T' or 'Y' for representaton of boolean property value, but got {:#x}", val);
                }
                // Check LSB.
                PropertyValue::Bool(val & 1 == 1)
            },
            // 2 byte signed integer.
            'Y' => {
                PropertyValue::I16(try_read_le_i16!(common.pos, reader))
            },
            // 4 byte signed integer.
            'I' => {
                PropertyValue::I32(try_read_le_i32!(common.pos, reader))
            },
            // 4 byte single-precision IEEE 754 floating-point number.
            'F' => {
                PropertyValue::F32(try_read_le_f32!(common.pos, reader))
            },
            // 8 byte double-precision IEEE 754 floating-point number.
            'D' => {
                PropertyValue::F64(try_read_le_f64!(common.pos, reader))
            },
            // 8 byte signed integer.
            'L' => {
                PropertyValue::I64(try_read_le_i64!(common.pos, reader))
            },
            // Array types
            'f'|'d'|'l'|'i'|'b' => {
                let array_header = try!(PropertyArrayHeader::read(reader, &mut common.pos));
                try!(self.read_property_value_array(reader, common, type_code, &array_header))
            },
            // String
            'S' => {
                let length = try_read_le_u32!(common.pos, reader);
                PropertyValue::String(try_read_fixstr!(common.pos, reader, length))
            },
            // Raw binary data
            'R' => {
                let length = try_read_le_u32!(common.pos, reader);
                PropertyValue::Binary(try_read_exact!(common.pos, reader, length))
            },
            _ => {
                return Err(Error::new(
                        common.pos,
                        ErrorKind::DataError(format!(
                                "Unsupported type code appears in node property: type_code={}({:#x})",
                                type_code, type_code as u8))));
            }
        };
        Ok(value)
    }

    /// Read property value of array type from given stream which maybe compressed.
    fn read_property_value_array<R: Read>(&mut self,
                                          reader: &mut R, common: &mut CommonState,
                                          type_code: char, array_header: &PropertyArrayHeader) -> Result<PropertyValue> {
        match array_header.encoding {
            // 0; raw
            0 => {
                self.read_property_value_array_from_plain_stream(reader, common, type_code, array_header)
            },
            // 1: zlib compressed data
            1 => {
                Err(Error::new(
                        common.pos,
                        ErrorKind::Unimplemented("Parser for zlib compressed property array is not implemented yet".to_string())))
            },
            // Unknown.
            e => {
                Err(Error::new(
                        common.pos,
                        ErrorKind::DataError(format!("Unsupported property array encoding, got {:#x}", e))))
            }
        }
    }

    fn read_property_value_array_from_plain_stream<R: Read>(&mut self, reader: &mut R, common: &mut CommonState, type_code: char,
                                                            array_header: &PropertyArrayHeader) -> Result<PropertyValue> {
        Err(Error::new(
                common.pos,
                ErrorKind::Unimplemented("Parser for array type of property value is not implemented yet".to_string())))
    }
}

#[derive(Debug, Copy, Clone)]
struct NodeRecordHeader {
    end_offset: u32,
    num_properties: u32,
    property_list_len: u32,
    name_len: u8,
}

impl NodeRecordHeader {
    pub fn read<R: Read>(reader: &mut R, pos: &mut u64) -> Result<Self> {<commit_msg>Implement binary parser for node property<commit_after>extern crate byteorder;
extern crate flate2;

use std::io::Read;
use error::{Result, Error, ErrorKind};
use reader::{FbxEvent, PropertyValue};
use super::CommonState;

#[derive(Debug, Clone)]
pub struct BinaryParser {
    version: u32,
    end_offset_stack: Vec<u32>,
}

impl BinaryParser {
    pub fn new(version: u32) -> Self {
        BinaryParser {
            version: version,
            end_offset_stack: vec![],
        }
    }

    pub fn next<R: Read>(&mut self, reader: &mut R, common: &mut CommonState) -> Result<FbxEvent> {
        // Check if the previously read node ends here.
        if let Some(&end_pos_top) = self.end_offset_stack.last() {
            if end_pos_top as u64 == common.pos {
                // Reached the end of previously read node.
                self.end_offset_stack.pop();
                return Ok(FbxEvent::EndNode);
            }
        }

        // Read a node record header.
        let node_record_header = try!(NodeRecordHeader::read(reader, &mut common.pos));
        if node_record_header.is_null_record() {
            // End of a node.
            return if let Some(expected_pos) = self.end_offset_stack.pop() {
                if common.pos == expected_pos as u64 {
                    Ok(FbxEvent::EndNode)
                } else {
                    // Data is collapsed (the node doesn't end at expected position).
                    Err(Error::new(
                            common.pos,
                            ErrorKind::DataError(format!("Node does not end at expected position (expected {}, now at {})", expected_pos, common.pos))))
                }
            } else {
                // Data is collapsed (an extra node end marker found).
                Err(Error::new(common.pos, ErrorKind::DataError("An extra node end marker found".to_string())))
            };
        } else {
            // Start of a node.
            self.end_offset_stack.push(node_record_header.end_offset);
        }

        // Read a node name.
        let name = try_read_fixstr!(common.pos, reader, node_record_header.name_len);

        // Read properties.
        let mut properties = Vec::<PropertyValue>::with_capacity(node_record_header.num_properties as usize);
        for _ in 0..node_record_header.num_properties {
            let prop = try!(self.read_property(reader, common));
            properties.push(prop);
        }

        Ok(FbxEvent::StartNode {
            name: name,
            properties: properties,
        })
    }

    fn read_property<R: Read>(&mut self, reader: &mut R, common: &mut CommonState) -> Result<PropertyValue> {
        let type_code = try_read_le_u8!(common.pos, reader);
        // type code must be ASCII.
        let type_code = if type_code > 0x80 {
            return Err(Error::new(common.pos-1, ErrorKind::DataError(format!("Expected property type code (ASCII) but got {:#x}", type_code))));
        } else {
            type_code as char
        };
        let value = match type_code {
            // 1 bit boolean (1: true, 0: false) encoded as the LSB of a 1 byte value.
            'C' => {
                let val = try_read_le_u8!(common.pos, reader);
                // It seems 'T' (0x54) is used as `false`, 'T' (0x59) is used as `true`.
                if (val != 'T' as u8) && (val != 'Y' as u8) {
                    // Should this treated as error?
                    // (I don't know whether other characters than 'T' and 'Y' are allowed...)
                    warn!("Expected 'T' or 'Y' for representaton of boolean property value, but got {:#x}", val);
                }
                // Check LSB.
                PropertyValue::Bool(val & 1 == 1)
            },
            // 2 byte signed integer.
            'Y' => {
                PropertyValue::I16(try_read_le_i16!(common.pos, reader))
            },
            // 4 byte signed integer.
            'I' => {
                PropertyValue::I32(try_read_le_i32!(common.pos, reader))
            },
            // 4 byte single-precision IEEE 754 floating-point number.
            'F' => {
                PropertyValue::F32(try_read_le_f32!(common.pos, reader))
            },
            // 8 byte double-precision IEEE 754 floating-point number.
            'D' => {
                PropertyValue::F64(try_read_le_f64!(common.pos, reader))
            },
            // 8 byte signed integer.
            'L' => {
                PropertyValue::I64(try_read_le_i64!(common.pos, reader))
            },
            // Array types
            'f'|'d'|'l'|'i'|'b' => {
                let array_header = try!(PropertyArrayHeader::read(reader, &mut common.pos));
                try!(self.read_property_value_array(reader, common, type_code, &array_header))
            },
            // String
            'S' => {
                let length = try_read_le_u32!(common.pos, reader);
                PropertyValue::String(try_read_fixstr!(common.pos, reader, length))
            },
            // Raw binary data
            'R' => {
                let length = try_read_le_u32!(common.pos, reader);
                PropertyValue::Binary(try_read_exact!(common.pos, reader, length))
            },
            _ => {
                return Err(Error::new(
                        common.pos,
                        ErrorKind::DataError(format!(
                                "Unsupported type code appears in node property: type_code={}({:#x})",
                                type_code, type_code as u8))));
            }
        };
        Ok(value)
    }

    /// Read property value of array type from given stream which maybe compressed.
    fn read_property_value_array<R: Read>(&mut self,
                                          reader: &mut R, common: &mut CommonState,
                                          type_code: char, array_header: &PropertyArrayHeader) -> Result<PropertyValue> {
        match array_header.encoding {
            // 0; raw
            0 => {
                let (val, byte_size) = try!(self.read_property_value_array_from_plain_stream(reader, common.pos, type_code, array_header.array_length));
                common.pos += byte_size;
                Ok(val)
            },
            // 1: zlib compressed data
            1 => {
                let mut decoded_stream = flate2::read::ZlibDecoder::new(reader.by_ref().take(array_header.compressed_length as u64));
                let (val, _) = try!(self.read_property_value_array_from_plain_stream(&mut decoded_stream, common.pos, type_code, array_header.array_length));
                common.pos += array_header.compressed_length as u64;
                Ok(val)
            },
            // Unknown.
            e => {
                Err(Error::new(
                        common.pos,
                        ErrorKind::DataError(format!("Unsupported property array encoding, got {:#x}", e))))
            }
        }
    }

    fn read_property_value_array_from_plain_stream<R: Read>(&mut self, reader: &mut R, abs_pos: u64, type_code: char,
                                                            num_elements: u32) -> Result<(PropertyValue, u64)> {
        use self::byteorder::{ReadBytesExt, LittleEndian};
        Ok(match type_code {
            // Array of 4 byte single-precision IEEE 754 floating-point number.
            'f' => {
                let mut data = Vec::<f32>::with_capacity(num_elements as usize);
                for _ in 0..num_elements {
                    data.push(try_with_pos!(abs_pos, reader.read_f32::<LittleEndian>()));
                }
                (PropertyValue::VecF32(data), num_elements as u64 * 4)
            },
            // Array of 8 byte double-precision IEEE 754 floating-point number.
            'd' => {
                let mut data = Vec::<f64>::with_capacity(num_elements as usize);
                for _ in 0..num_elements {
                    data.push(try_with_pos!(abs_pos, reader.read_f64::<LittleEndian>()));
                }
                (PropertyValue::VecF64(data), num_elements as u64 * 8)
            },
            // Array of 8 byte signed integer.
            'l' => {
                let mut data = Vec::<i64>::with_capacity(num_elements as usize);
                for _ in 0..num_elements {
                    data.push(try_with_pos!(abs_pos, reader.read_i64::<LittleEndian>()));
                }
                (PropertyValue::VecI64(data), num_elements as u64 * 8)
            },
            // Array of 4 byte signed integer.
            'i' => {
                let mut data = Vec::<i32>::with_capacity(num_elements as usize);
                for _ in 0..num_elements {
                    data.push(try_with_pos!(abs_pos, reader.read_i32::<LittleEndian>()));
                }
                (PropertyValue::VecI32(data), num_elements as u64 * 4)
            },
            // Array of 1 byte booleans (always 0 or 1?).
            'b' => {
                let mut data = Vec::<bool>::with_capacity(num_elements as usize);
                for _ in 0..num_elements {
                    // Check LSB.
                    data.push(try_with_pos!(abs_pos, reader.read_u8()) & 1 == 1);
                }
                (PropertyValue::VecBool(data), num_elements as u64)
            },
            _ => {
                // Unreachable because `read_property()` gives only 'f' , 'd', 'l', 'i', or 'b' to
                // `read_property_value_array()`.
                unreachable!();
            }
        })
    }
}

#[derive(Debug, Copy, Clone)]
struct NodeRecordHeader {
    end_offset: u32,
    num_properties: u32,
    property_list_len: u32,
    name_len: u8,
}

impl NodeRecordHeader {
    pub fn read<R: Read>(reader: &mut R, pos: &mut u64) -> Result<Self> {
{'fraction_non_alphanumeric': 0.11595389925700579, 'fraction_numerical': 0.013514304101094015, 'mean_word_length': 1.615361077111383, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 82, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1179388', 'n_tokens_mistral': 4949, 'n_tokens_neox': 4664, 'n_words': 1170}