<commit_before>use crate::codecs::DataFormat;
use crate::codecs::Encodable;
use std::io::Result;
use std::io::Write;

const ID: u64 = 2;
const MEDIA_TYPE: &str = "application/octet-stream";

/// The most basic data format. It just simply writes raw bytes into the channel, without
/// any regard of the underlying data's structure
pub struct RawBinDataFormat;
impl DataFormat for RawBinDataFormat {
    ///Returns 2, the id of the most basic encoder.
    #[inline]
    fn id() -> u64 {
        ID
    }
    ///Returns "application/octet-stream"
    #[inline]
    fn media_type() -> &'static str {
        MEDIA_TYPE
    }
}

impl<T: AsRef<[u8]>> Encodable<RawBinDataFormat> for T {
    #[inline]
    fn encode_to(&self, _format: &RawBinDataFormat, w: &mut impl Write) -> Result<usize> {
        w.write(self.as_ref())
    }
}

// impl<'a> Encodable<RawBinDataFormat> for &'a [u8] {
//     #[inline]
//     fn encode_to(&self, _format: &RawBinDataFormat, w: &mut impl Write) -> Result<usize> {
//         w.write(self)
//     }
// }

//Will see if this is need it or just a basic nice to have
//use std::io::Read;
// impl<'a> Encodable<RawBinDataFormat> for &'a Read {
//     fn encode_to(&mut self, _format: &RawBinDataFormat, w: &mut impl Write) {
//         std::io::copy(&mut self, w).unwrap();
//     }
// }

#[cfg(test)]<commit_msg>Source code cleanup.<commit_after>use crate::codecs::DataFormat;
use crate::codecs::Encodable;
use std::io::Result;
use std::io::Write;

const ID: u64 = 2;
const MEDIA_TYPE: &str = "application/octet-stream";

/// The most basic data format. It just simply writes raw bytes into the channel, without
/// any regard of the underlying data's structure
pub struct RawBinDataFormat;
impl DataFormat for RawBinDataFormat {
    ///Returns 2, the id of the most basic encoder.
    #[inline]
    fn id() -> u64 {
        ID
    }
    ///Returns "application/octet-stream"
    #[inline]
    fn media_type() -> &'static str {
        MEDIA_TYPE
    }
}

impl<T: AsRef<[u8]>> Encodable<RawBinDataFormat> for T {
    #[inline]
    fn encode_to(&self, _format: &RawBinDataFormat, w: &mut impl Write) -> Result<usize> {
        w.write(self.as_ref())
    }
}

#[cfg(test)]
{'fraction_non_alphanumeric': 0.15125284738041003, 'fraction_numerical': 0.00683371298405467, 'mean_word_length': 3.1433962264150943, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6177955', 'n_tokens_mistral': 761, 'n_tokens_neox': 756, 'n_words': 218}