<commit_before>use std::error::Error;
use std::fmt::{Formatter, Result as FmtResult};

use rustc_serialize::base64::{FromBase64, STANDARD, ToBase64};
use serde::{Deserializer, Serializer};
use serde::de::{Error as SerdeError, Visitor};

pub trait SerdeBlob<'de>: Sized {
    fn deserialize_blob<D>(deserializer: D) -> Result<Self, D::Error> where D: Deserializer<'de>;

    fn serialize_blob<S>(&self, serializer: S) -> Result<S::Ok, S::Error> where S: Serializer;
}

struct BlobVisitor;

impl <'de> Visitor<'de> for BlobVisitor {
    type Value = Vec<u8>;

    fn expecting(&self, formatter: &mut Formatter) -> FmtResult {
        write!(formatter, "a blob")
    }

    fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
        where E: SerdeError {
        v.from_base64().map_err(|err| SerdeError::custom(err.description()))
    }
}

impl <'de> SerdeBlob<'de> for Vec<u8> {
    fn deserialize_blob<D>(deserializer: D) -> Result<Vec<u8>, D::Error>
        where D: Deserializer<'de> {
        deserializer.deserialize_any(BlobVisitor)
    }

    fn serialize_blob<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
        where S: Serializer {
        serializer.serialize_str(&self.as_slice().to_base64(STANDARD))
    }
}

struct OptionalBlobVisitor;

impl <'de> Visitor<'de> for OptionalBlobVisitor {
    type Value = Option<Vec<u8>>;

    fn expecting(&self, formatter: &mut Formatter) -> FmtResult {
        write!(formatter, "a blob")
    }

    fn visit_none<E>(self) -> Result<Self::Value, E>
        where E: SerdeError {
        Ok(None)
    }

    fn visit_some<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
        where D: Deserializer<'de> {
        Ok(Some(try!(SerdeBlob::deserialize_blob(deserializer))))
    }
}

impl <'de> SerdeBlob<'de> for Option<Vec<u8>> {
    fn deserialize_blob<D>(deserializer: D) -> Result<Option<Vec<u8>>, D::Error>
        where D: Deserializer<'de> {
        deserializer.deserialize_option(OptionalBlobVisitor)
    }

    fn serialize_blob<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
        where S: Serializer {
        match *self {
            Some(ref vec) => serializer.serialize_some(vec),
            None => serializer.serialize_none(),
        }
    }
}<commit_msg>Fix serialization of optional blobs.<commit_after>use std::error::Error;
use std::fmt::{Formatter, Result as FmtResult};

use rustc_serialize::base64::{FromBase64, STANDARD, ToBase64};
use serde::{Deserializer, Serializer};
use serde::de::{Error as SerdeError, Visitor};

pub trait SerdeBlob<'de>: Sized {
    fn deserialize_blob<D>(deserializer: D) -> Result<Self, D::Error> where D: Deserializer<'de>;

    fn serialize_blob<S>(&self, serializer: S) -> Result<S::Ok, S::Error> where S: Serializer;
}

struct BlobVisitor;

impl <'de> Visitor<'de> for BlobVisitor {
    type Value = Vec<u8>;

    fn expecting(&self, formatter: &mut Formatter) -> FmtResult {
        write!(formatter, "a blob")
    }

    fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
        where E: SerdeError {
        v.from_base64().map_err(|err| SerdeError::custom(err.description()))
    }
}

impl <'de> SerdeBlob<'de> for Vec<u8> {
    fn deserialize_blob<D>(deserializer: D) -> Result<Vec<u8>, D::Error>
        where D: Deserializer<'de> {
        deserializer.deserialize_any(BlobVisitor)
    }

    fn serialize_blob<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
        where S: Serializer {
        serializer.serialize_str(&self.as_slice().to_base64(STANDARD))
    }
}

struct OptionalBlobVisitor;

impl <'de> Visitor<'de> for OptionalBlobVisitor {
    type Value = Option<Vec<u8>>;

    fn expecting(&self, formatter: &mut Formatter) -> FmtResult {
        write!(formatter, "a blob")
    }

    fn visit_none<E>(self) -> Result<Self::Value, E>
        where E: SerdeError {
        Ok(None)
    }

    fn visit_some<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
        where D: Deserializer<'de> {
        Ok(Some(try!(SerdeBlob::deserialize_blob(deserializer))))
    }
}

impl <'de> SerdeBlob<'de> for Option<Vec<u8>> {
    fn deserialize_blob<D>(deserializer: D) -> Result<Option<Vec<u8>>, D::Error>
        where D: Deserializer<'de> {
        deserializer.deserialize_option(OptionalBlobVisitor)
    }

    fn serialize_blob<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
        where S: Serializer {
        match *self {
            Some(ref vec) => SerdeBlob::serialize_blob(vec, serializer),
            None => serializer.serialize_none(),
        }
    }
}
{'fraction_non_alphanumeric': 0.1570138735961242, 'fraction_numerical': 0.0070469059678484915, 'mean_word_length': 3.606490872210953, 'pattern_counts': {'":': 0, '<': 83, '<?xml version=': 0, '>': 109, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1534812', 'n_tokens_mistral': 1583, 'n_tokens_neox': 1581, 'n_words': 327}