<commit_before>use serde;
use serde_json as json;

use errors::*;
use line::Line;

#[derive(Deserialize, Debug, PartialEq)]
pub enum OperationType {
    Copy_,
    Skip,
    Invalidate,
    Update,
    Insert,
}

#[derive(Deserialize, Debug, PartialEq)]
pub struct Operation {
    #[serde(rename="op")]
    #[serde(deserialize_with="deserialize_operation_type")]
    pub operation_type: OperationType,
    #[serde(rename="n")]
    pub nb_lines: u64,
    pub lines: Option<Vec<Line>>,
}

fn deserialize_operation_type<'de, D>(de: D) -> ::std::result::Result<OperationType, D::Error>
    where D: serde::Deserializer<'de>
{
    let value: json::Value = try!(serde::Deserialize::deserialize(de));
    match value {
        json::Value::String(ref s) if &*s == "copy" => Ok(OperationType::Copy_),
        json::Value::String(ref s) if &*s == "skip" => Ok(OperationType::Skip),
        json::Value::String(ref s) if &*s == "invalidate" => Ok(OperationType::Invalidate),
        json::Value::String(ref s) if &*s == "update" => Ok(OperationType::Update),
        json::Value::String(ref s) if &*s == "ins" => Ok(OperationType::Insert),
        _ => Err(serde::de::Error::custom("Unexpected value for operation type")),
    }
}

impl Operation {
    pub fn apply(&self, old_lines: &[Line], old_ix: u64, new_lines: &mut Vec<Line>) -> Result<u64> {
        // FIXME: this method panics if we don't check old_lines indices access.
        // we should check old_lines length and return an error when trying to access an out of
        // bound index.
        match self.operation_type {
            OperationType::Copy_ => {
                let new_ix = old_ix + self.nb_lines;
                debug!("copying line {} to {}", old_ix, new_ix);

                for i in old_ix..new_ix {
                    new_lines.push(old_lines[i as usize].clone());
                }
                Ok(new_ix)
            }
            OperationType::Skip => {
                debug!("skipping {} lines", self.nb_lines);
                Ok(old_ix + self.nb_lines)
            }
            OperationType::Invalidate => {
                let new_ix = old_ix + self.nb_lines;
                debug!("invalidating lines {} to {}", old_ix, new_ix);

                for _ in 0..self.nb_lines {
                    new_lines.push(Line::invalid());<commit_msg>Adds issue number to FIXME and reformats comment<commit_after>use serde;
use serde_json as json;

use errors::*;
use line::Line;

#[derive(Deserialize, Debug, PartialEq)]
pub enum OperationType {
    Copy_,
    Skip,
    Invalidate,
    Update,
    Insert,
}

#[derive(Deserialize, Debug, PartialEq)]
pub struct Operation {
    #[serde(rename="op")]
    #[serde(deserialize_with="deserialize_operation_type")]
    pub operation_type: OperationType,
    #[serde(rename="n")]
    pub nb_lines: u64,
    pub lines: Option<Vec<Line>>,
}

fn deserialize_operation_type<'de, D>(de: D) -> ::std::result::Result<OperationType, D::Error>
    where D: serde::Deserializer<'de>
{
    let value: json::Value = try!(serde::Deserialize::deserialize(de));
    match value {
        json::Value::String(ref s) if &*s == "copy" => Ok(OperationType::Copy_),
        json::Value::String(ref s) if &*s == "skip" => Ok(OperationType::Skip),
        json::Value::String(ref s) if &*s == "invalidate" => Ok(OperationType::Invalidate),
        json::Value::String(ref s) if &*s == "update" => Ok(OperationType::Update),
        json::Value::String(ref s) if &*s == "ins" => Ok(OperationType::Insert),
        _ => Err(serde::de::Error::custom("Unexpected value for operation type")),
    }
}

impl Operation {
    pub fn apply(&self, old_lines: &[Line], old_ix: u64, new_lines: &mut Vec<Line>) -> Result<u64> {
        // FIXME(#34): this method panics if we don't check old_lines indices
        // access. We should check old_lines length and return an error when
        // trying to access an out of bound index.
        match self.operation_type {
            OperationType::Copy_ => {
                let new_ix = old_ix + self.nb_lines;
                debug!("copying line {} to {}", old_ix, new_ix);

                for i in old_ix..new_ix {
                    new_lines.push(old_lines[i as usize].clone());
                }
                Ok(new_ix)
            }
            OperationType::Skip => {
                debug!("skipping {} lines", self.nb_lines);
                Ok(old_ix + self.nb_lines)
            }
            OperationType::Invalidate => {
                let new_ix = old_ix + self.nb_lines;
                debug!("invalidating lines {} to {}", old_ix, new_ix);

                for _ in 0..self.nb_lines {
                    new_lines.push(Line::invalid());
{'fraction_non_alphanumeric': 0.14774659863945577, 'fraction_numerical': 0.003401360544217687, 'mean_word_length': 2.569802731411229, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6565090', 'n_tokens_mistral': 1448, 'n_tokens_neox': 1460, 'n_words': 348}