<commit_before>use sass::color_value::ColorValue;
use sass::number_value::NumberValue;
use sass::value_part::ValuePart;
use sass::op::Op;

use error::Result;
use tokenizer_utils::*;

use std::borrow::Cow::Borrowed;

#[derive(Debug)]
pub struct ValueTokenizer<'a> {
    toker: Toker<'a>,
}

impl<'a> ValueTokenizer<'a> {
    pub fn new(inner_str: &'a str) -> ValueTokenizer<'a> {
        ValueTokenizer {
            toker: Toker {
                inner_str: &inner_str,
                bytes: &inner_str.as_bytes(),
                offset: 0,
            },
        }
    }

    fn limit(&self) -> usize {
        self.toker.limit()
    }

    pub fn parse(&mut self) -> Result<Option<ValuePart<'a>>> {
        self.toker.skip_leading_whitespace();

        let start = self.toker.offset;
        let mut i = self.toker.offset;

        let ret = if start == self.limit() {
            None
        } else if is_operator(self.toker.bytes[start]) {
            self.toker.offset = start + 1;
            Some(ValuePart::Operator(self.toker.inner_str[start..start + 1].parse().unwrap_or(Op::Plus)))
        } else if is_number(self.toker.bytes[start]) {
            i += self.toker.scan_while_or_end(i, is_number);
            self.toker.offset = i;
            if i < self.limit() && valid_unit_char(self.toker.bytes[i]) {
                let unit_start = i;
                i += self.toker.scan_while_or_end(i, valid_unit_char);
                self.toker.offset = i;
                Some(ValuePart::Number(NumberValue::with_units(
                    self.toker.inner_str[start..unit_start].parse().unwrap_or(0.0),
                    Borrowed(&self.toker.inner_str[unit_start..i]),
                )))
            }
            else {
                Some(ValuePart::Number(NumberValue::from_scalar(
                    self.toker.inner_str[start..i].parse().unwrap_or(0.0)
                )))
            }
        } else if self.toker.bytes[start] == b'$' {
            i += self.toker.scan_while_or_end(i, isnt_space);
            self.toker.offset = i;
            Some(ValuePart::Variable(Borrowed(&self.toker.inner_str[start..i])))
        } else if self.toker.eat("#").is_ok() {
            i = self.toker.offset;
            i += self.toker.scan_while_or_end(i, valid_hex_char);
            self.toker.offset = i;
            match ColorValue::from_hex(Borrowed(&self.toker.inner_str[start..i])) {
                Ok(v) => Some(ValuePart::Color(v)),
                Err(e) => return Err(e),
            }
        } else if self.toker.eat("rgb(").is_ok() {
            let r = self.eat_color_u8();
            try!(self.toker.eat(","));
            self.toker.skip_leading_whitespace();
            let g = self.eat_color_u8();
            try!(self.toker.eat(","));
            self.toker.skip_leading_whitespace();
            let b = self.eat_color_u8();
            try!(self.toker.eat(")"));

            Some(ValuePart::Color(
                ColorValue {
                    red: r, green: g, blue: b,
                    computed: false,<commit_msg>Differentiate between colors and interpolations<commit_after>use sass::color_value::ColorValue;
use sass::number_value::NumberValue;
use sass::value_part::ValuePart;
use sass::op::Op;

use error::Result;
use tokenizer_utils::*;

use std::borrow::Cow::Borrowed;

#[derive(Debug)]
pub struct ValueTokenizer<'a> {
    toker: Toker<'a>,
}

impl<'a> ValueTokenizer<'a> {
    pub fn new(inner_str: &'a str) -> ValueTokenizer<'a> {
        ValueTokenizer {
            toker: Toker {
                inner_str: &inner_str,
                bytes: &inner_str.as_bytes(),
                offset: 0,
            },
        }
    }

    fn limit(&self) -> usize {
        self.toker.limit()
    }

    pub fn parse(&mut self) -> Result<Option<ValuePart<'a>>> {
        self.toker.skip_leading_whitespace();

        let start = self.toker.offset;
        let mut i = self.toker.offset;

        let ret = if start == self.limit() {
            None
        } else if is_operator(self.toker.bytes[start]) {
            self.toker.offset = start + 1;
            Some(ValuePart::Operator(self.toker.inner_str[start..start + 1].parse().unwrap_or(Op::Plus)))
        } else if is_number(self.toker.bytes[start]) {
            i += self.toker.scan_while_or_end(i, is_number);
            self.toker.offset = i;
            if i < self.limit() && valid_unit_char(self.toker.bytes[i]) {
                let unit_start = i;
                i += self.toker.scan_while_or_end(i, valid_unit_char);
                self.toker.offset = i;
                Some(ValuePart::Number(NumberValue::with_units(
                    self.toker.inner_str[start..unit_start].parse().unwrap_or(0.0),
                    Borrowed(&self.toker.inner_str[unit_start..i]),
                )))
            }
            else {
                Some(ValuePart::Number(NumberValue::from_scalar(
                    self.toker.inner_str[start..i].parse().unwrap_or(0.0)
                )))
            }
        } else if self.toker.bytes[start] == b'$' {
            i += self.toker.scan_while_or_end(i, isnt_space);
            self.toker.offset = i;
            Some(ValuePart::Variable(Borrowed(&self.toker.inner_str[start..i])))
        } else if self.toker.eat("#").is_ok() {
            if self.toker.eat("{").is_ok() {
                i = self.toker.offset;
                let start_interpolation = i;
                i += self.toker.scan_while_or_end(i, (|c| c != b'}' ));
                self.toker.offset = i;
                try!(self.toker.eat("}"));
                Some(ValuePart::String(Borrowed(
                    &self.toker.inner_str[start_interpolation..i]
                )))
            } else {
                i = self.toker.offset;
                i += self.toker.scan_while_or_end(i, valid_hex_char);
                self.toker.offset = i;
                match ColorValue::from_hex(Borrowed(&self.toker.inner_str[start..i])) {
                    Ok(v) => Some(ValuePart::Color(v)),
                    Err(e) => return Err(e),
                }
            }
        } else if self.toker.eat("rgb(").is_ok() {
            let r = self.eat_color_u8();
            try!(self.toker.eat(","));
            self.toker.skip_leading_whitespace();
            let g = self.eat_color_u8();
            try!(self.toker.eat(","));
            self.toker.skip_leading_whitespace();
            let b = self.eat_color_u8();
            try!(self.toker.eat(")"));

            Some(ValuePart::Color(
                ColorValue {
                    red: r, green: g, blue: b,
                    computed: false,
{'fraction_non_alphanumeric': 0.14698360162479315, 'fraction_numerical': 0.0030088761847449977, 'mean_word_length': 1.780426599749059, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1742613', 'n_tokens_mistral': 2177, 'n_tokens_neox': 2126, 'n_words': 317}