<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use std::io::Read;

const LF: u8 = b'\n';
const CR: u8 = b'\r';
const QUOTE: u8 = b'"';
const COMMA: u8 = b',';

// Error messages
pub const UNEXPECTED_EOL: &str = "Unexpected end of line";
pub const UNEXPECTED_CHAR: &str = "Unexpected character after quote";
pub const EXPECTED_LF: &str = "Expected linefeed after carriage return";

enum CSVState {
    Start,
    NonQuotedValue,
    QuotedValue,
    QuoteQuote,
    ExpectLF,
    Error,
}

type CSVResult = Result<CSVState, &'static str>;

trait ParseByte {
    fn parse_byte(&self, byte: u8) -> CSVResult;
}

impl ParseByte for CSVState {
    fn parse_byte(&self, byte: u8) -> CSVResult {
        match (self, byte) {
            (&CSVState::Start, QUOTE) => Ok(CSVState::QuotedValue),
            (&CSVState::Start, COMMA) => Ok(CSVState::Start),
            (&CSVState::Start, _) => Ok(CSVState::NonQuotedValue),

            (&CSVState::NonQuotedValue, COMMA) => Ok(CSVState::Start),
            (&CSVState::NonQuotedValue, LF) => Ok(CSVState::Start),
            (&CSVState::NonQuotedValue, CR) => Ok(CSVState::ExpectLF),
            (&CSVState::NonQuotedValue, _) => Ok(CSVState::NonQuotedValue),

            (&CSVState::QuotedValue, QUOTE) => Ok(CSVState::QuoteQuote),
            (&CSVState::QuotedValue, CR) => Err(UNEXPECTED_EOL),
            (&CSVState::QuotedValue, LF) => Err(UNEXPECTED_EOL),
            (&CSVState::QuotedValue, _) => Ok(CSVState::QuotedValue),

            (&CSVState::QuoteQuote, QUOTE) => Ok(CSVState::QuotedValue),
            (&CSVState::QuoteQuote, COMMA) => Ok(CSVState::Start),
            (&CSVState::QuoteQuote, LF) => Ok(CSVState::Start),
            (&CSVState::QuoteQuote, CR) => Ok(CSVState::ExpectLF),
            (&CSVState::QuoteQuote, _) => Err(UNEXPECTED_CHAR),

            (&CSVState::ExpectLF, LF) => Ok(CSVState::Start),
            (&CSVState::ExpectLF, _) => Err(EXPECTED_LF),

            (&CSVState::Error, LF) => Ok(CSVState::Start),
            (&CSVState::Error, _) => Ok(CSVState::Error),
        }
    }
}
<commit_msg>Rust 1.26 will take care of references in matches<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

use std::io::Read;

const LF: u8 = b'\n';
const CR: u8 = b'\r';
const QUOTE: u8 = b'"';
const COMMA: u8 = b',';

// Error messages
pub const UNEXPECTED_EOL: &str = "Unexpected end of line";
pub const UNEXPECTED_CHAR: &str = "Unexpected character after quote";
pub const EXPECTED_LF: &str = "Expected linefeed after carriage return";

enum CSVState {
    Start,
    NonQuotedValue,
    QuotedValue,
    QuoteQuote,
    ExpectLF,
    Error,
}

type CSVResult = Result<CSVState, &'static str>;

trait ParseByte {
    fn parse_byte(&self, byte: u8) -> CSVResult;
}

impl ParseByte for CSVState {
    fn parse_byte(&self, byte: u8) -> CSVResult {
        match (self, byte) {
            (CSVState::Start, QUOTE) => Ok(CSVState::QuotedValue),
            (CSVState::Start, COMMA) => Ok(CSVState::Start),
            (CSVState::Start, _) => Ok(CSVState::NonQuotedValue),

            (CSVState::NonQuotedValue, COMMA) => Ok(CSVState::Start),
            (CSVState::NonQuotedValue, LF) => Ok(CSVState::Start),
            (CSVState::NonQuotedValue, CR) => Ok(CSVState::ExpectLF),
            (CSVState::NonQuotedValue, _) => Ok(CSVState::NonQuotedValue),

            (CSVState::QuotedValue, QUOTE) => Ok(CSVState::QuoteQuote),
            (CSVState::QuotedValue, CR) => Err(UNEXPECTED_EOL),
            (CSVState::QuotedValue, LF) => Err(UNEXPECTED_EOL),
            (CSVState::QuotedValue, _) => Ok(CSVState::QuotedValue),

            (CSVState::QuoteQuote, QUOTE) => Ok(CSVState::QuotedValue),
            (CSVState::QuoteQuote, COMMA) => Ok(CSVState::Start),
            (CSVState::QuoteQuote, LF) => Ok(CSVState::Start),
            (CSVState::QuoteQuote, CR) => Ok(CSVState::ExpectLF),
            (CSVState::QuoteQuote, _) => Err(UNEXPECTED_CHAR),

            (CSVState::ExpectLF, LF) => Ok(CSVState::Start),
            (CSVState::ExpectLF, _) => Err(EXPECTED_LF),

            (CSVState::Error, LF) => Ok(CSVState::Start),
            (CSVState::Error, _) => Ok(CSVState::Error),
        }
    }
}

{'fraction_non_alphanumeric': 0.15709235528060098, 'fraction_numerical': 0.005081749889527177, 'mean_word_length': 3.1456043956043955, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '556024', 'n_tokens_mistral': 1626, 'n_tokens_neox': 1539, 'n_words': 340}