<commit_before>/*!
 * Poorly categorised functions for reading things used in multiple places.
 *
 * (That is, typically, more than one of request-or-response reading-or-writing.)
 *
 * TODO: refactor all this to store things in more usefully categorised places.
 */
use std::num::{Zero, cast};
use std::io::{IoError, IoResult, OtherIoError};
#[cfg(test)]
use std::io::MemReader;

// XXX: IoError ain't a good representation of this.
fn bad_input() -> IoError {
    IoError {
        kind: OtherIoError,
        desc: "invalid number",
        detail: None,
    }
}

static ASCII_ZERO: u8 = b'0';
static ASCII_NINE: u8 = b'9';
static ASCII_LOWER_A: u8 = b'a';
static ASCII_LOWER_F: u8 = b'f';
static ASCII_UPPER_A: u8 = b'A';
static ASCII_UPPER_F: u8 = b'F';

/**
 * Read a positive decimal integer from the given reader.
 *
 * # Arguments
 *
 * - `reader` - the reader to read the decimal digits from (and whatever byte comes next)
 * - `expected_end` - this function is called with the byte that follows the last decimal digit
 *   and should return `true` if that byte is what is expected to follow the number.
 *
 * # Return value
 *
 * - `None`, if the number overflows;
 * - `None`, if there is a leading zero;
 * - `None`, if all the digits are read and the `expected_end` function is false
 *
 * Should everything work as designed (i.e. none of these conditions occur) a `Some` is returned.
 */
pub fn read_decimal<R: Reader, N: Unsigned + NumCast + PartialOrd + CheckedMul + CheckedAdd>
                   (reader: &mut R, expected_end: |u8| -> bool)
                   -> IoResult<N> {
    // Here and in `read_hexadecimal` there is the possibility of infinite sequence of zeroes. The
    // spec allows this, but it may not be a good thing to allow. It's not a particularly good
    // attack surface, though, because of the low return.
    let mut n: N = Zero::zero();
    let mut got_content = false;
    let ten: N = cast(10u32).unwrap();
    loop {
        n = match reader.read_byte() {
            Ok(b@ASCII_ZERO..ASCII_NINE) => {
                // Written sanely, this is: n * 10 + (b - '0'), but we avoid
                // (semantically unsound) overflow by using checked operations.
                // There is no need in the b - '0' part as it is safe.
                match n.checked_mul(&ten).and_then(
                        |n| n.checked_add(&cast(b - ASCII_ZERO).unwrap())) {
                    Some(new_n) => new_n,
                    None => return Err(bad_input()),  // overflow
                }
            },
            Ok(b) if got_content && expected_end(b) => return Ok(n),
            Ok(_) => return Err(bad_input()),  // not a valid number
            Err(err) => return Err(err),  // I/O error
        };
        got_content = true;
    }
}

/**
 * Read a positive hexadecimal integer from the given reader.
 *
 * # Arguments
 *
 * - `reader` - the reader to read the hexadecimal digits from (and whatever byte comes next)
 * - `expected_end` - this function is called with the byte that follows the last hexadecimal digit
 *   and should return `true` if that byte is what is expected to follow the number.
 *
 * # Return value
 *
 * - `None`, if the number overflows;
 * - `None`, if there is a leading zero;
 * - `None`, if all the digits are read and the `expected_end` function is false
 *
 * Should everything work as designed (i.e. none of these conditions occur) a `Some` is returned.
 */
pub fn read_hexadecimal<R: Reader, N: Unsigned + NumCast + PartialOrd + CheckedMul + CheckedAdd>
                       (reader: &mut R, expected_end: |u8| -> bool)
                       -> IoResult<N> {
    let mut n: N = Zero::zero();
    let mut got_content = false;
    let sixteen: N = cast(16u32).unwrap();
    loop {
        n = match reader.read_byte() {
            Ok(b@ASCII_ZERO..ASCII_NINE) => {
                match n.checked_mul(&sixteen).and_then(
                        |n| n.checked_add(&cast(b - ASCII_ZERO).unwrap())) {
                    Some(new_n) => new_n,
                    None => return Err(bad_input()),  // overflow
                }
            },
            Ok(b@ASCII_LOWER_A..ASCII_LOWER_F) => {
                match n.checked_mul(&sixteen).and_then(
                        |n| n.checked_add(&cast(b - ASCII_LOWER_A + 10).unwrap())) {
                    Some(new_n) => new_n,
                    None => return Err(bad_input()),  // overflow
                }
            },
            Ok(b@ASCII_UPPER_A..ASCII_UPPER_F) => {
                match n.checked_mul(&sixteen).and_then(
                        |n| n.checked_add(&cast(b - ASCII_UPPER_A + 10).unwrap())) {
                    Some(new_n) => new_n,
                    None => return Err(bad_input()),  // overflow
                }<commit_msg>Change ranges from A..B to A...B<commit_after>/*!
 * Poorly categorised functions for reading things used in multiple places.
 *
 * (That is, typically, more than one of request-or-response reading-or-writing.)
 *
 * TODO: refactor all this to store things in more usefully categorised places.
 */
use std::num::{Zero, cast};
use std::io::{IoError, IoResult, OtherIoError};
#[cfg(test)]
use std::io::MemReader;

// XXX: IoError ain't a good representation of this.
fn bad_input() -> IoError {
    IoError {
        kind: OtherIoError,
        desc: "invalid number",
        detail: None,
    }
}

static ASCII_ZERO: u8 = b'0';
static ASCII_NINE: u8 = b'9';
static ASCII_LOWER_A: u8 = b'a';
static ASCII_LOWER_F: u8 = b'f';
static ASCII_UPPER_A: u8 = b'A';
static ASCII_UPPER_F: u8 = b'F';

/**
 * Read a positive decimal integer from the given reader.
 *
 * # Arguments
 *
 * - `reader` - the reader to read the decimal digits from (and whatever byte comes next)
 * - `expected_end` - this function is called with the byte that follows the last decimal digit
 *   and should return `true` if that byte is what is expected to follow the number.
 *
 * # Return value
 *
 * - `None`, if the number overflows;
 * - `None`, if there is a leading zero;
 * - `None`, if all the digits are read and the `expected_end` function is false
 *
 * Should everything work as designed (i.e. none of these conditions occur) a `Some` is returned.
 */
pub fn read_decimal<R: Reader, N: Unsigned + NumCast + PartialOrd + CheckedMul + CheckedAdd>
                   (reader: &mut R, expected_end: |u8| -> bool)
                   -> IoResult<N> {
    // Here and in `read_hexadecimal` there is the possibility of infinite sequence of zeroes. The
    // spec allows this, but it may not be a good thing to allow. It's not a particularly good
    // attack surface, though, because of the low return.
    let mut n: N = Zero::zero();
    let mut got_content = false;
    let ten: N = cast(10u32).unwrap();
    loop {
        n = match reader.read_byte() {
            Ok(b@ASCII_ZERO...ASCII_NINE) => {
                // Written sanely, this is: n * 10 + (b - '0'), but we avoid
                // (semantically unsound) overflow by using checked operations.
                // There is no need in the b - '0' part as it is safe.
                match n.checked_mul(&ten).and_then(
                        |n| n.checked_add(&cast(b - ASCII_ZERO).unwrap())) {
                    Some(new_n) => new_n,
                    None => return Err(bad_input()),  // overflow
                }
            },
            Ok(b) if got_content && expected_end(b) => return Ok(n),
            Ok(_) => return Err(bad_input()),  // not a valid number
            Err(err) => return Err(err),  // I/O error
        };
        got_content = true;
    }
}

/**
 * Read a positive hexadecimal integer from the given reader.
 *
 * # Arguments
 *
 * - `reader` - the reader to read the hexadecimal digits from (and whatever byte comes next)
 * - `expected_end` - this function is called with the byte that follows the last hexadecimal digit
 *   and should return `true` if that byte is what is expected to follow the number.
 *
 * # Return value
 *
 * - `None`, if the number overflows;
 * - `None`, if there is a leading zero;
 * - `None`, if all the digits are read and the `expected_end` function is false
 *
 * Should everything work as designed (i.e. none of these conditions occur) a `Some` is returned.
 */
pub fn read_hexadecimal<R: Reader, N: Unsigned + NumCast + PartialOrd + CheckedMul + CheckedAdd>
                       (reader: &mut R, expected_end: |u8| -> bool)
                       -> IoResult<N> {
    let mut n: N = Zero::zero();
    let mut got_content = false;
    let sixteen: N = cast(16u32).unwrap();
    loop {
        n = match reader.read_byte() {
            Ok(b@ASCII_ZERO...ASCII_NINE) => {
                match n.checked_mul(&sixteen).and_then(
                        |n| n.checked_add(&cast(b - ASCII_ZERO).unwrap())) {
                    Some(new_n) => new_n,
                    None => return Err(bad_input()),  // overflow
                }
            },
            Ok(b@ASCII_LOWER_A...ASCII_LOWER_F) => {
                match n.checked_mul(&sixteen).and_then(
                        |n| n.checked_add(&cast(b - ASCII_LOWER_A + 10).unwrap())) {
                    Some(new_n) => new_n,
                    None => return Err(bad_input()),  // overflow
                }
            },
            Ok(b@ASCII_UPPER_A...ASCII_UPPER_F) => {
                match n.checked_mul(&sixteen).and_then(
                        |n| n.checked_add(&cast(b - ASCII_UPPER_A + 10).unwrap())) {
                    Some(new_n) => new_n,
                    None => return Err(bad_input()),  // overflow
                }
{'fraction_non_alphanumeric': 0.1198334200937012, 'fraction_numerical': 0.005413846954711088, 'mean_word_length': 2.093719806763285, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6770730', 'n_tokens_mistral': 2994, 'n_tokens_neox': 2892, 'n_words': 973}