<commit_before>//! Types and utilities for working with headers in HTTP requests and responses.
//!
//! This HTTP system is Special in that it uses lots of strong typing for its header system. All
//! known HTTP headers are type checked, rather than being dealt with as strings all the time. Only
//! unknown headers are stored in a map in the traditional way.

use std::rt::io::{Reader, Writer};
use std::rt::io::extensions::ReaderUtil;
use extra::time::{Tm, strptime};
use extra::url::Url;
use rfc2616::{is_token_item, is_separator, CR, LF, SP, HT, COLON, DOUBLE_QUOTE, BACKSLASH};
use method::Method;

use self::serialization_utils::{normalise_header_name};

pub enum HeaderLineErr { EndOfFile, EndOfHeaders, MalformedHeaderValue, MalformedHeaderSyntax }

pub mod test_utils;
pub mod serialization_utils;

/* TODO: ensure we've got all standard HTTP headers, not just those in RFC 2616.

From https://en.wikipedia.org/wiki/List_of_HTTP_headers:

- Known missing headers:

  - Access-Control-Allow-Origin
  - Content-Disposition
  - Link
  - P3P
  - Refresh
  - Set-Cookie
  - Status
  - Strict-Transport-Security

- Common non-standard headers:

  - X-Frame-Options
  - X-XSS-Protection
  - Content-Security-Policy, X-Content-Security-Policy, X-WebKit-CSP
  - X-Content-Type-Options
  - X-Powered-By
  - X-UA-Compatible

Also go through http://www.iana.org/assignments/message-headers/message-headers.xhtml which should
be the more canonical source.

*/

//pub mod accept;
//pub mod accept_charset;
//pub mod accept_encoding;
//pub mod accept_language;
pub mod accept_ranges;
//pub mod cache_control;
pub mod connection;
//pub mod content_encoding;
//pub mod content_range;
pub mod content_type;
pub mod etag;
pub mod host;
pub mod transfer_encoding;

pub type DeltaSeconds = u64;

#[deriving(Clone, DeepClone, Eq)]
pub enum ConsumeCommaLWSResult {
    CommaConsumed,
    EndOfValue,
    ErrCommaNotFound,
}

pub trait HeaderEnum {
    fn header_name(&self) -> ~str;
    fn header_value(&self) -> ~str;
    fn write_header<T: Writer>(&self, writer: &mut T);

    // FIXME: this method combination is temporary, to be merged with an efficient parser like that
    // of the request line. Also refactor to remove the need to return the next byte too.
    /// Return values:
    ///
    /// - Header
    /// - Next byte (consumed out of linear white space checking necessity)
    ///
    /// (Err, Option) means EOF/EOH/Malformed.
    /// (Ok, None) is impossible.
    /// (Ok, Some) means you have a valid header.
    //REMOVED BECAUSE OF ICE:
    //fn from_stream<T: Reader>(reader: &mut T) -> (Result<Self, HeaderLineErr>, Option<u8>) { ... }

    fn value_from_stream<T: Reader>(name: ~str, input: &mut HeaderValueByteIterator<T>)
        -> Option<Self>;
}

/// Shifted out of being a default method to fix an ICE (not yet reported, TODO)
pub fn header_enum_from_stream<R: Reader, E: HeaderEnum>(reader: &mut R)
        -> (Result<E, HeaderLineErr>, Option<u8>) {
    enum State { Start, ReadingName, NameFinished, GotCR }
    let mut state = Start;
    let mut header_name = ~"";
    loop {
        state = match (state, reader.read_byte()) {
            (Start, Some(b)) | (ReadingName, Some(b)) if is_token_item(b) => {
                header_name.push_char(b as char);
                ReadingName
            },
            // TODO: check up on the rules for a line like "Name : value". Full LWS?
            (Start, Some(b)) if b == CR => GotCR,
            (Start, Some(b)) | (GotCR, Some(b)) if b == LF => {
                return (Err(EndOfHeaders), None);
            },
            (_, Some(b)) if b == SP => NameFinished,
            (_, Some(b)) if b == COLON => break,
            (_, Some(_)) => return (Err(MalformedHeaderSyntax), None),
            (_, None) => return (Err(EndOfFile), None),
        }
    }
    let mut iter = HeaderValueByteIterator::new(reader);
    let header = HeaderEnum::value_from_stream(normalise_header_name(header_name), &mut iter);
    // Ensure that the entire header line is consumed (don't want to mess up next header!)
    for _ in iter { }
    match header {
        Some(h) => (Ok(h), iter.next_byte),
        None => {
            debug!("malformed header value for %s", header_name);
            // Alas, I can't tell you what the value actually was... TODO: improve that situation
            (Err(MalformedHeaderValue), iter.next_byte)
        },
    }
}

#[deriving(Eq)]
enum HeaderValueByteIteratorState {
    Normal,  // Anything other than the rest.
    InsideQuotedString,  // no LWS compacting here. TODO: check spec on CR LF SP in quoted-string.
    InsideQuotedStringEscape,  // don't let a " close quoted-string if it comes next
    CompactingLWS,  // Got at least one linear white space character; turning it into just one SP
    GotCR,  // Last character was CR
    GotCRLF,  // Last two characters were CR LF
    Finished,  // Finished, so next() should always return ``None`` immediately (no side effects)
}

/// An iterator over the bytes of a header value.
/// This ensures one cannot read past the end of a header mistakenly and that linear white space is
/// handled correctly so that nothing else needs to worry about it. Any linear whitespace (multiple
/// spaces outside of a quoted-string) is compacted into a single SP.
pub struct HeaderValueByteIterator<'self, R> {
    reader: &'self mut R,

    /// This field serves two purposes. *During* iteration, it will typically be ``None``, but
    /// certain cases will cause it to be a ``Some``, meaning that the next ``next()`` call will
    /// return that value rather than reading a new byte. At the *end* of iteration (after
    /// ``next()`` has returned ``None``), it will be the extra byte which it has had to consume
    /// from the stream because of the possibility of linear white space of the form ``CR LF SP``.
    /// It is guaranteed that if ``self.state == Finished`` this will be a ``Some``.
    next_byte: Option<u8>,

    at_start: bool,
    state: HeaderValueByteIteratorState,
}

impl<'self, R: Reader> HeaderValueByteIterator<'self, R> {

    fn new(reader: &'self mut R) -> HeaderValueByteIterator<'self, R> {
        HeaderValueByteIterator {
            reader: reader,
            next_byte: None,
            at_start: true,
            state: Normal,
        }
    }

    /// Check that the entire header value has been consumed.
    ///
    /// Be cautious using this function as it is destructive, losing a character in the case where
    /// the value has not been entirely consumed.
    ///
    /// This should only be called when finished with a value and ensuring that there aren't
    /// unexpected characters
    ///
    /// Suggested usage is in a ``from_stream`` method::
    ///
    ///     if reader.verify_consumed() {
    ///         Some(header)
    ///     } else {
    ///         None
    ///     }
    #[inline]
    fn verify_consumed(&mut self) -> bool {
        self.next() == None<commit_msg>Make verify_consumed consume trailing LWs.<commit_after>//! Types and utilities for working with headers in HTTP requests and responses.
//!
//! This HTTP system is Special in that it uses lots of strong typing for its header system. All
//! known HTTP headers are type checked, rather than being dealt with as strings all the time. Only
//! unknown headers are stored in a map in the traditional way.

use std::rt::io::{Reader, Writer};
use std::rt::io::extensions::ReaderUtil;
use extra::time::{Tm, strptime};
use extra::url::Url;
use rfc2616::{is_token_item, is_separator, CR, LF, SP, HT, COLON, DOUBLE_QUOTE, BACKSLASH};
use method::Method;

use self::serialization_utils::{normalise_header_name};

pub enum HeaderLineErr { EndOfFile, EndOfHeaders, MalformedHeaderValue, MalformedHeaderSyntax }

pub mod test_utils;
pub mod serialization_utils;

/* TODO: ensure we've got all standard HTTP headers, not just those in RFC 2616.

From https://en.wikipedia.org/wiki/List_of_HTTP_headers:

- Known missing headers:

  - Access-Control-Allow-Origin
  - Content-Disposition
  - Link
  - P3P
  - Refresh
  - Set-Cookie
  - Status
  - Strict-Transport-Security

- Common non-standard headers:

  - X-Frame-Options
  - X-XSS-Protection
  - Content-Security-Policy, X-Content-Security-Policy, X-WebKit-CSP
  - X-Content-Type-Options
  - X-Powered-By
  - X-UA-Compatible

Also go through http://www.iana.org/assignments/message-headers/message-headers.xhtml which should
be the more canonical source.

*/

//pub mod accept;
//pub mod accept_charset;
//pub mod accept_encoding;
//pub mod accept_language;
pub mod accept_ranges;
//pub mod cache_control;
pub mod connection;
//pub mod content_encoding;
//pub mod content_range;
pub mod content_type;
pub mod etag;
pub mod host;
pub mod transfer_encoding;

pub type DeltaSeconds = u64;

#[deriving(Clone, DeepClone, Eq)]
pub enum ConsumeCommaLWSResult {
    CommaConsumed,
    EndOfValue,
    ErrCommaNotFound,
}

pub trait HeaderEnum {
    fn header_name(&self) -> ~str;
    fn header_value(&self) -> ~str;
    fn write_header<T: Writer>(&self, writer: &mut T);

    // FIXME: this method combination is temporary, to be merged with an efficient parser like that
    // of the request line. Also refactor to remove the need to return the next byte too.
    /// Return values:
    ///
    /// - Header
    /// - Next byte (consumed out of linear white space checking necessity)
    ///
    /// (Err, Option) means EOF/EOH/Malformed.
    /// (Ok, None) is impossible.
    /// (Ok, Some) means you have a valid header.
    //REMOVED BECAUSE OF ICE:
    //fn from_stream<T: Reader>(reader: &mut T) -> (Result<Self, HeaderLineErr>, Option<u8>) { ... }

    fn value_from_stream<T: Reader>(name: ~str, input: &mut HeaderValueByteIterator<T>)
        -> Option<Self>;
}

/// Shifted out of being a default method to fix an ICE (not yet reported, TODO)
pub fn header_enum_from_stream<R: Reader, E: HeaderEnum>(reader: &mut R)
        -> (Result<E, HeaderLineErr>, Option<u8>) {
    enum State { Start, ReadingName, NameFinished, GotCR }
    let mut state = Start;
    let mut header_name = ~"";
    loop {
        state = match (state, reader.read_byte()) {
            (Start, Some(b)) | (ReadingName, Some(b)) if is_token_item(b) => {
                header_name.push_char(b as char);
                ReadingName
            },
            // TODO: check up on the rules for a line like "Name : value". Full LWS?
            (Start, Some(b)) if b == CR => GotCR,
            (Start, Some(b)) | (GotCR, Some(b)) if b == LF => {
                return (Err(EndOfHeaders), None);
            },
            (_, Some(b)) if b == SP => NameFinished,
            (_, Some(b)) if b == COLON => break,
            (_, Some(_)) => return (Err(MalformedHeaderSyntax), None),
            (_, None) => return (Err(EndOfFile), None),
        }
    }
    let mut iter = HeaderValueByteIterator::new(reader);
    let header = HeaderEnum::value_from_stream(normalise_header_name(header_name), &mut iter);
    // Ensure that the entire header line is consumed (don't want to mess up next header!)
    for _ in iter { }
    match header {
        Some(h) => (Ok(h), iter.next_byte),
        None => {
            debug!("malformed header value for %s", header_name);
            // Alas, I can't tell you what the value actually was... TODO: improve that situation
            (Err(MalformedHeaderValue), iter.next_byte)
        },
    }
}

#[deriving(Eq)]
enum HeaderValueByteIteratorState {
    Normal,  // Anything other than the rest.
    InsideQuotedString,  // no LWS compacting here. TODO: check spec on CR LF SP in quoted-string.
    InsideQuotedStringEscape,  // don't let a " close quoted-string if it comes next
    CompactingLWS,  // Got at least one linear white space character; turning it into just one SP
    GotCR,  // Last character was CR
    GotCRLF,  // Last two characters were CR LF
    Finished,  // Finished, so next() should always return ``None`` immediately (no side effects)
}

/// An iterator over the bytes of a header value.
/// This ensures one cannot read past the end of a header mistakenly and that linear white space is
/// handled correctly so that nothing else needs to worry about it. Any linear whitespace (multiple
/// spaces outside of a quoted-string) is compacted into a single SP.
pub struct HeaderValueByteIterator<'self, R> {
    reader: &'self mut R,

    /// This field serves two purposes. *During* iteration, it will typically be ``None``, but
    /// certain cases will cause it to be a ``Some``, meaning that the next ``next()`` call will
    /// return that value rather than reading a new byte. At the *end* of iteration (after
    /// ``next()`` has returned ``None``), it will be the extra byte which it has had to consume
    /// from the stream because of the possibility of linear white space of the form ``CR LF SP``.
    /// It is guaranteed that if ``self.state == Finished`` this will be a ``Some``.
    next_byte: Option<u8>,

    at_start: bool,
    state: HeaderValueByteIteratorState,
}

impl<'self, R: Reader> HeaderValueByteIterator<'self, R> {

    fn new(reader: &'self mut R) -> HeaderValueByteIterator<'self, R> {
        HeaderValueByteIterator {
            reader: reader,
            next_byte: None,
            at_start: true,
            state: Normal,
        }
    }

    /// Check that the entire header value has been consumed.
    ///
    /// Should there be any trailing linear white space, it is dropped.
    ///
    /// Be cautious using this function as it is destructive, losing a character in the case where
    /// the value has not been entirely consumed.
    ///
    /// This should only be called when finished with a value and ensuring that there aren't
    /// unexpected characters
    ///
    /// Suggested usage is in a ``from_stream`` method::
    ///
    ///     if reader.verify_consumed() {
    ///         Some(header)
    ///     } else {
    ///         None
    ///     }
    ///
    /// ... however, this common case is handled with the ``some_if_consumed`` method, so you may
    /// very well not need to call this function directly.
    #[inline]
    pub fn verify_consumed(&mut self) -> bool {
        self.consume_optional_lws();
        self.next() == None
{'fraction_non_alphanumeric': 0.115403491551567, 'fraction_numerical': 0.0019631213629671177, 'mean_word_length': 2.7301255230125525, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 65, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7190703', 'n_tokens_mistral': 4224, 'n_tokens_neox': 4142, 'n_words': 1572}