<commit_before>// Copyright 2013-2015 The rust-url developers.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

/*!

<a href="https://github.com/servo/rust-url"><img style="position: absolute; top: 0; left: 0; border: 0;" src="../github.png" alt="Fork me on GitHub"></a>
<style>.sidebar { margin-top: 53px }</style>

rust-url is an implementation of the [URL Standard](http://url.spec.whatwg.org/)
for the [Rust](http://rust-lang.org/) programming language.

It builds with [Cargo](http://crates.io/).
To use it in your project, add this to your `Cargo.toml` file:

```Cargo
[dependencies.url]
git = "https://github.com/servo/rust-url"
```

Supporting encodings other than UTF-8 in query strings is an optional feature
that requires [rust-encoding](https://github.com/lifthrasiir/rust-encoding)
and is off by default.
You can enable it with
[Cargo’s *features* mechanism](http://doc.crates.io/manifest.html#the-[features]-section):

```Cargo
[dependencies.url]
git = "https://github.com/servo/rust-url"
features = ["query_encoding"]
```

… or by passing `--cfg 'feature="query_encoding"'` to rustc.


# URL parsing and data structures

First, URL parsing may fail for various reasons and therefore returns a `Result`.

```
use url::{Url, ParseError};

assert!(Url::parse("http://[:::1]") == Err(ParseError::InvalidIpv6Address))
```

Let’s parse a valid URL and look at its components.

```
use url::{Url, Host};

let issue_list_url = Url::parse(
    "https://github.com/rust-lang/rust/issues?labels=E-easy&state=open"
).unwrap();


assert!(issue_list_url.scheme() == "https");
assert!(issue_list_url.username() == "");
assert!(issue_list_url.password() == None);
assert!(issue_list_url.host_str() == Some("github.com"));
assert!(issue_list_url.host() == Some(Host::Domain("github.com")));
assert!(issue_list_url.port() == None);
assert!(issue_list_url.path() == "/rust-lang/rust/issues");
assert!(issue_list_url.path_segments().map(|c| c.collect::<Vec<_>>()) ==
        Some(vec!["rust-lang", "rust", "issues"]));
assert!(issue_list_url.query() == Some("labels=E-easy&state=open"));
assert!(issue_list_url.fragment() == None);
assert!(!issue_list_url.cannot_be_a_base());
```

Some URLs are said to be *cannot-be-a-base*:
they don’t have a username, password, host, or port,
and their "path" is an arbitrary string rather than slash-separated segments:

```
use url::Url;

let data_url = Url::parse("data:text/plain,Hello?World#").unwrap();

assert!(data_url.cannot_be_a_base());
assert!(data_url.scheme() == "data");
assert!(data_url.path() == "text/plain,Hello");
assert!(data_url.path_segments().is_none());
assert!(data_url.query() == Some("World"));
assert!(data_url.fragment() == Some(""));
```


# Base URL

Many contexts allow URL *references* that can be relative to a *base URL*:

```html
<link rel="stylesheet" href="../main.css">
```

Since parsed URL are absolute, giving a base is required for parsing relative URLs:

```
use url::{Url, ParseError};

assert!(Url::parse("../main.css") == Err(ParseError::RelativeUrlWithoutBase))
```

Use the `join` method on an `Url` to use it as a base URL:

```
use url::Url;

let this_document = Url::parse("http://servo.github.io/rust-url/url/index.html").unwrap();
let css_url = this_document.join("../main.css").unwrap();
assert_eq!(css_url.as_str(), "http://servo.github.io/rust-url/main.css")
*/

#![cfg_attr(feature="heap_size", feature(plugin, custom_derive))]
#![cfg_attr(feature="heap_size", plugin(heapsize_plugin))]

#[cfg(feature="rustc-serialize")] extern crate rustc_serialize;
#[macro_use] extern crate matches;
#[cfg(feature="serde")] extern crate serde;
#[cfg(feature="heap_size")] #[macro_use] extern crate heapsize;

pub extern crate idna;

use encoding::EncodingOverride;
use host::HostInternal;
use parser::{Parser, Context, SchemeType, to_u32};
use percent_encoding::{PATH_SEGMENT_ENCODE_SET, USERINFO_ENCODE_SET,
                       percent_encode, percent_decode, utf8_percent_encode};
use std::cmp;
use std::fmt::{self, Write};
use std::hash;
use std::io;
use std::mem;
use std::net::{ToSocketAddrs, IpAddr};
use std::ops::{Range, RangeFrom, RangeTo};
use std::path::{Path, PathBuf};
use std::str;

pub use origin::{Origin, OpaqueOrigin};
pub use host::{Host, HostAndPort, SocketAddrs};
pub use parser::ParseError;
pub use slicing::Position;

mod encoding;
mod host;
mod origin;
mod parser;
mod slicing;

pub mod form_urlencoded;
pub mod percent_encoding;
pub mod quirks;

/// A parsed URL record.
#[derive(Clone)]
#[cfg_attr(feature="heap_size", derive(HeapSizeOf))]
pub struct Url {
    /// Syntax in pseudo-BNF:
    ///
    ///   url = scheme ":" [ hierarchical | non-hierarchical ] [ "?" query ]? [ "#" fragment ]?
    ///   non-hierarchical = non-hierarchical-path
    ///   non-hierarchical-path = /* Does not start with "/" */
    ///   hierarchical = authority? hierarchical-path
    ///   authority = "//" userinfo? host [ ":" port ]?
    ///   userinfo = username [ ":" password ]? "@"
    ///   hierarchical-path = [ "/" path-segment ]+
    serialization: String,

    // Components
    scheme_end: u32,  // Before ':'
    username_end: u32,  // Before ':' (if a password is given) or '@' (if not)
    host_start: u32,
    host_end: u32,
    host: HostInternal,
    port: Option<u16>,
    path_start: u32,  // Before initial '/', if any
    query_start: Option<u32>,  // Before '?', unlike Position::QueryStart
    fragment_start: Option<u32>,  // Before '#', unlike Position::FragmentStart
}

/// Full configuration for the URL parser.
#[derive(Copy, Clone)]
pub struct ParseOptions<'a> {
    base_url: Option<&'a Url>,
    encoding_override: encoding::EncodingOverride,
    log_syntax_violation: Option<&'a Fn(&'static str)>,
}

impl<'a> ParseOptions<'a> {
    /// Change the base URL
    pub fn base_url(mut self, new: Option<&'a Url>) -> Self {
        self.base_url = new;
        self
    }

    /// Override the character encoding of query strings.
    /// This is a legacy concept only relevant for HTML.
    #[cfg(feature = "query_encoding")]
    pub fn encoding_override(mut self, new: Option<encoding::EncodingRef>) -> Self {
        self.encoding_override = EncodingOverride::from_opt_encoding(new).to_output_encoding();
        self
    }

    /// Call the provided function or closure on non-fatal parse errors.
    pub fn log_syntax_violation(mut self, new: Option<&'a Fn(&'static str)>) -> Self {
        self.log_syntax_violation = new;
        self
    }

    /// Parse an URL string with the configuration so far.
    pub fn parse(self, input: &str) -> Result<Url, ::ParseError> {
        Parser {
            serialization: String::with_capacity(input.len()),
            base_url: self.base_url,
            query_encoding_override: self.encoding_override,
            log_syntax_violation: self.log_syntax_violation,
            context: Context::UrlParser,
        }.parse_url(input)
    }
}

impl Url {
    /// Parse an absolute URL from a string.
    #[inline]
    pub fn parse(input: &str) -> Result<Url, ::ParseError> {
        Url::options().parse(input)
    }

    /// Parse a string as an URL, with this URL as the base URL.
    #[inline]
    pub fn join(&self, input: &str) -> Result<Url, ::ParseError> {
        Url::options().base_url(Some(self)).parse(input)
    }

    /// Return a default `ParseOptions` that can fully configure the URL parser.
    pub fn options<'a>() -> ParseOptions<'a> {
        ParseOptions {
            base_url: None,
            encoding_override: EncodingOverride::utf8(),
            log_syntax_violation: None,
        }
    }

    /// Return the serialization of this URL.
    ///
    /// This is fast since that serialization is already stored in the `Url` struct.
    #[inline]
    pub fn as_str(&self) -> &str {
        &self.serialization
    }

    /// Return the serialization of this URL.
    ///
    /// This consumes the `Url` and takes ownership of the `String` stored in it.
    #[inline]
    pub fn into_string(self) -> String {
        self.serialization
    }

    /// For internal testing.
    ///
    /// Methods of the `Url` struct assume a number of invariants.
    /// This checks each of these invariants and panic if one is not met.
    /// This is for testing rust-url itself.
    #[doc(hidden)]
    pub fn assert_invariants(&self) {
        macro_rules! assert {
            ($x: expr) => {
                if !$x {
                    panic!("!( {} ) for URL {:?}", stringify!($x), self.serialization)
                }
            }
        }

        macro_rules! assert_eq {
            ($a: expr, $b: expr) => {
                {
                    let a = $a;
                    let b = $b;
                    if a != b {
                        panic!("{:?} != {:?} ({} != {}) for URL {:?}",
                               a, b, stringify!($a), stringify!($b), self.serialization)
                    }
                }
            }
        }

        assert!(self.scheme_end >= 1);
        assert!(matches!(self.byte_at(0), b'a'...b'z' | b'A'...b'Z'));
        assert!(self.slice(1..self.scheme_end).chars()
                .all(|c| matches!(c, 'a'...'z' | 'A'...'Z' | '0'...'9' | '+' | '-' | '.')));
        assert_eq!(self.byte_at(self.scheme_end), b':');

        if self.slice(self.scheme_end + 1 ..).starts_with("//") {
            // URL with authority
            match self.byte_at(self.username_end) {
                b':' => {
                    assert!(self.host_start >= self.username_end + 2);
                    assert_eq!(self.byte_at(self.host_start - 1), b'@');
                }
                b'@' => assert!(self.host_start == self.username_end + 1),
                _ => assert_eq!(self.username_end, self.scheme_end + 3),
            }
            assert!(self.host_start >= self.username_end);
            assert!(self.host_end >= self.host_start);
            let host_str = self.slice(self.host_start..self.host_end);
            match self.host {
                HostInternal::None => assert_eq!(host_str, ""),
                HostInternal::Ipv4(address) => assert_eq!(host_str, address.to_string()),
                HostInternal::Ipv6(address) => assert_eq!(host_str, format!("[{}]", address)),
                HostInternal::Domain => {
                    if SchemeType::from(self.scheme()).is_special() {
                        assert!(!host_str.is_empty())
                    }
                }
            }
            if self.path_start == self.host_end {
                assert_eq!(self.port, None);
            } else {
                assert_eq!(self.byte_at(self.host_end), b':');
                let port_str = self.slice(self.host_end + 1..self.path_start);
                assert_eq!(self.port, Some(port_str.parse::<u16>().unwrap()));
            }
            assert_eq!(self.byte_at(self.path_start), b'/');
        } else {
            // Anarchist URL (no authority)
            assert_eq!(self.username_end, self.scheme_end + 1);
            assert_eq!(self.host_start, self.scheme_end + 1);
            assert_eq!(self.host_end, self.scheme_end + 1);
            assert_eq!(self.host, HostInternal::None);
            assert_eq!(self.port, None);
            assert_eq!(self.path_start, self.scheme_end + 1);
        }
        if let Some(start) = self.query_start {
            assert!(start > self.path_start);
            assert_eq!(self.byte_at(start), b'?');
        }
        if let Some(start) = self.fragment_start {
            assert!(start > self.path_start);
            assert_eq!(self.byte_at(start), b'#');
        }
        if let (Some(query_start), Some(fragment_start)) = (self.query_start, self.fragment_start) {
            assert!(fragment_start > query_start);
        }
    }

    /// Return the origin of this URL (https://url.spec.whatwg.org/#origin)
    ///
    /// Note: this return an opaque origin for `file:` URLs, which causes
    /// `url.origin() != url.origin()`.
    #[inline]
    pub fn origin(&self) -> Origin {
        origin::url_origin(self)
    }

    /// Return the scheme of this URL, lower-cased, as an ASCII string without the ':' delimiter.
    #[inline]
    pub fn scheme(&self) -> &str {
        self.slice(..self.scheme_end)
    }

    /// Return whether the URL has an 'authority',
    /// which can contain a username, password, host, and port number.
    ///
    /// URLs that do *not* are either path-only like `unix:/run/foo.socket`
    /// or cannot-be-a-base like `data:text/plain,Stuff`.
    #[inline]
    pub fn has_authority(&self) -> bool {
        debug_assert!(self.byte_at(self.scheme_end) == b':');
        self.slice(self.scheme_end..).starts_with("://")
    }

    /// Return whether this URL is a cannot-be-a-base URL,
    /// meaning that parsing a relative URL string with this URL as the base will return an error.
    ///
    /// This is the case if the scheme and `:` delimiter are not followed by a `/` slash,
    /// as is typically the case of `data:` and `mailto:` URLs.
    #[inline]
    pub fn cannot_be_a_base(&self) -> bool {
        self.byte_at(self.path_start) != b'/'
    }

    /// Return the username for this URL (typically the empty string)
    /// as a percent-encoded ASCII string.
    pub fn username(&self) -> &str {
        if self.has_authority() {
            self.slice(self.scheme_end + ("://".len() as u32)..self.username_end)
        } else {
            ""
        }
    }

    /// Return the password for this URL, if any, as a percent-encoded ASCII string.
    pub fn password(&self) -> Option<&str> {
        // This ':' is not the one marking a port number since a host can not be empty.
        // (Except for file: URLs, which do not have port numbers.)
        if self.has_authority() && self.byte_at(self.username_end) == b':' {
            debug_assert!(self.byte_at(self.host_start - 1) == b'@');
            Some(self.slice(self.username_end + 1..self.host_start - 1))
        } else {
            None
        }
    }

    /// Equivalent to `url.host().is_some()`.
    pub fn has_host(&self) -> bool {
        !matches!(self.host, HostInternal::None)
    }

    /// Return the string representation of the host (domain or IP address) for this URL, if any.
    ///
    /// Non-ASCII domains are punycode-encoded per IDNA.
    /// IPv6 addresses are given between `[` and `]` brackets.
    ///
    /// Cannot-be-a-base URLs (typical of `data:` and `mailto:`) and some `file:` URLs
    /// don’t have a host.
    ///
    /// See also the `host` method.
    pub fn host_str(&self) -> Option<&str> {
        if self.has_host() {
            Some(self.slice(self.host_start..self.host_end))
        } else {
            None
        }
    }

    /// Return the parsed representation of the host for this URL.
    /// Non-ASCII domain labels are punycode-encoded per IDNA.
    ///
    /// Cannot-be-a-base URLs (typical of `data:` and `mailto:`) and some `file:` URLs
    /// don’t have a host.
    ///
    /// See also the `host_str` method.
    pub fn host(&self) -> Option<Host<&str>> {
        match self.host {
            HostInternal::None => None,
            HostInternal::Domain => Some(Host::Domain(self.slice(self.host_start..self.host_end))),
            HostInternal::Ipv4(address) => Some(Host::Ipv4(address)),
            HostInternal::Ipv6(address) => Some(Host::Ipv6(address)),
        }
    }

    /// If this URL has a host and it is a domain name (not an IP address), return it.
    pub fn domain(&self) -> Option<&str> {
        match self.host {
            HostInternal::Domain => Some(self.slice(self.host_start..self.host_end)),
            _ => None,
        }
    }

    /// Return the port number for this URL, if any.
    #[inline]
    pub fn port(&self) -> Option<u16> {
        self.port
    }

    /// Return the port number for this URL, or the default port number if it is known.
    ///
    /// This method only knows the default port number
    /// of the `http`, `https`, `ws`, `wss`, `ftp`, and `gopher` schemes.
    ///
    /// For URLs in these schemes, this method always returns `Some(_)`.
    /// For other schemes, it is the same as `Url::port()`.
    #[inline]
    pub fn port_or_known_default(&self) -> Option<u16> {
        self.port.or_else(|| parser::default_port(self.scheme()))
    }

    /// If the URL has a host, return something that implements `ToSocketAddrs`.
    ///
    /// If the URL has no port number and the scheme’s default port number is not known
    /// (see `Url::port_or_known_default`),
    /// the closure is called to obtain a port number.
    /// Typically, this closure can match on the result `Url::scheme`
    /// to have per-scheme default port numbers,
    /// and panic for schemes it’s not prepared to handle.
    /// For example:
    ///
    /// ```rust
    /// # use url::Url;
    /// # use std::net::TcpStream;
    /// # use std::io;
    ///
    /// fn connect(url: &Url) -> io::Result<TcpStream> {
    ///     TcpStream::connect(try!(url.with_default_port(default_port)))
    /// }
    ///
    /// fn default_port(url: &Url) -> Result<u16, ()> {
    ///     match url.scheme() {
    ///         "git" => Ok(9418),
    ///         "git+ssh" => Ok(22),
    ///         "git+https" => Ok(443),
    ///         "git+http" => Ok(80),
    ///         _ => Err(()),
    ///     }
    /// }
    /// ```
    pub fn with_default_port<F>(&self, f: F) -> io::Result<HostAndPort<&str>>
    where F: FnOnce(&Url) -> Result<u16, ()> {
        Ok(HostAndPort {
            host: try!(self.host()
                           .ok_or(())
                           .or_else(|()| io_error("URL has no host"))),
            port: try!(self.port_or_known_default()
                           .ok_or(())
                           .or_else(|()| f(self))
                           .or_else(|()| io_error("URL has no port number")))
        })
    }

    /// Return the path for this URL, as a percent-encoded ASCII string.
    /// For cannot-be-a-base URLs, this is an arbitrary string that doesn’t start with '/'.
    /// For other URLs, this starts with a '/' slash
    /// and continues with slash-separated path segments.
    pub fn path(&self) -> &str {
        match (self.query_start, self.fragment_start) {
            (None, None) => self.slice(self.path_start..),
            (Some(next_component_start), _) |
            (None, Some(next_component_start)) => {
                self.slice(self.path_start..next_component_start)
            }
        }
    }

    /// Unless this URL is cannot-be-a-base,
    /// return an iterator of '/' slash-separated path segments,
    /// each as a percent-encoded ASCII string.
    ///
    /// Return `None` for cannot-be-a-base URLs, or an iterator of at least one string.
    pub fn path_segments(&self) -> Option<str::Split<char>> {
        let path = self.path();
        if path.starts_with('/') {
            Some(path[1..].split('/'))
        } else {
            None
        }
    }

    /// Return this URL’s query string, if any, as a percent-encoded ASCII string.
    pub fn query(&self) -> Option<&str> {
        match (self.query_start, self.fragment_start) {
            (None, _) => None,
            (Some(query_start), None) => {
                debug_assert!(self.byte_at(query_start) == b'?');
                Some(self.slice(query_start + 1..))
            }
            (Some(query_start), Some(fragment_start)) => {
                debug_assert!(self.byte_at(query_start) == b'?');
                Some(self.slice(query_start + 1..fragment_start))
            }
        }
    }

    /// Parse the URL’s query string, if any, as `application/x-www-form-urlencoded`
    /// and return an iterator of (key, value) pairs.
    #[inline]
    pub fn query_pairs(&self) -> form_urlencoded::Parse {
        form_urlencoded::parse(self.query().unwrap_or("").as_bytes())
    }

    /// Return this URL’s fragment identifier, if any.
    ///
    /// **Note:** the parser did *not* percent-encode this component,
    /// but the input may have been percent-encoded already.
    pub fn fragment(&self) -> Option<&str> {
        self.fragment_start.map(|start| {
            debug_assert!(self.byte_at(start) == b'#');
            self.slice(start + 1..)
        })
    }

    fn mutate<F: FnOnce(&mut Parser) -> R, R>(&mut self, f: F) -> R {
        let mut parser = Parser::for_setter(mem::replace(&mut self.serialization, String::new()));
        let result = f(&mut parser);
        self.serialization = parser.serialization;
        result
    }

    /// Change this URL’s fragment identifier.
    pub fn set_fragment(&mut self, fragment: Option<&str>) {
        // Remove any previous fragment
        if let Some(start) = self.fragment_start {
            debug_assert!(self.byte_at(start) == b'#');
            self.serialization.truncate(start as usize);
        }
        // Write the new one
        if let Some(input) = fragment {
            self.fragment_start = Some(to_u32(self.serialization.len()).unwrap());
            self.serialization.push('#');
            self.mutate(|parser| parser.parse_fragment(input))
        } else {
            self.fragment_start = None
        }
    }

    fn take_fragment(&mut self) -> Option<String> {
        self.fragment_start.take().map(|start| {
            debug_assert!(self.byte_at(start) == b'#');
            let fragment = self.slice(start + 1..).to_owned();
            self.serialization.truncate(start as usize);
            fragment
        })
    }

    fn restore_already_parsed_fragment(&mut self, fragment: Option<String>) {
        if let Some(ref fragment) = fragment {
            assert!(self.fragment_start.is_none());
            self.fragment_start = Some(to_u32(self.serialization.len()).unwrap());
            self.serialization.push('#');
            self.serialization.push_str(fragment);
        }
    }

    /// Change this URL’s query string.
    pub fn set_query(&mut self, query: Option<&str>) {
        let fragment = self.take_fragment();

        // Remove any previous query
        if let Some(start) = self.query_start.take() {
            debug_assert!(self.byte_at(start) == b'?');
            self.serialization.truncate(start as usize);
        }
        // Write the new query, if any
        if let Some(input) = query {
            self.query_start = Some(to_u32(self.serialization.len()).unwrap());
            self.serialization.push('?');
            let scheme_end = self.scheme_end;
            self.mutate(|parser| parser.parse_query(scheme_end, input));
        }

        self.restore_already_parsed_fragment(fragment);
    }

    /// Manipulate this URL’s query string, viewed as a sequence of name/value pairs
    /// in `application/x-www-form-urlencoded` syntax.
    ///
    /// The return value has a method-chaining API:
    ///
    /// ```rust
    /// # use url::Url;
    /// let mut url = Url::parse("https://example.net?lang=fr#nav").unwrap();
    /// assert_eq!(url.query(), Some("lang=fr"));
    ///
    /// url.query_pairs_mut().append_pair("foo", "bar");
    /// assert_eq!(url.query(), Some("lang=fr&foo=bar"));
    /// assert_eq!(url.as_str(), "https://example.net/?lang=fr&foo=bar#nav");
    ///
    /// url.query_pairs_mut()
    ///     .clear()
    ///     .append_pair("foo", "bar & baz")
    ///     .append_pair("saisons", "Été+hiver");
    /// assert_eq!(url.query(), Some("foo=bar+%26+baz&saisons=%C3%89t%C3%A9%2Bhiver"));
    /// assert_eq!(url.as_str(),
    ///            "https://example.net/?foo=bar+%26+baz&saisons=%C3%89t%C3%A9%2Bhiver#nav");
    /// ```
    ///
    /// Note: `url.query_pairs_mut().clear();` is equivalent to `url.set_query(Some(""))`,
    /// not `url.set_query(None)`.
    ///
    /// The state of `Url` is unspecified if this return value is leaked without being dropped.
    pub fn query_pairs_mut(&mut self) -> form_urlencoded::Serializer<UrlQuery> {
        let fragment = self.take_fragment();

        let query_start;
        if let Some(start) = self.query_start {
            debug_assert!(self.byte_at(start) == b'?');
            query_start = start as usize;
        } else {
            query_start = self.serialization.len();
            self.query_start = Some(to_u32(query_start).unwrap());
            self.serialization.push('?');
        }

        let query = UrlQuery { url: self, fragment: fragment };
        form_urlencoded::Serializer::for_suffix(query, query_start + "?".len())
    }

    /// Change this URL’s path.
    pub fn set_path(&mut self, path: &str) {
        let (old_after_path_pos, after_path) = match (self.query_start, self.fragment_start) {
            (Some(i), _) | (None, Some(i)) => (i, self.slice(i..).to_owned()),
            (None, None) => (to_u32(self.serialization.len()).unwrap(), String::new())
        };
        let cannot_be_a_base = self.cannot_be_a_base();
        let scheme_type = SchemeType::from(self.scheme());
        self.serialization.truncate(self.path_start as usize);
        self.mutate(|parser| {
            if cannot_be_a_base {
                if path.starts_with('/') {
                    parser.serialization.push_str("%2F");
                    parser.parse_cannot_be_a_base_path(&path[1..]);
                } else {
                    parser.parse_cannot_be_a_base_path(path);
                }
            } else {
                let mut has_host = true;  // FIXME
                parser.parse_path_start(scheme_type, &mut has_host, path);
            }
        });
        let new_after_path_pos = to_u32(self.serialization.len()).unwrap();
        let adjust = |index: &mut u32| {
            *index -= old_after_path_pos;
            *index += new_after_path_pos;
        };
        if let Some(ref mut index) = self.query_start { adjust(index) }
        if let Some(ref mut index) = self.fragment_start { adjust(index) }
        self.serialization.push_str(&after_path)
    }

    /// Remove the last segment of this URL’s path.
    ///
    /// If this URL is cannot-be-a-base, do nothing and return `Err`.
    /// If this URL is not cannot-be-a-base and its path is `/`, do nothing and return `Ok`.
    pub fn pop_path_segment(&mut self) -> Result<(), ()> {
        if self.cannot_be_a_base() {
            return Err(())
        }
        let last_slash;
        let path_len;
        {
            let path = self.path();
            last_slash = path.rfind('/').unwrap();
            path_len = path.len();
        };
        if last_slash > 0 {
            // Found a slash other than the initial one
            let last_slash = last_slash + self.path_start as usize;
            let path_end = path_len + self.path_start as usize;
            self.serialization.drain(last_slash..path_end);
            let offset = (path_end - last_slash) as u32;
            if let Some(ref mut index) = self.query_start { *index -= offset }
            if let Some(ref mut index) = self.fragment_start { *index -= offset }
        }
        Ok(())
    }

    /// Add a segment at the end of this URL’s path.
    ///
    /// If this URL is cannot-be-a-base, do nothing and return `Err`.
    pub fn push_path_segment(&mut self, segment: &str) -> Result<(), ()> {
        if self.cannot_be_a_base() {
            return Err(())
        }
        let after_path = match (self.query_start, self.fragment_start) {
            (Some(i), _) | (None, Some(i)) => {
                let s = self.slice(i..).to_owned();
                self.serialization.truncate(i as usize);
                s
            },
            (None, None) => String::new()
        };
        let scheme_type = SchemeType::from(self.scheme());
        let path_start = self.path_start as usize;
        self.serialization.push('/');
        self.mutate(|parser| {
            parser.context = parser::Context::PathSegmentSetter;
            let mut has_host = true;  // FIXME account for this?
            parser.parse_path(scheme_type, &mut has_host, path_start, segment)
        });
        let offset = to_u32(self.serialization.len()).unwrap() - self.path_start;
        if let Some(ref mut index) = self.query_start { *index += offset }
        if let Some(ref mut index) = self.fragment_start { *index += offset }
        self.serialization.push_str(&after_path);
        Ok(())
    }

    /// Change this URL’s port number.
    ///
    /// If this URL is cannot-be-a-base, does not have a host, or has the `file` scheme;
    /// do nothing and return `Err`.<commit_msg>Make pop_path_segment and push_path_segment private for now.<commit_after>// Copyright 2013-2015 The rust-url developers.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

/*!

<a href="https://github.com/servo/rust-url"><img style="position: absolute; top: 0; left: 0; border: 0;" src="../github.png" alt="Fork me on GitHub"></a>
<style>.sidebar { margin-top: 53px }</style>

rust-url is an implementation of the [URL Standard](http://url.spec.whatwg.org/)
for the [Rust](http://rust-lang.org/) programming language.

It builds with [Cargo](http://crates.io/).
To use it in your project, add this to your `Cargo.toml` file:

```Cargo
[dependencies.url]
git = "https://github.com/servo/rust-url"
```

Supporting encodings other than UTF-8 in query strings is an optional feature
that requires [rust-encoding](https://github.com/lifthrasiir/rust-encoding)
and is off by default.
You can enable it with
[Cargo’s *features* mechanism](http://doc.crates.io/manifest.html#the-[features]-section):

```Cargo
[dependencies.url]
git = "https://github.com/servo/rust-url"
features = ["query_encoding"]
```

… or by passing `--cfg 'feature="query_encoding"'` to rustc.


# URL parsing and data structures

First, URL parsing may fail for various reasons and therefore returns a `Result`.

```
use url::{Url, ParseError};

assert!(Url::parse("http://[:::1]") == Err(ParseError::InvalidIpv6Address))
```

Let’s parse a valid URL and look at its components.

```
use url::{Url, Host};

let issue_list_url = Url::parse(
    "https://github.com/rust-lang/rust/issues?labels=E-easy&state=open"
).unwrap();


assert!(issue_list_url.scheme() == "https");
assert!(issue_list_url.username() == "");
assert!(issue_list_url.password() == None);
assert!(issue_list_url.host_str() == Some("github.com"));
assert!(issue_list_url.host() == Some(Host::Domain("github.com")));
assert!(issue_list_url.port() == None);
assert!(issue_list_url.path() == "/rust-lang/rust/issues");
assert!(issue_list_url.path_segments().map(|c| c.collect::<Vec<_>>()) ==
        Some(vec!["rust-lang", "rust", "issues"]));
assert!(issue_list_url.query() == Some("labels=E-easy&state=open"));
assert!(issue_list_url.fragment() == None);
assert!(!issue_list_url.cannot_be_a_base());
```

Some URLs are said to be *cannot-be-a-base*:
they don’t have a username, password, host, or port,
and their "path" is an arbitrary string rather than slash-separated segments:

```
use url::Url;

let data_url = Url::parse("data:text/plain,Hello?World#").unwrap();

assert!(data_url.cannot_be_a_base());
assert!(data_url.scheme() == "data");
assert!(data_url.path() == "text/plain,Hello");
assert!(data_url.path_segments().is_none());
assert!(data_url.query() == Some("World"));
assert!(data_url.fragment() == Some(""));
```


# Base URL

Many contexts allow URL *references* that can be relative to a *base URL*:

```html
<link rel="stylesheet" href="../main.css">
```

Since parsed URL are absolute, giving a base is required for parsing relative URLs:

```
use url::{Url, ParseError};

assert!(Url::parse("../main.css") == Err(ParseError::RelativeUrlWithoutBase))
```

Use the `join` method on an `Url` to use it as a base URL:

```
use url::Url;

let this_document = Url::parse("http://servo.github.io/rust-url/url/index.html").unwrap();
let css_url = this_document.join("../main.css").unwrap();
assert_eq!(css_url.as_str(), "http://servo.github.io/rust-url/main.css")
*/

#![cfg_attr(feature="heap_size", feature(plugin, custom_derive))]
#![cfg_attr(feature="heap_size", plugin(heapsize_plugin))]

#[cfg(feature="rustc-serialize")] extern crate rustc_serialize;
#[macro_use] extern crate matches;
#[cfg(feature="serde")] extern crate serde;
#[cfg(feature="heap_size")] #[macro_use] extern crate heapsize;

pub extern crate idna;

use encoding::EncodingOverride;
use host::HostInternal;
use parser::{Parser, Context, SchemeType, to_u32};
use percent_encoding::{PATH_SEGMENT_ENCODE_SET, USERINFO_ENCODE_SET,
                       percent_encode, percent_decode, utf8_percent_encode};
use std::cmp;
use std::fmt::{self, Write};
use std::hash;
use std::io;
use std::mem;
use std::net::{ToSocketAddrs, IpAddr};
use std::ops::{Range, RangeFrom, RangeTo};
use std::path::{Path, PathBuf};
use std::str;

pub use origin::{Origin, OpaqueOrigin};
pub use host::{Host, HostAndPort, SocketAddrs};
pub use parser::ParseError;
pub use slicing::Position;

mod encoding;
mod host;
mod origin;
mod parser;
mod slicing;

pub mod form_urlencoded;
pub mod percent_encoding;
pub mod quirks;

/// A parsed URL record.
#[derive(Clone)]
#[cfg_attr(feature="heap_size", derive(HeapSizeOf))]
pub struct Url {
    /// Syntax in pseudo-BNF:
    ///
    ///   url = scheme ":" [ hierarchical | non-hierarchical ] [ "?" query ]? [ "#" fragment ]?
    ///   non-hierarchical = non-hierarchical-path
    ///   non-hierarchical-path = /* Does not start with "/" */
    ///   hierarchical = authority? hierarchical-path
    ///   authority = "//" userinfo? host [ ":" port ]?
    ///   userinfo = username [ ":" password ]? "@"
    ///   hierarchical-path = [ "/" path-segment ]+
    serialization: String,

    // Components
    scheme_end: u32,  // Before ':'
    username_end: u32,  // Before ':' (if a password is given) or '@' (if not)
    host_start: u32,
    host_end: u32,
    host: HostInternal,
    port: Option<u16>,
    path_start: u32,  // Before initial '/', if any
    query_start: Option<u32>,  // Before '?', unlike Position::QueryStart
    fragment_start: Option<u32>,  // Before '#', unlike Position::FragmentStart
}

/// Full configuration for the URL parser.
#[derive(Copy, Clone)]
pub struct ParseOptions<'a> {
    base_url: Option<&'a Url>,
    encoding_override: encoding::EncodingOverride,
    log_syntax_violation: Option<&'a Fn(&'static str)>,
}

impl<'a> ParseOptions<'a> {
    /// Change the base URL
    pub fn base_url(mut self, new: Option<&'a Url>) -> Self {
        self.base_url = new;
        self
    }

    /// Override the character encoding of query strings.
    /// This is a legacy concept only relevant for HTML.
    #[cfg(feature = "query_encoding")]
    pub fn encoding_override(mut self, new: Option<encoding::EncodingRef>) -> Self {
        self.encoding_override = EncodingOverride::from_opt_encoding(new).to_output_encoding();
        self
    }

    /// Call the provided function or closure on non-fatal parse errors.
    pub fn log_syntax_violation(mut self, new: Option<&'a Fn(&'static str)>) -> Self {
        self.log_syntax_violation = new;
        self
    }

    /// Parse an URL string with the configuration so far.
    pub fn parse(self, input: &str) -> Result<Url, ::ParseError> {
        Parser {
            serialization: String::with_capacity(input.len()),
            base_url: self.base_url,
            query_encoding_override: self.encoding_override,
            log_syntax_violation: self.log_syntax_violation,
            context: Context::UrlParser,
        }.parse_url(input)
    }
}

impl Url {
    /// Parse an absolute URL from a string.
    #[inline]
    pub fn parse(input: &str) -> Result<Url, ::ParseError> {
        Url::options().parse(input)
    }

    /// Parse a string as an URL, with this URL as the base URL.
    #[inline]
    pub fn join(&self, input: &str) -> Result<Url, ::ParseError> {
        Url::options().base_url(Some(self)).parse(input)
    }

    /// Return a default `ParseOptions` that can fully configure the URL parser.
    pub fn options<'a>() -> ParseOptions<'a> {
        ParseOptions {
            base_url: None,
            encoding_override: EncodingOverride::utf8(),
            log_syntax_violation: None,
        }
    }

    /// Return the serialization of this URL.
    ///
    /// This is fast since that serialization is already stored in the `Url` struct.
    #[inline]
    pub fn as_str(&self) -> &str {
        &self.serialization
    }

    /// Return the serialization of this URL.
    ///
    /// This consumes the `Url` and takes ownership of the `String` stored in it.
    #[inline]
    pub fn into_string(self) -> String {
        self.serialization
    }

    /// For internal testing.
    ///
    /// Methods of the `Url` struct assume a number of invariants.
    /// This checks each of these invariants and panic if one is not met.
    /// This is for testing rust-url itself.
    #[doc(hidden)]
    pub fn assert_invariants(&self) {
        macro_rules! assert {
            ($x: expr) => {
                if !$x {
                    panic!("!( {} ) for URL {:?}", stringify!($x), self.serialization)
                }
            }
        }

        macro_rules! assert_eq {
            ($a: expr, $b: expr) => {
                {
                    let a = $a;
                    let b = $b;
                    if a != b {
                        panic!("{:?} != {:?} ({} != {}) for URL {:?}",
                               a, b, stringify!($a), stringify!($b), self.serialization)
                    }
                }
            }
        }

        assert!(self.scheme_end >= 1);
        assert!(matches!(self.byte_at(0), b'a'...b'z' | b'A'...b'Z'));
        assert!(self.slice(1..self.scheme_end).chars()
                .all(|c| matches!(c, 'a'...'z' | 'A'...'Z' | '0'...'9' | '+' | '-' | '.')));
        assert_eq!(self.byte_at(self.scheme_end), b':');

        if self.slice(self.scheme_end + 1 ..).starts_with("//") {
            // URL with authority
            match self.byte_at(self.username_end) {
                b':' => {
                    assert!(self.host_start >= self.username_end + 2);
                    assert_eq!(self.byte_at(self.host_start - 1), b'@');
                }
                b'@' => assert!(self.host_start == self.username_end + 1),
                _ => assert_eq!(self.username_end, self.scheme_end + 3),
            }
            assert!(self.host_start >= self.username_end);
            assert!(self.host_end >= self.host_start);
            let host_str = self.slice(self.host_start..self.host_end);
            match self.host {
                HostInternal::None => assert_eq!(host_str, ""),
                HostInternal::Ipv4(address) => assert_eq!(host_str, address.to_string()),
                HostInternal::Ipv6(address) => assert_eq!(host_str, format!("[{}]", address)),
                HostInternal::Domain => {
                    if SchemeType::from(self.scheme()).is_special() {
                        assert!(!host_str.is_empty())
                    }
                }
            }
            if self.path_start == self.host_end {
                assert_eq!(self.port, None);
            } else {
                assert_eq!(self.byte_at(self.host_end), b':');
                let port_str = self.slice(self.host_end + 1..self.path_start);
                assert_eq!(self.port, Some(port_str.parse::<u16>().unwrap()));
            }
            assert_eq!(self.byte_at(self.path_start), b'/');
        } else {
            // Anarchist URL (no authority)
            assert_eq!(self.username_end, self.scheme_end + 1);
            assert_eq!(self.host_start, self.scheme_end + 1);
            assert_eq!(self.host_end, self.scheme_end + 1);
            assert_eq!(self.host, HostInternal::None);
            assert_eq!(self.port, None);
            assert_eq!(self.path_start, self.scheme_end + 1);
        }
        if let Some(start) = self.query_start {
            assert!(start > self.path_start);
            assert_eq!(self.byte_at(start), b'?');
        }
        if let Some(start) = self.fragment_start {
            assert!(start > self.path_start);
            assert_eq!(self.byte_at(start), b'#');
        }
        if let (Some(query_start), Some(fragment_start)) = (self.query_start, self.fragment_start) {
            assert!(fragment_start > query_start);
        }
    }

    /// Return the origin of this URL (https://url.spec.whatwg.org/#origin)
    ///
    /// Note: this return an opaque origin for `file:` URLs, which causes
    /// `url.origin() != url.origin()`.
    #[inline]
    pub fn origin(&self) -> Origin {
        origin::url_origin(self)
    }

    /// Return the scheme of this URL, lower-cased, as an ASCII string without the ':' delimiter.
    #[inline]
    pub fn scheme(&self) -> &str {
        self.slice(..self.scheme_end)
    }

    /// Return whether the URL has an 'authority',
    /// which can contain a username, password, host, and port number.
    ///
    /// URLs that do *not* are either path-only like `unix:/run/foo.socket`
    /// or cannot-be-a-base like `data:text/plain,Stuff`.
    #[inline]
    pub fn has_authority(&self) -> bool {
        debug_assert!(self.byte_at(self.scheme_end) == b':');
        self.slice(self.scheme_end..).starts_with("://")
    }

    /// Return whether this URL is a cannot-be-a-base URL,
    /// meaning that parsing a relative URL string with this URL as the base will return an error.
    ///
    /// This is the case if the scheme and `:` delimiter are not followed by a `/` slash,
    /// as is typically the case of `data:` and `mailto:` URLs.
    #[inline]
    pub fn cannot_be_a_base(&self) -> bool {
        self.byte_at(self.path_start) != b'/'
    }

    /// Return the username for this URL (typically the empty string)
    /// as a percent-encoded ASCII string.
    pub fn username(&self) -> &str {
        if self.has_authority() {
            self.slice(self.scheme_end + ("://".len() as u32)..self.username_end)
        } else {
            ""
        }
    }

    /// Return the password for this URL, if any, as a percent-encoded ASCII string.
    pub fn password(&self) -> Option<&str> {
        // This ':' is not the one marking a port number since a host can not be empty.
        // (Except for file: URLs, which do not have port numbers.)
        if self.has_authority() && self.byte_at(self.username_end) == b':' {
            debug_assert!(self.byte_at(self.host_start - 1) == b'@');
            Some(self.slice(self.username_end + 1..self.host_start - 1))
        } else {
            None
        }
    }

    /// Equivalent to `url.host().is_some()`.
    pub fn has_host(&self) -> bool {
        !matches!(self.host, HostInternal::None)
    }

    /// Return the string representation of the host (domain or IP address) for this URL, if any.
    ///
    /// Non-ASCII domains are punycode-encoded per IDNA.
    /// IPv6 addresses are given between `[` and `]` brackets.
    ///
    /// Cannot-be-a-base URLs (typical of `data:` and `mailto:`) and some `file:` URLs
    /// don’t have a host.
    ///
    /// See also the `host` method.
    pub fn host_str(&self) -> Option<&str> {
        if self.has_host() {
            Some(self.slice(self.host_start..self.host_end))
        } else {
            None
        }
    }

    /// Return the parsed representation of the host for this URL.
    /// Non-ASCII domain labels are punycode-encoded per IDNA.
    ///
    /// Cannot-be-a-base URLs (typical of `data:` and `mailto:`) and some `file:` URLs
    /// don’t have a host.
    ///
    /// See also the `host_str` method.
    pub fn host(&self) -> Option<Host<&str>> {
        match self.host {
            HostInternal::None => None,
            HostInternal::Domain => Some(Host::Domain(self.slice(self.host_start..self.host_end))),
            HostInternal::Ipv4(address) => Some(Host::Ipv4(address)),
            HostInternal::Ipv6(address) => Some(Host::Ipv6(address)),
        }
    }

    /// If this URL has a host and it is a domain name (not an IP address), return it.
    pub fn domain(&self) -> Option<&str> {
        match self.host {
            HostInternal::Domain => Some(self.slice(self.host_start..self.host_end)),
            _ => None,
        }
    }

    /// Return the port number for this URL, if any.
    #[inline]
    pub fn port(&self) -> Option<u16> {
        self.port
    }

    /// Return the port number for this URL, or the default port number if it is known.
    ///
    /// This method only knows the default port number
    /// of the `http`, `https`, `ws`, `wss`, `ftp`, and `gopher` schemes.
    ///
    /// For URLs in these schemes, this method always returns `Some(_)`.
    /// For other schemes, it is the same as `Url::port()`.
    #[inline]
    pub fn port_or_known_default(&self) -> Option<u16> {
        self.port.or_else(|| parser::default_port(self.scheme()))
    }

    /// If the URL has a host, return something that implements `ToSocketAddrs`.
    ///
    /// If the URL has no port number and the scheme’s default port number is not known
    /// (see `Url::port_or_known_default`),
    /// the closure is called to obtain a port number.
    /// Typically, this closure can match on the result `Url::scheme`
    /// to have per-scheme default port numbers,
    /// and panic for schemes it’s not prepared to handle.
    /// For example:
    ///
    /// ```rust
    /// # use url::Url;
    /// # use std::net::TcpStream;
    /// # use std::io;
    ///
    /// fn connect(url: &Url) -> io::Result<TcpStream> {
    ///     TcpStream::connect(try!(url.with_default_port(default_port)))
    /// }
    ///
    /// fn default_port(url: &Url) -> Result<u16, ()> {
    ///     match url.scheme() {
    ///         "git" => Ok(9418),
    ///         "git+ssh" => Ok(22),
    ///         "git+https" => Ok(443),
    ///         "git+http" => Ok(80),
    ///         _ => Err(()),
    ///     }
    /// }
    /// ```
    pub fn with_default_port<F>(&self, f: F) -> io::Result<HostAndPort<&str>>
    where F: FnOnce(&Url) -> Result<u16, ()> {
        Ok(HostAndPort {
            host: try!(self.host()
                           .ok_or(())
                           .or_else(|()| io_error("URL has no host"))),
            port: try!(self.port_or_known_default()
                           .ok_or(())
                           .or_else(|()| f(self))
                           .or_else(|()| io_error("URL has no port number")))
        })
    }

    /// Return the path for this URL, as a percent-encoded ASCII string.
    /// For cannot-be-a-base URLs, this is an arbitrary string that doesn’t start with '/'.
    /// For other URLs, this starts with a '/' slash
    /// and continues with slash-separated path segments.
    pub fn path(&self) -> &str {
        match (self.query_start, self.fragment_start) {
            (None, None) => self.slice(self.path_start..),
            (Some(next_component_start), _) |
            (None, Some(next_component_start)) => {
                self.slice(self.path_start..next_component_start)
            }
        }
    }

    /// Unless this URL is cannot-be-a-base,
    /// return an iterator of '/' slash-separated path segments,
    /// each as a percent-encoded ASCII string.
    ///
    /// Return `None` for cannot-be-a-base URLs, or an iterator of at least one string.
    pub fn path_segments(&self) -> Option<str::Split<char>> {
        let path = self.path();
        if path.starts_with('/') {
            Some(path[1..].split('/'))
        } else {
            None
        }
    }

    /// Return this URL’s query string, if any, as a percent-encoded ASCII string.
    pub fn query(&self) -> Option<&str> {
        match (self.query_start, self.fragment_start) {
            (None, _) => None,
            (Some(query_start), None) => {
                debug_assert!(self.byte_at(query_start) == b'?');
                Some(self.slice(query_start + 1..))
            }
            (Some(query_start), Some(fragment_start)) => {
                debug_assert!(self.byte_at(query_start) == b'?');
                Some(self.slice(query_start + 1..fragment_start))
            }
        }
    }

    /// Parse the URL’s query string, if any, as `application/x-www-form-urlencoded`
    /// and return an iterator of (key, value) pairs.
    #[inline]
    pub fn query_pairs(&self) -> form_urlencoded::Parse {
        form_urlencoded::parse(self.query().unwrap_or("").as_bytes())
    }

    /// Return this URL’s fragment identifier, if any.
    ///
    /// **Note:** the parser did *not* percent-encode this component,
    /// but the input may have been percent-encoded already.
    pub fn fragment(&self) -> Option<&str> {
        self.fragment_start.map(|start| {
            debug_assert!(self.byte_at(start) == b'#');
            self.slice(start + 1..)
        })
    }

    fn mutate<F: FnOnce(&mut Parser) -> R, R>(&mut self, f: F) -> R {
        let mut parser = Parser::for_setter(mem::replace(&mut self.serialization, String::new()));
        let result = f(&mut parser);
        self.serialization = parser.serialization;
        result
    }

    /// Change this URL’s fragment identifier.
    pub fn set_fragment(&mut self, fragment: Option<&str>) {
        // Remove any previous fragment
        if let Some(start) = self.fragment_start {
            debug_assert!(self.byte_at(start) == b'#');
            self.serialization.truncate(start as usize);
        }
        // Write the new one
        if let Some(input) = fragment {
            self.fragment_start = Some(to_u32(self.serialization.len()).unwrap());
            self.serialization.push('#');
            self.mutate(|parser| parser.parse_fragment(input))
        } else {
            self.fragment_start = None
        }
    }

    fn take_fragment(&mut self) -> Option<String> {
        self.fragment_start.take().map(|start| {
            debug_assert!(self.byte_at(start) == b'#');
            let fragment = self.slice(start + 1..).to_owned();
            self.serialization.truncate(start as usize);
            fragment
        })
    }

    fn restore_already_parsed_fragment(&mut self, fragment: Option<String>) {
        if let Some(ref fragment) = fragment {
            assert!(self.fragment_start.is_none());
            self.fragment_start = Some(to_u32(self.serialization.len()).unwrap());
            self.serialization.push('#');
            self.serialization.push_str(fragment);
        }
    }

    /// Change this URL’s query string.
    pub fn set_query(&mut self, query: Option<&str>) {
        let fragment = self.take_fragment();

        // Remove any previous query
        if let Some(start) = self.query_start.take() {
            debug_assert!(self.byte_at(start) == b'?');
            self.serialization.truncate(start as usize);
        }
        // Write the new query, if any
        if let Some(input) = query {
            self.query_start = Some(to_u32(self.serialization.len()).unwrap());
            self.serialization.push('?');
            let scheme_end = self.scheme_end;
            self.mutate(|parser| parser.parse_query(scheme_end, input));
        }

        self.restore_already_parsed_fragment(fragment);
    }

    /// Manipulate this URL’s query string, viewed as a sequence of name/value pairs
    /// in `application/x-www-form-urlencoded` syntax.
    ///
    /// The return value has a method-chaining API:
    ///
    /// ```rust
    /// # use url::Url;
    /// let mut url = Url::parse("https://example.net?lang=fr#nav").unwrap();
    /// assert_eq!(url.query(), Some("lang=fr"));
    ///
    /// url.query_pairs_mut().append_pair("foo", "bar");
    /// assert_eq!(url.query(), Some("lang=fr&foo=bar"));
    /// assert_eq!(url.as_str(), "https://example.net/?lang=fr&foo=bar#nav");
    ///
    /// url.query_pairs_mut()
    ///     .clear()
    ///     .append_pair("foo", "bar & baz")
    ///     .append_pair("saisons", "Été+hiver");
    /// assert_eq!(url.query(), Some("foo=bar+%26+baz&saisons=%C3%89t%C3%A9%2Bhiver"));
    /// assert_eq!(url.as_str(),
    ///            "https://example.net/?foo=bar+%26+baz&saisons=%C3%89t%C3%A9%2Bhiver#nav");
    /// ```
    ///
    /// Note: `url.query_pairs_mut().clear();` is equivalent to `url.set_query(Some(""))`,
    /// not `url.set_query(None)`.
    ///
    /// The state of `Url` is unspecified if this return value is leaked without being dropped.
    pub fn query_pairs_mut(&mut self) -> form_urlencoded::Serializer<UrlQuery> {
        let fragment = self.take_fragment();

        let query_start;
        if let Some(start) = self.query_start {
            debug_assert!(self.byte_at(start) == b'?');
            query_start = start as usize;
        } else {
            query_start = self.serialization.len();
            self.query_start = Some(to_u32(query_start).unwrap());
            self.serialization.push('?');
        }

        let query = UrlQuery { url: self, fragment: fragment };
        form_urlencoded::Serializer::for_suffix(query, query_start + "?".len())
    }

    /// Change this URL’s path.
    pub fn set_path(&mut self, path: &str) {
        let (old_after_path_pos, after_path) = match (self.query_start, self.fragment_start) {
            (Some(i), _) | (None, Some(i)) => (i, self.slice(i..).to_owned()),
            (None, None) => (to_u32(self.serialization.len()).unwrap(), String::new())
        };
        let cannot_be_a_base = self.cannot_be_a_base();
        let scheme_type = SchemeType::from(self.scheme());
        self.serialization.truncate(self.path_start as usize);
        self.mutate(|parser| {
            if cannot_be_a_base {
                if path.starts_with('/') {
                    parser.serialization.push_str("%2F");
                    parser.parse_cannot_be_a_base_path(&path[1..]);
                } else {
                    parser.parse_cannot_be_a_base_path(path);
                }
            } else {
                let mut has_host = true;  // FIXME
                parser.parse_path_start(scheme_type, &mut has_host, path);
            }
        });
        let new_after_path_pos = to_u32(self.serialization.len()).unwrap();
        let adjust = |index: &mut u32| {
            *index -= old_after_path_pos;
            *index += new_after_path_pos;
        };
        if let Some(ref mut index) = self.query_start { adjust(index) }
        if let Some(ref mut index) = self.fragment_start { adjust(index) }
        self.serialization.push_str(&after_path)
    }

    /// Remove the last segment of this URL’s path.
    ///
    /// If this URL is cannot-be-a-base, do nothing and return `Err`.
    /// If this URL is not cannot-be-a-base and its path is `/`, do nothing and return `Ok`.
    // Temporarily private: https://github.com/servo/rust-url/issues/188
    /*pub*/ fn pop_path_segment(&mut self) -> Result<(), ()> {
        if self.cannot_be_a_base() {
            return Err(())
        }
        let last_slash;
        let path_len;
        {
            let path = self.path();
            last_slash = path.rfind('/').unwrap();
            path_len = path.len();
        };
        if last_slash > 0 {
            // Found a slash other than the initial one
            let last_slash = last_slash + self.path_start as usize;
            let path_end = path_len + self.path_start as usize;
            self.serialization.drain(last_slash..path_end);
            let offset = (path_end - last_slash) as u32;
            if let Some(ref mut index) = self.query_start { *index -= offset }
            if let Some(ref mut index) = self.fragment_start { *index -= offset }
        }
        Ok(())
    }

    /// Add a segment at the end of this URL’s path.
    ///
    /// If this URL is cannot-be-a-base, do nothing and return `Err`.
    // Temporarily private: https://github.com/servo/rust-url/issues/188
    /*pub*/ fn push_path_segment(&mut self, segment: &str) -> Result<(), ()> {
        if self.cannot_be_a_base() {
            return Err(())
        }
        let after_path = match (self.query_start, self.fragment_start) {
            (Some(i), _) | (None, Some(i)) => {
                let s = self.slice(i..).to_owned();
                self.serialization.truncate(i as usize);
                s
            },
            (None, None) => String::new()
        };
        let scheme_type = SchemeType::from(self.scheme());
        let path_start = self.path_start as usize;
        self.serialization.push('/');
        self.mutate(|parser| {
            parser.context = parser::Context::PathSegmentSetter;
            let mut has_host = true;  // FIXME account for this?
            parser.parse_path(scheme_type, &mut has_host, path_start, segment)
        });
        let offset = to_u32(self.serialization.len()).unwrap() - self.path_start;
        if let Some(ref mut index) = self.query_start { *index += offset }
        if let Some(ref mut index) = self.fragment_start { *index += offset }
        self.serialization.push_str(&after_path);
        Ok(())
    }

    /// Change this URL’s port number.
    ///
    /// If this URL is cannot-be-a-base, does not have a host, or has the `file` scheme;
    /// do nothing and return `Err`.
{'fraction_non_alphanumeric': 0.15636552980994534, 'fraction_numerical': 0.004616853250021696, 'mean_word_length': 2.2588235294117647, 'pattern_counts': {'":': 10, '<': 107, '<?xml version=': 0, '>': 253, 'https://': 20, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '556025', 'n_tokens_mistral': 18179, 'n_tokens_neox': 17746, 'n_words': 4385}