<commit_before>// This is a part of rust-encoding.
// Copyright (c) 2013-2014, <NAME>.
// See README.md and LICENSE.txt for details.

/*!

# Rust-encoding

Character encoding support for Rust.
It is based on [WHATWG Encoding Standard](http://encoding.spec.whatwg.org/),
and also provides an advanced interface for error detection and recovery.

## Simple Usage

To encode a string:

~~~~ {.rust}
use encoding::{Encoding, EncodeStrict};
use encoding::all::ISO_8859_1;

assert_eq!(ISO_8859_1.encode("caf\xe9", EncodeStrict),
           Ok(vec!(99,97,102,233)));
~~~~

To encode a string with unrepresentable characters:

~~~~ {.rust}
use encoding::{Encoding, EncodeStrict, EncodeReplace, EncodeIgnore, EncodeNcrEscape};
use encoding::all::ISO_8859_2;

assert!(ISO_8859_2.encode("Acme\xa9", EncodeStrict).is_err());
assert_eq!(ISO_8859_2.encode("Acme\xa9", EncodeReplace),
           Ok(vec!(65,99,109,101,63)));
assert_eq!(ISO_8859_2.encode("Acme\xa9", EncodeIgnore),
           Ok(vec!(65,99,109,101)));
assert_eq!(ISO_8859_2.encode("Acme\xa9", EncodeNcrEscape),
           Ok(vec!(65,99,109,101,38,35,49,54,57,59)));
~~~~

To decode a byte sequence:

~~~~ {.rust}
use encoding::{Encoding, DecodeStrict};
use encoding::all::ISO_8859_1;

assert_eq!(ISO_8859_1.decode([99,97,102,233], DecodeStrict),
           Ok("caf\xe9".to_string()));
~~~~

To decode a byte sequence with invalid sequences:

~~~~ {.rust}
use encoding::{Encoding, DecodeStrict, DecodeReplace, DecodeIgnore};
use encoding::all::ISO_8859_6;

assert!(ISO_8859_6.decode([65,99,109,101,169], DecodeStrict).is_err());
assert_eq!(ISO_8859_6.decode([65,99,109,101,169], DecodeReplace),
           Ok("Acme\ufffd".to_string()));
assert_eq!(ISO_8859_6.decode([65,99,109,101,169], DecodeIgnore),
           Ok("Acme".to_string()));
~~~~

A practical example of custom encoder traps:

~~~~ {.rust}
use encoding::{Encoding, Encoder, ByteWriter, EncoderTrap, EncoderCall, DecodeStrict};
use encoding::all::ASCII;

// hexadecimal numeric character reference replacement
fn hex_ncr_escape(_encoder: &mut Encoder, input: &str, output: &mut ByteWriter) -> bool {
    let escapes: Vec<String> =
        input.chars().map(|ch| format!("&#x{:x};", ch as int)).collect();
    let escapes = escapes.concat();
    output.write_bytes(escapes.as_bytes());
    true
}<commit_msg>doctest needs #![feature] as well.<commit_after>// This is a part of rust-encoding.
// Copyright (c) 2013-2014, <NAME>.
// See README.md and LICENSE.txt for details.

/*!

# Rust-encoding

Character encoding support for Rust.
It is based on [WHATWG Encoding Standard](http://encoding.spec.whatwg.org/),
and also provides an advanced interface for error detection and recovery.

## Simple Usage

To encode a string:

~~~~ {.rust}
use encoding::{Encoding, EncodeStrict};
use encoding::all::ISO_8859_1;

assert_eq!(ISO_8859_1.encode("caf\xe9", EncodeStrict),
           Ok(vec!(99,97,102,233)));
~~~~

To encode a string with unrepresentable characters:

~~~~ {.rust}
use encoding::{Encoding, EncodeStrict, EncodeReplace, EncodeIgnore, EncodeNcrEscape};
use encoding::all::ISO_8859_2;

assert!(ISO_8859_2.encode("Acme\xa9", EncodeStrict).is_err());
assert_eq!(ISO_8859_2.encode("Acme\xa9", EncodeReplace),
           Ok(vec!(65,99,109,101,63)));
assert_eq!(ISO_8859_2.encode("Acme\xa9", EncodeIgnore),
           Ok(vec!(65,99,109,101)));
assert_eq!(ISO_8859_2.encode("Acme\xa9", EncodeNcrEscape),
           Ok(vec!(65,99,109,101,38,35,49,54,57,59)));
~~~~

To decode a byte sequence:

~~~~ {.rust}
use encoding::{Encoding, DecodeStrict};
use encoding::all::ISO_8859_1;

assert_eq!(ISO_8859_1.decode([99,97,102,233], DecodeStrict),
           Ok("caf\xe9".to_string()));
~~~~

To decode a byte sequence with invalid sequences:

~~~~ {.rust}
use encoding::{Encoding, DecodeStrict, DecodeReplace, DecodeIgnore};
use encoding::all::ISO_8859_6;

assert!(ISO_8859_6.decode([65,99,109,101,169], DecodeStrict).is_err());
assert_eq!(ISO_8859_6.decode([65,99,109,101,169], DecodeReplace),
           Ok("Acme\ufffd".to_string()));
assert_eq!(ISO_8859_6.decode([65,99,109,101,169], DecodeIgnore),
           Ok("Acme".to_string()));
~~~~

A practical example of custom encoder traps:

~~~~ {.rust}
# #![feature(slicing_syntax)]
use encoding::{Encoding, Encoder, ByteWriter, EncoderTrap, EncoderCall, DecodeStrict};
use encoding::all::ASCII;

// hexadecimal numeric character reference replacement
fn hex_ncr_escape(_encoder: &mut Encoder, input: &str, output: &mut ByteWriter) -> bool {
    let escapes: Vec<String> =
        input.chars().map(|ch| format!("&#x{:x};", ch as int)).collect();
    let escapes = escapes.concat();
    output.write_bytes(escapes.as_bytes());
    true
}
{'fraction_non_alphanumeric': 0.1744508423970996, 'fraction_numerical': 0.07762849221582427, 'mean_word_length': 5.389645776566757, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7515731', 'n_tokens_mistral': 1857, 'n_tokens_neox': 1626, 'n_words': 337}