<commit_before>#[cfg(feature="bigdecimal")]
mod bigdecimal {
    extern crate num_traits;
    extern crate num_bigint;
    extern crate num_integer;
    extern crate bigdecimal;

    use std::error::Error;
    use std::io::prelude::*;

    use pg::Pg;

    use self::num_traits::{Signed, Zero, ToPrimitive};
    use self::num_bigint::{Sign, BigInt, BigUint, ToBigInt};
    use self::num_integer::Integer;
    use self::bigdecimal::BigDecimal;

    use pg::data_types::PgNumeric;
    use types::{self, FromSql, ToSql, IsNull};

    impl ToSql<types::Numeric, Pg> for BigDecimal {
        fn to_sql<W: Write>(&self, out: &mut W) -> Result<IsNull, Box<Error + Send + Sync>> {
            use std::str::FromStr;

            let absolute = self.abs();

            let mut digits = vec![];
            let ten_k = BigInt::from(10000);
            let mut integer_part = absolute.to_bigint().expect("Can always take integer part of BigDecimal");
            let decimal_part = &absolute;
            let mut decimal_part = decimal_part - absolute.with_scale(0);
            // scale is the amount of digits to print. to_string() includes a "0.",
            // that's why the -2 is there.
            let scale = if decimal_part == Zero::zero() {
                0
            } else {
                decimal_part.to_string().len() as u16 - 2
            };
            let mut weight = 0;<commit_msg>Add comment to decimal coding.<commit_after>#[cfg(feature="bigdecimal")]
mod bigdecimal {
    extern crate num_traits;
    extern crate num_bigint;
    extern crate num_integer;
    extern crate bigdecimal;

    use std::error::Error;
    use std::io::prelude::*;

    use pg::Pg;

    use self::num_traits::{Signed, Zero, ToPrimitive};
    use self::num_bigint::{Sign, BigInt, BigUint, ToBigInt};
    use self::num_integer::Integer;
    use self::bigdecimal::BigDecimal;

    use pg::data_types::PgNumeric;
    use types::{self, FromSql, ToSql, IsNull};

    impl ToSql<types::Numeric, Pg> for BigDecimal {
        fn to_sql<W: Write>(&self, out: &mut W) -> Result<IsNull, Box<Error + Send + Sync>> {
            use std::str::FromStr;

            // The encoding of the BigDecimal type for PostgreSQL is a bit complicated:
            // PostgreSQL expects the data in base-10000 (so two bytes per 10k),
            // and the decimal point should lie on a boundary (as per definition of "base-10000").

            // BigDecimal, internally, holds an int vector (base-256, one byte per byte),
            // and a base (u64, base-10) shift.

            // Therefore, we split up the encoding in three parts:
            // the sign, the (integer) part before the decimal, and the part after the decimal.

            let absolute = self.abs();

            let mut digits = vec![];
            let ten_k = BigInt::from(10000);
            let mut integer_part = absolute.to_bigint().expect("Can always take integer part of BigDecimal");
            let decimal_part = &absolute;
            let mut decimal_part = decimal_part - absolute.with_scale(0);
            // scale is the amount of digits to print. to_string() includes a "0.",
            // that's why the -2 is there.
            let scale = if decimal_part == Zero::zero() {
                0
            } else {
                decimal_part.to_string().len() as u16 - 2
            };
            let mut weight = 0;
{'fraction_non_alphanumeric': 0.11616908069760568, 'fraction_numerical': 0.013301803133313627, 'mean_word_length': 2.2790697674418605, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4601060', 'n_tokens_mistral': 1023, 'n_tokens_neox': 977, 'n_words': 318}