<commit_before>extern crate byteorder;
extern crate crc;
extern crate futures;
extern crate futures_cpupool;
extern crate inflate;
extern crate slog;

use byteorder::ReadBytesExt;

use crc::crc32::Hasher32;

use futures::Future;

use std::collections::VecDeque;

use std::io::{
    BufRead,
    Error,
    Read,
    Seek,
    SeekFrom,
    Write,
};

use std::str;

// 100 blocks of 64 kiB, even accounting for a huge overhead,
// is still less than 10 MiB, which is trivially manageable.
// Additionally, there's no chance that 100 threads or more
// give any speedup inflating blocks of at most 64 kiB.
const MAX_FUTURES: usize = 100;

const BUFFER_SIZE: u64 = 65536;

const GZIP_IDENTIFIER: [u8; 2] = [0x1f, 0x8b];
const BGZF_IDENTIFIER: [u8; 2] = [0x42, 0x43];

const DEFLATE: u8 = 8;

const FEXTRA: u8 = 1 << 2;

pub fn version() -> &'static str {
    return option_env!("CARGO_PKG_VERSION").unwrap_or("unknown");
}

pub trait Rescuable: BufRead + Seek {}
impl<T: BufRead + Seek> Rescuable for T {}

struct BGZFBlock {
    header_bytes: Vec<u8>,
    deflated_payload_bytes: Vec<u8>,
    inflated_payload_crc32: u32,
    inflated_payload_size: u32,
    corrupted: bool,
}

struct BGZFBlockStatus {
    corrupted: bool,
    inflated_payload_size: u32,
}

pub struct Results {
    pub blocks_count: u64,
    pub blocks_size: u64,
    pub bad_blocks_count: u64,
    pub bad_blocks_size: u64,
    pub truncated_in_block: bool,
    pub truncated_between_blocks: bool,
}

fn seek_next_block(reader: &mut Rescuable, block_position: u64) {
    let mut current_position = block_position;
    reader.seek(SeekFrom::Start(current_position)).unwrap();

    let mut bytes = vec![];

    'seek: loop {
        let mut buffer_reader = reader.take(BUFFER_SIZE);
        let buffer_size = buffer_reader.read_to_end(&mut bytes).unwrap();
        for window in bytes.windows(4) {
            let mut correct_bytes = 0;
            if window[0] == GZIP_IDENTIFIER[0] {
                correct_bytes += 1;
            }
            if window[1] == GZIP_IDENTIFIER[1] {
                correct_bytes += 1;
            }
            if window[2] == DEFLATE {
                correct_bytes += 1;
            }
            if window[3] == FEXTRA {
                correct_bytes += 1;
            }

            if correct_bytes >= 3 {
                break 'seek;
            }
            current_position += 1;
        }
        if buffer_size < BUFFER_SIZE as usize {
            return;
        }
        {
            let (beginning, end) = bytes.split_at_mut(4);
            beginning.copy_from_slice(&end[end.len() - 4..]);
        }
        bytes.resize(4, 0);
        current_position -= 4;
    }

    reader.seek(SeekFrom::Start(current_position)).unwrap();
}

fn process_payload(block: &Option<BGZFBlock>) -> Result<BGZFBlockStatus, Error> {
    match *block {
        None => Ok(BGZFBlockStatus {
            corrupted: false,
            inflated_payload_size: 0,
        }),
        Some(ref block) => {
            let inflated_payload_bytes = match inflate::inflate_bytes(&block.deflated_payload_bytes) {
                Ok(inflated_payload_bytes) => inflated_payload_bytes,
                Err(_) => return Ok(BGZFBlockStatus {
                    corrupted: true,
                    inflated_payload_size: block.inflated_payload_size,
                }),
            };

            let mut inflated_payload_digest = crc::crc32::Digest::new(crc::crc32::IEEE);
            inflated_payload_digest.write(&inflated_payload_bytes);
            let inflated_payload_crc32 = inflated_payload_digest.sum32();
            if inflated_payload_crc32 != block.inflated_payload_crc32 {
                return Ok(BGZFBlockStatus {
                    corrupted: true,
                    inflated_payload_size: block.inflated_payload_size,
                });<commit_msg>Remove workarounds that are not necessary anymore with Rust 1.26<commit_after>extern crate byteorder;
extern crate crc;
extern crate futures;
extern crate futures_cpupool;
extern crate inflate;
extern crate slog;

use byteorder::ReadBytesExt;

use crc::crc32::Hasher32;

use futures::Future;

use std::collections::VecDeque;

use std::io::{
    BufRead,
    Error,
    Read,
    Seek,
    SeekFrom,
    Write,
};

use std::str;

// 100 blocks of 64 kiB, even accounting for a huge overhead,
// is still less than 10 MiB, which is trivially manageable.
// Additionally, there's no chance that 100 threads or more
// give any speedup inflating blocks of at most 64 kiB.
const MAX_FUTURES: usize = 100;

const BUFFER_SIZE: u64 = 65536;

const GZIP_IDENTIFIER: [u8; 2] = [0x1f, 0x8b];
const BGZF_IDENTIFIER: [u8; 2] = [0x42, 0x43];

const DEFLATE: u8 = 8;

const FEXTRA: u8 = 1 << 2;

pub fn version() -> &'static str {
    return option_env!("CARGO_PKG_VERSION").unwrap_or("unknown");
}

pub trait Rescuable: BufRead + Seek {}
impl<T: BufRead + Seek> Rescuable for T {}

struct BGZFBlock {
    header_bytes: Vec<u8>,
    deflated_payload_bytes: Vec<u8>,
    inflated_payload_crc32: u32,
    inflated_payload_size: u32,
    corrupted: bool,
}

struct BGZFBlockStatus {
    corrupted: bool,
    inflated_payload_size: u32,
}

pub struct Results {
    pub blocks_count: u64,
    pub blocks_size: u64,
    pub bad_blocks_count: u64,
    pub bad_blocks_size: u64,
    pub truncated_in_block: bool,
    pub truncated_between_blocks: bool,
}

fn seek_next_block(reader: &mut Rescuable, block_position: u64) {
    let mut current_position = block_position;
    reader.seek(SeekFrom::Start(current_position)).unwrap();

    let mut bytes = vec![];

    'seek: loop {
        let mut buffer_reader = reader.take(BUFFER_SIZE);
        let buffer_size = buffer_reader.read_to_end(&mut bytes).unwrap();
        for window in bytes.windows(4) {
            let mut correct_bytes = 0;
            if window[0] == GZIP_IDENTIFIER[0] {
                correct_bytes += 1;
            }
            if window[1] == GZIP_IDENTIFIER[1] {
                correct_bytes += 1;
            }
            if window[2] == DEFLATE {
                correct_bytes += 1;
            }
            if window[3] == FEXTRA {
                correct_bytes += 1;
            }

            if correct_bytes >= 3 {
                break 'seek;
            }
            current_position += 1;
        }
        if buffer_size < BUFFER_SIZE as usize {
            return;
        }
        {
            let (beginning, end) = bytes.split_at_mut(4);
            beginning.copy_from_slice(&end[end.len() - 4..]);
        }
        bytes.resize(4, 0);
        current_position -= 4;
    }

    reader.seek(SeekFrom::Start(current_position)).unwrap();
}

fn process_payload(block: &Option<BGZFBlock>) -> Result<BGZFBlockStatus, Error> {
    match block {
        None => Ok(BGZFBlockStatus {
            corrupted: false,
            inflated_payload_size: 0,
        }),
        Some(block) => {
            let inflated_payload_bytes = match inflate::inflate_bytes(&block.deflated_payload_bytes) {
                Ok(inflated_payload_bytes) => inflated_payload_bytes,
                Err(_) => return Ok(BGZFBlockStatus {
                    corrupted: true,
                    inflated_payload_size: block.inflated_payload_size,
                }),
            };

            let mut inflated_payload_digest = crc::crc32::Digest::new(crc::crc32::IEEE);
            inflated_payload_digest.write(&inflated_payload_bytes);
            let inflated_payload_crc32 = inflated_payload_digest.sum32();
            if inflated_payload_crc32 != block.inflated_payload_crc32 {
                return Ok(BGZFBlockStatus {
                    corrupted: true,
                    inflated_payload_size: block.inflated_payload_size,
                });
{'fraction_non_alphanumeric': 0.10208816705336426, 'fraction_numerical': 0.025393142562516114, 'mean_word_length': 2.3472821397756687, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1483762', 'n_tokens_mistral': 2661, 'n_tokens_neox': 2467, 'n_words': 576}