<commit_before>pub mod session;

pub(crate) mod atcoder;
pub(crate) mod hackerrank;
pub(crate) mod yukicoder;

pub(self) mod download;

use crate::config::Config;
use crate::errors::{FileErrorKind, FileResult, ServiceResult};
use crate::path::{AbsPath, AbsPathBuf};
use crate::service::session::{HttpSession, UrlBase};
use crate::template::Template;
use crate::terminal::{Term, WriteAnsi};
use crate::testsuite::DownloadDestinations;
use crate::util;

use failure::ResultExt as _ResultExt;
use heck::KebabCase as _KebabCase;
use maplit::hashmap;
use rayon::iter::{
    IntoParallelIterator as _IntoParalleIterator, ParallelIterator as _ParallelIterator,
};
use regex::Regex;
use reqwest::header::{self, HeaderMap};
use reqwest::RedirectPolicy;
use serde_derive::{Deserialize, Serialize};
use strum_macros::{AsStaticStr, EnumString};
use tokio::runtime::Runtime;
use url::Host;
use zip::result::ZipResult;
use zip::ZipArchive;

use std::collections::HashMap;
use std::io::{self, Cursor, Write as _Write};
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use std::{cmp, fmt, mem, slice};

#[derive(
    Clone,
    Copy,
    PartialEq,
    Eq,
    PartialOrd,
    Ord,
    Debug,
    strum_macros::Display,
    AsStaticStr,
    EnumString,
    Serialize,
    Deserialize,
)]
#[serde(rename_all = "lowercase")]
pub enum ServiceName {
    #[strum(to_string = "atcoder")]
    Atcoder,
    #[strum(to_string = "hackerrank")]
    Hackerrank,
    #[strum(to_string = "yukicoder")]
    Yukicoder,
    #[strum(to_string = "other")]
    Other,
}

impl Default for ServiceName {
    fn default() -> Self {
        ServiceName::Other
    }
}

impl ServiceName {
    pub(crate) fn domain(self) -> Option<&'static str> {
        match self {
            ServiceName::Atcoder => Some("atcoder.jp"),
            ServiceName::Hackerrank => Some("www.hackerrank.com"),
            ServiceName::Yukicoder => Some("yukicoder.me"),
            ServiceName::Other => None,
        }
    }
}

pub(self) static USER_AGENT: &str = "snowchains <https://github.com/wariuni/snowchains>";

pub(self) trait Service {
    type Write: WriteAnsi;

    fn requirements(&mut self) -> (&mut Self::Write, &mut HttpSession, &mut Runtime);

    fn get(&mut self, url: &str) -> session::Request<&mut Self::Write> {
        let (out, sess, runtime) = self.requirements();
        sess.get(url, out, runtime)
    }

    fn post(&mut self, url: &str) -> session::Request<&mut Self::Write> {
        let (out, sess, runtime) = self.requirements();
        sess.post(url, out, runtime)
    }

    fn open_in_browser(&mut self, url: &str) -> ServiceResult<()> {
        let (out, sess, _) = self.requirements();
        sess.open_in_browser(url, out)
    }
}

pub(self) trait ExtractZip {
    type Write: WriteAnsi;

    fn out(&mut self) -> &mut Self::Write;

    fn extract_zip(
        &mut self,
        name: &str,
        zip: &[u8],
        dir: &AbsPath,
        entries: &'static ZipEntries,
    ) -> FileResult<Vec<(String, AbsPathBuf, AbsPathBuf)>> {
        let out = self.out();
        let ZipEntries {
            in_entry,
            in_match_group,
            in_crlf_to_lf,
            out_entry,
            out_match_group,
            out_crlf_to_lf,
            sortings,
        } = entries;

        out.with_reset(|o| o.bold()?.write_str(name))?;
        out.write_str(": Unzipping...\n")?;
        out.flush()?;

        let zip = ZipArchive::new(Cursor::new(zip)).with_context(|_| FileErrorKind::ReadZip)?;
        let pairs = Arc::new(Mutex::new(hashmap!()));

        (0..zip.len())
            .into_par_iter()
            .map(|i| {
                let mut zip = zip.clone();
                let (filename, content) = {
                    let file = zip.by_index(i)?;
                    let filename = file.sanitized_name().to_string_lossy().into_owned();
                    let cap = file.size() as usize + 1;
                    let content = util::string_from_read(file, cap)?;
                    (filename, content)
                };
                if let Some(caps) = in_entry.captures(&filename) {
                    let name = caps.get(*in_match_group).unwrap().as_str().to_owned();
                    let content = if *in_crlf_to_lf && content.contains("\r\n") {
                        content.replace("\r\n", "\n")
                    } else {
                        content
                    };
                    let filename = PathBuf::from(&filename);
                    let mut pairs = pairs.lock().unwrap();
                    if let Some((_, output)) = pairs.remove(&name) {
                        pairs.insert(name, (Some((filename, content)), output));
                    } else {
                        pairs.insert(name, (Some((filename, content)), None));
                    }
                } else if let Some(caps) = out_entry.captures(&filename) {
                    let name = caps.get(*out_match_group).unwrap().as_str().to_owned();
                    let content = if *out_crlf_to_lf && content.contains("\r\n") {
                        content.replace("\r\n", "\n")
                    } else {
                        content
                    };
                    let filename = PathBuf::from(filename);
                    let mut pairs = pairs.lock().unwrap();
                    if let Some((input, _)) = pairs.remove(&name) {
                        pairs.insert(name, (input, Some((filename, content))));
                    } else {
                        pairs.insert(name, (None, Some((filename, content))));
                    }
                }
                Ok(())
            })
            .collect::<ZipResult<()>>()
            .with_context(|_| FileErrorKind::ReadZip)?;

        let mut cases = mem::replace::<HashMap<_, _>>(&mut pairs.lock().unwrap(), hashmap!())
            .into_iter()
            .filter_map(|(name, (input, output))| match (input, output) {
                (Some(input), Some(output)) => Some((name, input, output)),
                _ => None,<commit_msg>Fix ExtractZip::extract_zip<commit_after>pub mod session;

pub(crate) mod atcoder;
pub(crate) mod hackerrank;
pub(crate) mod yukicoder;

pub(self) mod download;

use crate::config::Config;
use crate::errors::{FileErrorKind, FileResult, ServiceResult};
use crate::path::{AbsPath, AbsPathBuf};
use crate::service::session::{HttpSession, UrlBase};
use crate::template::Template;
use crate::terminal::{Term, WriteAnsi};
use crate::testsuite::DownloadDestinations;
use crate::util;

use failure::ResultExt as _ResultExt;
use heck::KebabCase as _KebabCase;
use maplit::hashmap;
use rayon::iter::{
    IntoParallelIterator as _IntoParalleIterator, ParallelIterator as _ParallelIterator,
};
use regex::Regex;
use reqwest::header::{self, HeaderMap};
use reqwest::RedirectPolicy;
use serde_derive::{Deserialize, Serialize};
use strum_macros::{AsStaticStr, EnumString};
use tokio::runtime::Runtime;
use url::Host;
use zip::result::ZipResult;
use zip::ZipArchive;

use std::collections::HashMap;
use std::io::{self, Cursor, Write as _Write};
use std::ops::Deref;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use std::{cmp, fmt, mem, slice};

#[derive(
    Clone,
    Copy,
    PartialEq,
    Eq,
    PartialOrd,
    Ord,
    Debug,
    strum_macros::Display,
    AsStaticStr,
    EnumString,
    Serialize,
    Deserialize,
)]
#[serde(rename_all = "lowercase")]
pub enum ServiceName {
    #[strum(to_string = "atcoder")]
    Atcoder,
    #[strum(to_string = "hackerrank")]
    Hackerrank,
    #[strum(to_string = "yukicoder")]
    Yukicoder,
    #[strum(to_string = "other")]
    Other,
}

impl Default for ServiceName {
    fn default() -> Self {
        ServiceName::Other
    }
}

impl ServiceName {
    pub(crate) fn domain(self) -> Option<&'static str> {
        match self {
            ServiceName::Atcoder => Some("atcoder.jp"),
            ServiceName::Hackerrank => Some("www.hackerrank.com"),
            ServiceName::Yukicoder => Some("yukicoder.me"),
            ServiceName::Other => None,
        }
    }
}

pub(self) static USER_AGENT: &str = "snowchains <https://github.com/wariuni/snowchains>";

pub(self) trait Service {
    type Write: WriteAnsi;

    fn requirements(&mut self) -> (&mut Self::Write, &mut HttpSession, &mut Runtime);

    fn get(&mut self, url: &str) -> session::Request<&mut Self::Write> {
        let (out, sess, runtime) = self.requirements();
        sess.get(url, out, runtime)
    }

    fn post(&mut self, url: &str) -> session::Request<&mut Self::Write> {
        let (out, sess, runtime) = self.requirements();
        sess.post(url, out, runtime)
    }

    fn open_in_browser(&mut self, url: &str) -> ServiceResult<()> {
        let (out, sess, _) = self.requirements();
        sess.open_in_browser(url, out)
    }
}

pub(self) trait ExtractZip {
    type Write: WriteAnsi;

    fn out(&mut self) -> &mut Self::Write;

    fn extract_zip(
        &mut self,
        name: &str,
        zip: &[u8],
        dir: &AbsPath,
        entries: &'static ZipEntries,
    ) -> FileResult<Vec<(String, AbsPathBuf, AbsPathBuf)>> {
        let out = self.out();
        let ZipEntries {
            in_entry,
            in_match_group,
            in_crlf_to_lf,
            out_entry,
            out_match_group,
            out_crlf_to_lf,
            sortings,
        } = entries;

        out.with_reset(|o| o.bold()?.write_str(name))?;
        out.write_str(": Unzipping...\n")?;
        out.flush()?;

        let zip = ZipArchive::new(Cursor::new(zip)).with_context(|_| FileErrorKind::ReadZip)?;
        let pairs = Arc::new(Mutex::new(hashmap!()));

        (0..zip.len())
            .into_par_iter()
            .map(|i| {
                let mut zip = zip.clone();
                let (filename, filename_sanitized, content) = {
                    let file = zip.by_index(i)?;
                    let filename = file.name().to_owned();
                    let filename_sanitized = file.sanitized_name();
                    let cap = file.size() as usize + 1;
                    let content = util::string_from_read(file, cap)?;
                    (filename, filename_sanitized, content)
                };
                if let Some(caps) = in_entry.captures(&filename) {
                    let name = caps.get(*in_match_group).unwrap().as_str().to_owned();
                    let content = if *in_crlf_to_lf && content.contains("\r\n") {
                        content.replace("\r\n", "\n")
                    } else {
                        content
                    };
                    let mut pairs = pairs.lock().unwrap();
                    if let Some((_, output)) = pairs.remove(&name) {
                        pairs.insert(name, (Some((filename_sanitized, content)), output));
                    } else {
                        pairs.insert(name, (Some((filename_sanitized, content)), None));
                    }
                } else if let Some(caps) = out_entry.captures(&filename) {
                    let name = caps.get(*out_match_group).unwrap().as_str().to_owned();
                    let content = if *out_crlf_to_lf && content.contains("\r\n") {
                        content.replace("\r\n", "\n")
                    } else {
                        content
                    };
                    let mut pairs = pairs.lock().unwrap();
                    if let Some((input, _)) = pairs.remove(&name) {
                        pairs.insert(name, (input, Some((filename_sanitized, content))));
                    } else {
                        pairs.insert(name, (None, Some((filename_sanitized, content))));
                    }
                }
                Ok(())
            })
            .collect::<ZipResult<()>>()
            .with_context(|_| FileErrorKind::ReadZip)?;

        let mut cases = mem::replace::<HashMap<_, _>>(&mut pairs.lock().unwrap(), hashmap!())
            .into_iter()
            .filter_map(|(name, (input, output))| match (input, output) {
                (Some(input), Some(output)) => Some((name, input, output)),
                _ => None,
{'fraction_non_alphanumeric': 0.14786823297461596, 'fraction_numerical': 0.0004928941099153865, 'mean_word_length': 2.1327843540916107, 'pattern_counts': {'":': 2, '<': 25, '<?xml version=': 0, '>': 53, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5960025', 'n_tokens_mistral': 3765, 'n_tokens_neox': 3693, 'n_words': 783}