<commit_before>use std::env;
use std::fs;
use std::io::Read;
use std::path::PathBuf;

#[cfg(unix)]
use std::os::unix::fs::MetadataExt;

use reqwest;
use flate2::read::GzDecoder;
use tar::Archive;
use time;
use walkdir::{DirEntry, WalkDir};
use xdg::BaseDirectories;

use error::TealdeerError::{self, CacheError, UpdateError};
use types::OsType;

#[derive(Debug)]
pub struct Cache {
    url: String,
    os: OsType,
}

impl Cache {
    pub fn new<S>(url: S, os: OsType) -> Cache
    where
        S: Into<String>,
    {
        Cache {
            url: url.into(),
            os: os,
        }
    }

    /// Return the path to the cache directory.
    fn get_cache_dir(&self) -> Result<PathBuf, TealdeerError> {
        // Allow overriding the cache directory by setting the
        // $TEALDEER_CACHE_DIR env variable.
        if let Ok(value) = env::var("TEALDEER_CACHE_DIR") {
            let path = PathBuf::from(value);

            if path.exists() && path.is_dir() {
                return Ok(path);
            } else {
                return Err(CacheError(
                    "Path specified by $TEALDEER_CACHE_DIR \
                     does not exist or is not a directory."
                        .into(),
                ));
            }
        };

        // Otherwise, fall back to $XDG_CACHE_HOME/tealdeer.
        let xdg_dirs = match BaseDirectories::with_prefix(::NAME) {
            Ok(dirs) => dirs,
            Err(_) => return Err(CacheError("Could not determine XDG base directory.".into())),
        };
        Ok(xdg_dirs.get_cache_home())
    }

    /// Download the archive
    fn download(&self) -> Result<Vec<u8>, TealdeerError> {
        let mut resp = reqwest::get(&self.url)?;
        let mut buf: Vec<u8> = vec![];
        let bytes_downloaded = resp.copy_to(&mut buf)?;
        debug!("{} bytes downloaded", bytes_downloaded);
        Ok(buf)
    }

    /// Decompress and open the archive
    fn decompress<R: Read>(&self, reader: R) -> Archive<GzDecoder<R>> {<commit_msg>Add (back) support for proxies (#68)<commit_after>use std::env;
use std::fs;
use std::io::Read;
use std::path::PathBuf;

#[cfg(unix)]
use std::os::unix::fs::MetadataExt;

use reqwest::{Client, Proxy};
use flate2::read::GzDecoder;
use tar::Archive;
use time;
use walkdir::{DirEntry, WalkDir};
use xdg::BaseDirectories;

use error::TealdeerError::{self, CacheError, UpdateError};
use types::OsType;

#[derive(Debug)]
pub struct Cache {
    url: String,
    os: OsType,
}

impl Cache {
    pub fn new<S>(url: S, os: OsType) -> Cache
    where
        S: Into<String>,
    {
        Cache {
            url: url.into(),
            os: os,
        }
    }

    /// Return the path to the cache directory.
    fn get_cache_dir(&self) -> Result<PathBuf, TealdeerError> {
        // Allow overriding the cache directory by setting the
        // $TEALDEER_CACHE_DIR env variable.
        if let Ok(value) = env::var("TEALDEER_CACHE_DIR") {
            let path = PathBuf::from(value);

            if path.exists() && path.is_dir() {
                return Ok(path);
            } else {
                return Err(CacheError(
                    "Path specified by $TEALDEER_CACHE_DIR \
                     does not exist or is not a directory."
                        .into(),
                ));
            }
        };

        // Otherwise, fall back to $XDG_CACHE_HOME/tealdeer.
        let xdg_dirs = match BaseDirectories::with_prefix(::NAME) {
            Ok(dirs) => dirs,
            Err(_) => return Err(CacheError("Could not determine XDG base directory.".into())),
        };
        Ok(xdg_dirs.get_cache_home())
    }

    /// Download the archive
    fn download(&self) -> Result<Vec<u8>, TealdeerError> {
        let mut builder = Client::builder();
        if let Ok(ref host) = env::var("HTTP_PROXY") {
            if let Ok(proxy) = Proxy::http(host) {
                builder = builder.proxy(proxy);
            }
        }
        if let Ok(ref host) = env::var("HTTPS_PROXY") {
            if let Ok(proxy) = Proxy::https(host) {
                builder = builder.proxy(proxy);
            }
        }
        let client = builder.build().unwrap_or_else(|_| Client::new());
        let mut resp = client.get(&self.url).send()?;
        let mut buf: Vec<u8> = vec![];
        let bytes_downloaded = resp.copy_to(&mut buf)?;
        debug!("{} bytes downloaded", bytes_downloaded);
        Ok(buf)
    }

    /// Decompress and open the archive
    fn decompress<R: Read>(&self, reader: R) -> Archive<GzDecoder<R>> {
{'fraction_non_alphanumeric': 0.13940193491644678, 'fraction_numerical': 0.001759014951627089, 'mean_word_length': 2.1093643198906356, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1399353', 'n_tokens_mistral': 1451, 'n_tokens_neox': 1426, 'n_words': 358}