<commit_before>use std::env;
use std::io::Read;
use std::ascii::AsciiExt;
use std::default::Default;
use std::collections::BTreeMap;

use url::Url;
use tendril::SliceExt;
use hyper::header::UserAgent;
use hyper::Client as HyperClient;
use hyper::client::RedirectPolicy;
use hyper::client::Response as HttpResponse;
use hyper::status::StatusCode;
use robotparser::RobotFileParser;
use html5ever::tokenizer::Tokenizer;
use sidekiq::Client as SidekiqClient;
use sidekiq::ClientOpts as SidekiqClientOpts;
use sidekiq::create_redis_pool;
use encoding::{Encoding, DecoderTrap};
use encoding::all::UTF_8;

pub use self::page::Page;
mod page;

const MAMAN_ENV: &'static str = "MAMAN_ENV";

pub struct Spider<'a> {
    pub base_url: Url,
    pub visited_urls: Vec<Url>,
    pub unvisited_urls: Vec<Url>,
    pub extra: Vec<String>,
    pub env: String,
    pub limit: isize,
    sidekiq: SidekiqClient,
    robot_parser: RobotFileParser<'a>,
}

impl<'a> Spider<'a> {
    pub fn new(base_url: Url, limit: isize, extra: Vec<String>) -> Spider<'a> {
        let maman_env = env::var(&MAMAN_ENV.to_string()).unwrap_or("development".to_string());
        let robots_txt = base_url.join("/robots.txt").unwrap();
        let robot_file_parser = RobotFileParser::new(robots_txt);
        let client_opts =
            SidekiqClientOpts { namespace: Some(maman_env.to_string()), ..Default::default() };
        let sidekiq = SidekiqClient::new(create_redis_pool(), client_opts);
        Spider {
            base_url: base_url,
            visited_urls: Vec::new(),
            unvisited_urls: Vec::new(),
            extra: extra,
            sidekiq: sidekiq,
            env: maman_env,
            robot_parser: robot_file_parser,
            limit: limit,
        }
    }

    pub fn visit_page(&mut self, page: Page) {
        self.visited_urls.push(page.url.clone());
        for u in page.urls.iter() {
            self.unvisited_urls.push(u.clone());
        }
        match self.sidekiq.push(page.to_job()) {
            Err(err) => {
                println!("SidekiqClient push failed: {}", err);<commit_msg>Print sidekiq error to stderr<commit_after>use std::env;
use std::io::{Read, Write};
use std::ascii::AsciiExt;
use std::default::Default;
use std::collections::BTreeMap;

use url::Url;
use tendril::SliceExt;
use hyper::header::UserAgent;
use hyper::Client as HyperClient;
use hyper::client::RedirectPolicy;
use hyper::client::Response as HttpResponse;
use hyper::status::StatusCode;
use robotparser::RobotFileParser;
use html5ever::tokenizer::Tokenizer;
use sidekiq::Client as SidekiqClient;
use sidekiq::ClientOpts as SidekiqClientOpts;
use sidekiq::create_redis_pool;
use encoding::{Encoding, DecoderTrap};
use encoding::all::UTF_8;

pub use self::page::Page;
mod page;

const MAMAN_ENV: &'static str = "MAMAN_ENV";

macro_rules! println_err(
    ($($arg:tt)*) => { {
        let r = writeln!(&mut ::std::io::stderr(), $($arg)*);
        r.expect("failed printing to stderr");
    } }
    );

pub struct Spider<'a> {
    pub base_url: Url,
    pub visited_urls: Vec<Url>,
    pub unvisited_urls: Vec<Url>,
    pub extra: Vec<String>,
    pub env: String,
    pub limit: isize,
    sidekiq: SidekiqClient,
    robot_parser: RobotFileParser<'a>,
}

impl<'a> Spider<'a> {
    pub fn new(base_url: Url, limit: isize, extra: Vec<String>) -> Spider<'a> {
        let maman_env = env::var(&MAMAN_ENV.to_string()).unwrap_or("development".to_string());
        let robots_txt = base_url.join("/robots.txt").unwrap();
        let robot_file_parser = RobotFileParser::new(robots_txt);
        let client_opts =
            SidekiqClientOpts { namespace: Some(maman_env.to_string()), ..Default::default() };
        let sidekiq = SidekiqClient::new(create_redis_pool(), client_opts);
        Spider {
            base_url: base_url,
            visited_urls: Vec::new(),
            unvisited_urls: Vec::new(),
            extra: extra,
            sidekiq: sidekiq,
            env: maman_env,
            robot_parser: robot_file_parser,
            limit: limit,
        }
    }

    pub fn visit_page(&mut self, page: Page) {
        self.visited_urls.push(page.url.clone());
        for u in page.urls.iter() {
            self.unvisited_urls.push(u.clone());
        }
        match self.sidekiq.push(page.to_job()) {
            Err(err) => {
                println_err!("SidekiqClient push failed: {}", err);
{'fraction_non_alphanumeric': 0.1417555001134044, 'fraction_numerical': 0.0009072352007257881, 'mean_word_length': 3.353405725567621, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '183040', 'n_tokens_mistral': 1467, 'n_tokens_neox': 1496, 'n_words': 312}