<commit_before>//!
//! robots.txt parser for Rust
//!
//! The robots.txt Exclusion Protocol is implemented as specified in
//! http://www.robotstxt.org/norobots-rfc.txt
//!
extern crate url;
extern crate time;
extern crate hyper;

use std::io::Read;
use std::cell::{Cell, RefCell};
use url::Url;
use hyper::{Client};
use hyper::status::StatusCode;

/// A rule line is a single "Allow:" (allowance==True) or "Disallow:"
/// (allowance==False) followed by a path."""
#[derive(Debug, Eq, PartialEq, Clone)]
struct RuleLine {
    path: String,
    allowance: bool,
}

/// An entry has one or more user-agents and zero or more rulelines
#[derive(Debug, Eq, PartialEq, Clone)]
struct Entry {
    useragents: RefCell<Vec<String>>,
    rulelines: RefCell<Vec<RuleLine>>,
}

#[derive(Debug, Eq, PartialEq, Clone)]
pub struct RobotFileParser {
    entries: RefCell<Vec<Entry>>,
    default_entry: RefCell<Entry>,
    disallow_all: Cell<bool>,
    allow_all: Cell<bool>,
    url: Url,
    host: String,
    path: String,
    last_checked: Cell<i64>,
}


impl RuleLine {
    fn new(path: &str, allowance: bool) -> RuleLine {
        let mut allow = allowance;
        if path == "" && !allowance {
            // an empty value means allow all
            allow = true;
        }
        RuleLine {
            path: path.to_owned(),
            allowance: allow,
        }
    }

    fn applies_to(&self, filename: &str) -> bool {
        &self.path == "*" || filename.starts_with(&self.path)
    }
}


impl Entry {
    fn new() -> Entry {
        Entry {
            useragents: RefCell::new(vec![]),
            rulelines: RefCell::new(vec![]),
        }
    }

    /// check if this entry applies to the specified agent
    fn applies_to(&self, useragent: &str) -> bool {
        let ua = useragent.split("/").nth(0).unwrap_or("").to_lowercase();
        let useragents = self.useragents.borrow();
        for agent in &*useragents {
            if agent == "*" {
                return true;
            }
            if ua.contains(agent) {
                return true;
            }
        }
        false
    }

    
    /// Preconditions:
    /// - our agent applies to this entry
    /// - filename is URL decoded
    fn allowance(&self, filename: &str) -> bool {
        let rulelines = self.rulelines.borrow();
        for line in &*rulelines {
            if line.applies_to(filename) {
                return line.allowance
            }
        }
        true
    }

    fn push_useragent(&self, useragent: &str) {
        let mut useragents = self.useragents.borrow_mut();
        useragents.push(useragent.to_lowercase().to_owned());
    }

    fn push_ruleline(&self, ruleline: RuleLine) {
        let mut rulelines = self.rulelines.borrow_mut();
        rulelines.push(ruleline);
    }

    fn has_useragent(&self, useragent: &str) -> bool {
        let useragents = self.useragents.borrow();
        useragents.contains(&useragent.to_owned())
    }

    fn is_empty(&self) -> bool {
        let useragents = self.useragents.borrow();
        let rulelines = self.rulelines.borrow();
        useragents.is_empty() && rulelines.is_empty()
    }
}


impl RobotFileParser {
    pub fn new(url: &str) -> RobotFileParser {
        let parsed_url = Url::parse(url).unwrap();
        RobotFileParser {
            entries: RefCell::new(vec![]),
            default_entry: RefCell::new(Entry::new()),
            disallow_all: Cell::new(false),
            allow_all: Cell::new(false),
            url: parsed_url.clone(),
            host: parsed_url.domain().unwrap().to_owned(),
            path: parsed_url.path().unwrap().connect("/"),
            last_checked: Cell::new(0i64),
        }
    }

    /// Returns the time the robots.txt file was last fetched.
    ///
    /// This is useful for long-running web spiders that need to
    /// check for new robots.txt files periodically.
    pub fn mtime(&self) -> i64 {
        self.last_checked.get()
    }

    /// Sets the time the robots.txt file was last fetched to the
    /// current time.
    pub fn modified(&self) {
        use time::get_time;

        self.last_checked.set(get_time().sec);
    }

    /// Sets the URL referring to a robots.txt file.
    pub fn set_url(&mut self, url: &str) {
        let parsed_url = Url::parse(url).unwrap();
        self.url = parsed_url.clone();
        self.host = parsed_url.domain().unwrap().to_owned();
        self.path = parsed_url.path().unwrap().connect("/");
        self.last_checked.set(0i64);
    }

    /// Reads the robots.txt URL and feeds it to the parser.
    pub fn read(&self) {
        let client = Client::new();
        let mut res = match client.get(self.url.clone()).send() {
            Ok(res) => res,
            Err(_) => {
                return;
            }
        };
        match res.status {<commit_msg>Use join instead of connect<commit_after>//!
//! robots.txt parser for Rust
//!
//! The robots.txt Exclusion Protocol is implemented as specified in
//! http://www.robotstxt.org/norobots-rfc.txt
//!
extern crate url;
extern crate time;
extern crate hyper;

use std::io::Read;
use std::cell::{Cell, RefCell};
use url::Url;
use hyper::{Client};
use hyper::status::StatusCode;

/// A rule line is a single "Allow:" (allowance==True) or "Disallow:"
/// (allowance==False) followed by a path."""
#[derive(Debug, Eq, PartialEq, Clone)]
struct RuleLine {
    path: String,
    allowance: bool,
}

/// An entry has one or more user-agents and zero or more rulelines
#[derive(Debug, Eq, PartialEq, Clone)]
struct Entry {
    useragents: RefCell<Vec<String>>,
    rulelines: RefCell<Vec<RuleLine>>,
}

#[derive(Debug, Eq, PartialEq, Clone)]
pub struct RobotFileParser {
    entries: RefCell<Vec<Entry>>,
    default_entry: RefCell<Entry>,
    disallow_all: Cell<bool>,
    allow_all: Cell<bool>,
    url: Url,
    host: String,
    path: String,
    last_checked: Cell<i64>,
}


impl RuleLine {
    fn new(path: &str, allowance: bool) -> RuleLine {
        let mut allow = allowance;
        if path == "" && !allowance {
            // an empty value means allow all
            allow = true;
        }
        RuleLine {
            path: path.to_owned(),
            allowance: allow,
        }
    }

    fn applies_to(&self, filename: &str) -> bool {
        &self.path == "*" || filename.starts_with(&self.path)
    }
}


impl Entry {
    fn new() -> Entry {
        Entry {
            useragents: RefCell::new(vec![]),
            rulelines: RefCell::new(vec![]),
        }
    }

    /// check if this entry applies to the specified agent
    fn applies_to(&self, useragent: &str) -> bool {
        let ua = useragent.split("/").nth(0).unwrap_or("").to_lowercase();
        let useragents = self.useragents.borrow();
        for agent in &*useragents {
            if agent == "*" {
                return true;
            }
            if ua.contains(agent) {
                return true;
            }
        }
        false
    }

    
    /// Preconditions:
    /// - our agent applies to this entry
    /// - filename is URL decoded
    fn allowance(&self, filename: &str) -> bool {
        let rulelines = self.rulelines.borrow();
        for line in &*rulelines {
            if line.applies_to(filename) {
                return line.allowance
            }
        }
        true
    }

    fn push_useragent(&self, useragent: &str) {
        let mut useragents = self.useragents.borrow_mut();
        useragents.push(useragent.to_lowercase().to_owned());
    }

    fn push_ruleline(&self, ruleline: RuleLine) {
        let mut rulelines = self.rulelines.borrow_mut();
        rulelines.push(ruleline);
    }

    fn has_useragent(&self, useragent: &str) -> bool {
        let useragents = self.useragents.borrow();
        useragents.contains(&useragent.to_owned())
    }

    fn is_empty(&self) -> bool {
        let useragents = self.useragents.borrow();
        let rulelines = self.rulelines.borrow();
        useragents.is_empty() && rulelines.is_empty()
    }
}


impl RobotFileParser {
    pub fn new(url: &str) -> RobotFileParser {
        let parsed_url = Url::parse(url).unwrap();
        RobotFileParser {
            entries: RefCell::new(vec![]),
            default_entry: RefCell::new(Entry::new()),
            disallow_all: Cell::new(false),
            allow_all: Cell::new(false),
            url: parsed_url.clone(),
            host: parsed_url.domain().unwrap().to_owned(),
            path: parsed_url.path().unwrap().join("/"),
            last_checked: Cell::new(0i64),
        }
    }

    /// Returns the time the robots.txt file was last fetched.
    ///
    /// This is useful for long-running web spiders that need to
    /// check for new robots.txt files periodically.
    pub fn mtime(&self) -> i64 {
        self.last_checked.get()
    }

    /// Sets the time the robots.txt file was last fetched to the
    /// current time.
    pub fn modified(&self) {
        use time::get_time;

        self.last_checked.set(get_time().sec);
    }

    /// Sets the URL referring to a robots.txt file.
    pub fn set_url(&mut self, url: &str) {
        let parsed_url = Url::parse(url).unwrap();
        self.url = parsed_url.clone();
        self.host = parsed_url.domain().unwrap().to_owned();
        self.path = parsed_url.path().unwrap().join("/");
        self.last_checked.set(0i64);
    }

    /// Reads the robots.txt URL and feeds it to the parser.
    pub fn read(&self) {
        let client = Client::new();
        let mut res = match client.get(self.url.clone()).send() {
            Ok(res) => res,
            Err(_) => {
                return;
            }
        };
        match res.status {
{'fraction_non_alphanumeric': 0.13973168214654283, 'fraction_numerical': 0.0022703818369453044, 'mean_word_length': 2.1130742049469964, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4455537', 'n_tokens_mistral': 3045, 'n_tokens_neox': 2946, 'n_words': 756}