<commit_before>(ns irobot.t-core
  (:use [midje.sweet])
  (:require [irobot.core :refer :all]))

(def normal-records
  "User-Agent: *
Allow: /

User-Agent: MyBot
Allow: /
Disallow: /
Disallow: /

User-Agent: OtherA
Disallow: /

User-Agent: OtherB
Disallow: /") 

(def normal-robots (parse normal-records))
(def empty-robots (parse ""))
(def blocks-all-robots (parse "User-Agent: *\nDisallow: /"))

(facts "about path matching"
  
  (fact "Root paths always match"
    (matching-path "/" #{"/"}) => truthy)

  (fact "Exact matches always match"
    (matching-path "/foo" #{"/foo"}) => truthy)

  (fact "If one of the Paths is a prefix of Path, we match"
    (matching-path "/foobar" #{"/foo"}) => truthy
    (matching-path "/foobar" #{"/fooba"}) => truthy)

  (fact "If there is one element in Paths and it's not a prefix, we don't match"
    (matching-path "/foobar" #{"/baz"}) => falsey)


  (fact "If there are multiple Paths and one of them is a prefix of Path, we match"
    (matching-path "/foobar" #{"/bar" "/baz" "/foo"}) => truthy)

  (fact "If there are multiple Paths and NONE of them is a prefix of Path, we DON'T match"
    (matching-path "/foobar" #{"/baz" "/bar"}) => falsey)


  (fact "empty robots file allows crawling by any UA on any path"
    (allows? empty-robots "NaughtyBot 1.0" "/random/path") => true)


  (fact "allows"
    (allows? normal-robots "MyBot" "/") => true)


  (fact "allows UA when we have an allow / and a disallow /"
    (allows? normal-robots "OtherA" "/") => truthy)


  (fact "disallowing / for all UA prevents accessing any path"
    (allows? blocks-all-robots "MyBot 2.3" "/some/path") => false)


  (fact "collates rules by UA"
    (rules-for "MyBot" normal-robots) => {:allow #{"/"} :disallow #{ "/"}}))
<commit_msg>Whitespace<commit_after>(ns irobot.t-core
  (:use [midje.sweet])
  (:require [irobot.core :refer :all]))

(def normal-records
  "User-Agent: *
Allow: /

User-Agent: MyBot
Allow: /
Disallow: /
Disallow: /

User-Agent: OtherA
Disallow: /

User-Agent: OtherB
Disallow: /") 

(def normal-robots (parse normal-records))
(def empty-robots (parse ""))
(def blocks-all-robots (parse "User-Agent: *\nDisallow: /"))

(facts "about path matching"
  
  (fact "Root paths always match"
    (matching-path "/" #{"/"}) => truthy)

  
  (fact "Exact matches always match"
    (matching-path "/foo" #{"/foo"}) => truthy)

  
  (fact "If one of the Paths is a prefix of Path, we match"
    (matching-path "/foobar" #{"/foo"}) => truthy
    (matching-path "/foobar" #{"/fooba"}) => truthy)

  
  (fact "If there is one element in Paths and it's not a prefix, we don't match"
    (matching-path "/foobar" #{"/baz"}) => falsey)


  (fact "If there are multiple Paths and one of them is a prefix of Path, we match"
    (matching-path "/foobar" #{"/bar" "/baz" "/foo"}) => truthy)

  (fact "If there are multiple Paths and NONE of them is a prefix of Path, we DON'T match"
    (matching-path "/foobar" #{"/baz" "/bar"}) => falsey)


  (fact "empty robots file allows crawling by any UA on any path"
    (allows? empty-robots "NaughtyBot 1.0" "/random/path") => true)


  (fact "allows"
    (allows? normal-robots "MyBot" "/") => true)


  (fact "allows UA when we have an allow / and a disallow /"
    (allows? normal-robots "OtherA" "/") => truthy)


  (fact "disallowing / for all UA prevents accessing any path"
    (allows? blocks-all-robots "MyBot 2.3" "/some/path") => false)


  (fact "collates rules by UA"
    (rules-for "MyBot" normal-robots) => {:allow #{"/"} :disallow #{ "/"}}))

{'fraction_non_alphanumeric': 0.17667044167610418, 'fraction_numerical': 0.0022650056625141564, 'mean_word_length': 3.298053527980535, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3912726', 'n_tokens_mistral': 1361, 'n_tokens_neox': 1260, 'n_words': 412}