<commit_before>! Copyright (C) 2008 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors fry html.parser html.parser.analyzer
http.client kernel tools.time sets assocs sequences
concurrency.combinators io threads namespaces math multiline
math.parser inspector urls logging combinators.short-circuit
continuations calendar prettyprint dlists deques locals
spider.unique-deque combinators concurrency.semaphores
io.pathnames ;
IN: spider

TUPLE: spider
    base
    { count integer initial: 0 }
    { max-count number initial: 1/0. }
    sleep
    { max-depth integer initial: 0 }
    initial-links
    filters
    spidered
    todo
    nonmatching
    quiet?
    currently-spidering
    { #threads integer initial: 1 }
    semaphore
    follow-robots?
    robots ;

TUPLE: spider-result url depth headers
fetched-in parsed-html links processed-in fetched-at ;

: <spider> ( base -- spider )
    >url
    spider new
        over >>base
        over >>currently-spidering
        swap 0 <unique-deque> [ push-url ] keep >>todo
        <unique-deque> >>nonmatching
        H{ } clone >>spidered
        1 <semaphore> >>semaphore ;

: <spider-result> ( url depth -- spider-result )
    spider-result new
        swap >>depth
        swap >>url ; inline

<PRIVATE

: apply-filters ( links spider -- links' )
    filters>> [
        '[ _ 1&& ] filter<commit_msg>Revert "spider: simplify apply-filters."<commit_after>! Copyright (C) 2008 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors fry html.parser html.parser.analyzer
http.client kernel tools.time sets assocs sequences
concurrency.combinators io threads namespaces math multiline
math.parser inspector urls logging combinators.short-circuit
continuations calendar prettyprint dlists deques locals
spider.unique-deque combinators concurrency.semaphores
io.pathnames ;
IN: spider

TUPLE: spider
    base
    { count integer initial: 0 }
    { max-count number initial: 1/0. }
    sleep
    { max-depth integer initial: 0 }
    initial-links
    filters
    spidered
    todo
    nonmatching
    quiet?
    currently-spidering
    { #threads integer initial: 1 }
    semaphore
    follow-robots?
    robots ;

TUPLE: spider-result url depth headers
fetched-in parsed-html links processed-in fetched-at ;

: <spider> ( base -- spider )
    >url
    spider new
        over >>base
        over >>currently-spidering
        swap 0 <unique-deque> [ push-url ] keep >>todo
        <unique-deque> >>nonmatching
        H{ } clone >>spidered
        1 <semaphore> >>semaphore ;

: <spider-result> ( url depth -- spider-result )
    spider-result new
        swap >>depth
        swap >>url ; inline

<PRIVATE

: apply-filters ( links spider -- links' )
    filters>> [
        '[ [ _ 1&& ] filter ] call( seq -- seq' )
{'fraction_non_alphanumeric': 0.09641970932293513, 'fraction_numerical': 0.008507621410847217, 'mean_word_length': 2.8083670715249665, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1750378', 'n_tokens_mistral': 914, 'n_tokens_neox': 860, 'n_words': 259}