<commit_before>
# crawls the web find videos

# @todo needs error checking... what happens if offline or if imdb changes something?

# lists of titles on imdb.com, these are retrieved on the first run
imdb_list_urls = [

    # public domain movies
    'http://www.imdb.com/list/ls003915205/?view=compact&sort=created:desc',
    'http://www.imdb.com/list/ls003915205/?view=compact&sort=created:desc&start=251'

]

# lists of new released on imdb.com updated once a day
imdb_update_urls = [

    # new dvd releases
    'http://www.imdb.com/sections/dvd/'

]

# pages with magnets or torrents to scrape
crawl_urls = [
    'https://kat.cr/tv/',
    'https://kat.cr/movies/'
]

# urls to query for a list of magnets/torrents based on imdb id %d
search_urls = [
    'https://kat.cr/usearch/imdb%%3A%d/?field=seeders&sorder=desc'
]


# urls to query for a list of imdb id's for a keyword search query %s
title_search_urls = [
    'http://www.imdb.com/find?q=%s'
]

import os
import time
import requests
import re
import urllib
from bs4 import BeautifulSoup


def start(settings, db_connect):

    db = db_connect()
    requests_session = requests.Session()

    print('spiderd started')


    # skip existing
    dbc = db.execute("""
SELECT
    COUNT(*)
FROM
    movie
WHERE
    public_domain = 1
""")
    existing_publicdomain_movie_count = dbc.fetchone()[0]

    #if settings['first_run']:
    if 0 == existing_publicdomain_movie_count:
        print('Initilializing database with public domain movies from imdb')

        # start by adding Wizard of Oz as a demo that works right away
        crawl_imdb_title(settings, db, requests_session, 16544)

        # add imdb's public domain movies
        for url in imdb_list_urls:
            crawl_imdb_list(settings, db, requests_session, url, 1)


    loop_number = 0

    while True:

        # every second
        # @todo running every second find search results faster, but for production it should sleep longer when idle to save resources
        propagate_queries(settings, db, requests_session)
<commit_msg>fix public domain spidering<commit_after>
# crawls the web find videos

# @todo needs error checking... what happens if offline or if imdb changes something?

# lists of titles on imdb.com, these are retrieved on the first run
imdb_list_urls = [

    # public domain movies
    'http://www.imdb.com/list/ls003915205/?view=compact&sort=created:desc',
    'http://www.imdb.com/list/ls003915205/?view=compact&sort=created:desc&start=251'

]

# lists of new released on imdb.com updated once a day
imdb_update_urls = [

    # new dvd releases
    'http://www.imdb.com/sections/dvd/'

]

# pages with magnets or torrents to scrape
crawl_urls = [
    'https://kat.cr/tv/',
    'https://kat.cr/movies/'
]

# urls to query for a list of magnets/torrents based on imdb id %d
search_urls = [
    'https://kat.cr/usearch/imdb%%3A%d/?field=seeders&sorder=desc'
]


# urls to query for a list of imdb id's for a keyword search query %s
title_search_urls = [
    'http://www.imdb.com/find?q=%s'
]

import os
import time
import requests
import re
import urllib
from bs4 import BeautifulSoup


def start(settings, db_connect):

    db = db_connect()
    requests_session = requests.Session()

    print('spiderd started')

    if settings['first_run']:

        # start by adding Wizard of Oz as a demo that works right away
        crawl_imdb_title(settings, db, requests_session, 16544)

    # skip existing
    dbc = db.execute("""
SELECT
    COUNT(*)
FROM
    movie
WHERE
    public_domain = 1
""")
    existing_publicdomain_movie_count = dbc.fetchone()[0]

    if existing_publicdomain_movie_count < 2:
        print('Initilializing database with public domain movies from imdb')

        # add imdb's public domain movies
        for url in imdb_list_urls:
            crawl_imdb_list(settings, db, requests_session, url, 1)


    loop_number = 0

    while True:

        # every second
        # @todo running every second find search results faster, but for production it should sleep longer when idle to save resources
        propagate_queries(settings, db, requests_session)

{'fraction_non_alphanumeric': 0.09636803874092009, 'fraction_numerical': 0.015980629539951573, 'mean_word_length': 3.2852697095435683, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2448845', 'n_tokens_mistral': 1415, 'n_tokens_neox': 1309, 'n_words': 409}