# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
# def get_db(db_name, client='mongo'):


import _env
    import chardet
from pprint import pprint as pp
from tornado.util import ObjectDict
from lib._db import get_db, redis_client as r
from html_parser import Bs4HtmlParser
from lagou_parser import LagouHtmlParser
from spider import LagouCrawler
from web_util import logged
    print(text)


def count_how_many_block_html():
    col = lagou_html_col
    cnt = 0
    for html_doc in col.find(modifiers={"$snapshot": True}):
        url = html_doc['url']
        html = html_doc['html']
        if LagouCrawler.is_block_html(html, False):
            lagou_html_col.delete_one({'url': url})
            print(url)
            cnt += 1
    return cnt


def remove_deleted_html():
    cnt = 0
    for html_doc in lagou_html_col.find(modifiers={"$snapshot": True}):
        url = html_doc['url']
        html = html_doc['html']
        if LagouCrawler.is_deleted_html(html, False):
            lagou_html_col.delete_one({'url': url})
            print(url)
            cnt += 1
    return cnt


def count_how_many_check_html():
    col = lagou_html_col

# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
# def get_db(db_name, client='mongo'):


# Path: crawler/lagou/handle_db.py
import _env
    import chardet
from pprint import pprint as pp
from tornado.util import ObjectDict
from lib._db import get_db, redis_client as r
from html_parser import Bs4HtmlParser
from lagou_parser import LagouHtmlParser
from spider import LagouCrawler
from web_util import logged

    print(text)


def count_how_many_block_html():
    col = lagou_html_col
    cnt = 0
    for html_doc in col.find(modifiers={"$snapshot": True}):
        url = html_doc['url']
        html = html_doc['html']
        if LagouCrawler.is_block_html(html, False):
            lagou_html_col.delete_one({'url': url})
            print(url)
            cnt += 1
    return cnt


def remove_deleted_html():
    cnt = 0
    for html_doc in lagou_html_col.find(modifiers={"$snapshot": True}):
        url = html_doc['url']
        html = html_doc['html']
        if LagouCrawler.is_deleted_html(html, False):
            lagou_html_col.delete_one({'url': url})
            print(url)
            cnt += 1
    return cnt


def count_how_many_check_html():
    col = lagou_html_col

    print(col.count())