# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
# def get_db(db_name, client='mongo'):


import _env
    import chardet
from pprint import pprint as pp
from tornado.util import ObjectDict
from lib._db import get_db, redis_client as r
from html_parser import Bs4HtmlParser
from lagou_parser import LagouHtmlParser
from spider import LagouCrawler
from web_util import logged

def count_how_many_check_html():
    col = lagou_html_col
    print(col.count())
    cnt = 0
    _id_list = []
    for html_doc in col.find(modifiers={"$snapshot": True}):
        html = html_doc['html']
        if LagouCrawler.is_check_html(html, verbose=False):
            cnt += 1
            _id_list.append(html_doc._id)
    print(cnt)
    return cnt
    print(col.count())
    # col.remove({'_id':{'$in':_id_list}})


@logged
class ParseJob(object):
    """用来处理抓下来的html页面，把需要的数据从html中提取出来单独存储"""

    db = get_db('htmldb')

    def __init__(self):
        self.from_col = getattr(self.db, 'lagou_html')
        self.to_col = getattr(self.db, 'lagou_job')
        self.key = self.__class__.__name__
        self.last_id = int(r.get(self.key) or 0)

    def set_id(self, last_id=0):

# Path: lib/_db.py
# def get_collection(db_name, collection_name, client='mongo'):
# def get_db(db_name, client='mongo'):


# Path: crawler/lagou/handle_db.py
import _env
    import chardet
from pprint import pprint as pp
from tornado.util import ObjectDict
from lib._db import get_db, redis_client as r
from html_parser import Bs4HtmlParser
from lagou_parser import LagouHtmlParser
from spider import LagouCrawler
from web_util import logged


def count_how_many_check_html():
    col = lagou_html_col
    print(col.count())
    cnt = 0
    _id_list = []
    for html_doc in col.find(modifiers={"$snapshot": True}):
        html = html_doc['html']
        if LagouCrawler.is_check_html(html, verbose=False):
            cnt += 1
            _id_list.append(html_doc._id)
    print(cnt)
    return cnt
    print(col.count())
    # col.remove({'_id':{'$in':_id_list}})


@logged
class ParseJob(object):
    """用来处理抓下来的html页面，把需要的数据从html中提取出来单独存储"""

    db = get_db('htmldb')

    def __init__(self):
        self.from_col = getattr(self.db, 'lagou_html')
        self.to_col = getattr(self.db, 'lagou_job')
        self.key = self.__class__.__name__
        self.last_id = int(r.get(self.key) or 0)

    def set_id(self, last_id=0):

        r.set(self.key, last_id)