<commit_before># #coding=utf-8
import time
import urllib, urllib2
import re
import anydbm as dbm
from codepy import menulog

'''
(python bgtask.py &)
用于抓取菜单信息

地址格式变更？这个可以直接访问
http://wap.plus.yixin.im/wap/material/viewImageText?id=31613351
'''

pattern_title = r"<title>(.+)</title>"
pattern_weekday = ur"（星期(.)）"
pattern_year = ur'20(\d\d)-'
pattern_month_update = r'-(\d+)-'
pattern_month = r'>(\d+)</span>'
pattern_day = ur'月(\d+)日'
pattern_day2 = ur'>(\d+)日'
urlhead = 'http://numenplus.yixin.im/singleNewsWap.do?materialId='
datafile = 'datafile'
startId = 47650


def getWebContent(url):
    try:
        url += '&companyId=1'
        req = urllib2.Request(url)
        req.add_header('User-Agent', 'Mozilla/5.0 (Linux; Android 6.0; PRO 6 Build/MRA58K; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.130 Mobile Safari/537.36 YiXin/4.8.3')
        res = urllib2.urlopen(req)
        html = res.read().decode('utf-8')
        return html
    except Exception as e:
        menulog.debug(str(e))
        return ''


class Background:
    def __init__(self):
        self.frequency = 10800        # 间隔(秒)
        self.interval = 150           # 每次爬的id数量
        self.back = 0                 # 每次从self.startId - self.back开始查找，防止被占坑
        self.firstRun = True          # 是否在程序开始后先执行一次

        self.today = 0
        self.running = False          # 是否正在运行(否则同一秒会重复执行多次)
        self.startId = 0
        self.count = 0
        self.usedId  = 0              # 记录中间最大的非空id
        self.nowId = self.startId
        self.result = u'未找到菜单'
        self.lastQuery = 0
        self.cache = {}               # {151019:15163}  # 日期:id
        self.maybe = []               # 爬到的报错的页面
        self.empty = 0
        self.maxEmpty = 200           # 连续多少空页后中断; 近期总是跳着使用id

    def getTime(self):
        return int(time.time())

    def schedule(self):
        if self.firstRun:
            self.firstRun = False
            self.process()
            self.schedule()        # 可用Timer().start()替换
        else:
            while True:
                time.sleep(0.1)    # 可以极大减少cpu占用
                if self.getTime() % self.frequency == 0 and not self.running:
                    self.running = True
                    self.process()
                elif self.getTime() % 3600 == 0:   # 每3600s记录一次存活信息
                    menulog.info('%s@%d'% (time.strftime('20%y-%m-%d %H:%M:%S', time.localtime()), self.getTime()))
                    time.sleep(1)

    def process(self):
        self.count += 1
        self.today = int(time.strftime('%y%m%d', time.localtime()))
        menulog.info(u'开始第%d次查找@%d'% (self.count, self.getTime()))

        try:
            db = dbm.open(datafile, 'c')
            if not len(db):
                # 没有之前的数据文件
                db['startId'] = str(startId)
                db['lastQuery'] = str(self.getTime())
                db['cache'] = str(self.cache)<commit_msg>适配新url<commit_after># #coding=utf-8
import time
import urllib, urllib2
import re
import anydbm as dbm
from codepy import menulog

'''
(python bgtask.py &)
用于抓取菜单信息

地址格式变更？这个可以直接访问
http://wap.plus.yixin.im/wap/material/viewImageText?id=31613351
'''

pattern_title = r"<title>(.+)</title>"
pattern_weekday = ur"（星期(.)）"
pattern_year = ur'20(\d\d)-'
pattern_month_update = r'-(\d+)-'
pattern_month = r'>(\d+)</span>'
pattern_day = ur'月(\d+)日'
pattern_day2 = ur'>(\d+)日'
urlhead = 'http://numenplus.yixin.im/singleNewsWap.do?materialId='
datafile = 'datafile'
startId = 47650


def getWebContent(url):
    try:
        url += '&companyId=1'
        req = urllib2.Request(url)
        req.add_header('User-Agent', 'Mozilla/5.0 (Linux; Android 6.0; PRO 6 Build/MRA58K; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.130 Mobile Safari/537.36 YiXin/4.8.3')
        res = urllib2.urlopen(req)
        html = res.read().decode('utf-8')
        return html
    except Exception as e:
        menulog.debug(str(e))
        return ''


class Background:
    def __init__(self):
        self.frequency = 10800        # 间隔(秒)
        self.interval = 150           # 每次爬的id数量
        self.back = 0                 # 每次从self.startId - self.back开始查找，防止被占坑
        self.firstRun = True          # 是否在程序开始后先执行一次

        self.today = 0
        self.running = False          # 是否正在运行(否则同一秒会重复执行多次)
        self.startId = 0
        self.count = 0
        self.usedId  = 0              # 记录中间最大的非空id
        self.nowId = self.startId
        self.result = u'未找到菜单'
        self.lastQuery = 0
        self.cache = {}               # {151019:15163}  # 日期:id
        self.maybe = []               # 爬到的报错的页面
        self.empty = 0
        self.maxEmpty = 100           # 连续多少空页后中断; 近期总是跳着使用id

    def getTime(self):
        return int(time.time())

    def schedule(self):
        if self.firstRun:
            self.firstRun = False
            self.process()
            self.schedule()        # 可用Timer().start()替换
        else:
            while True:
                time.sleep(0.1)    # 可以极大减少cpu占用
                if self.getTime() % self.frequency == 0 and not self.running:
                    self.running = True
                    self.process()
                elif self.getTime() % 3600 == 0:   # 每3600s记录一次存活信息
                    menulog.info('%s@%d'% (time.strftime('20%y-%m-%d %H:%M:%S', time.localtime()), self.getTime()))
                    time.sleep(1)

    def process(self):
        self.count += 1
        self.today = int(time.strftime('%y%m%d', time.localtime()))
        menulog.info(u'开始第%d次查找@%d'% (self.count, self.getTime()))

        try:
            db = dbm.open(datafile, 'c')
            if not len(db):
                # 没有之前的数据文件
                db['startId'] = str(startId)
                db['lastQuery'] = str(self.getTime())
                db['cache'] = str(self.cache)
{'fraction_non_alphanumeric': 0.14155955441302484, 'fraction_numerical': 0.0339331619537275, 'mean_word_length': 1.8621873467385974, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1000329', 'n_tokens_mistral': 2356, 'n_tokens_neox': 2267, 'n_words': 375}