<commit_before>#encoding=utf-8 
from lxml import html
import popen2
import sqlite3
import requests
import gl
import string
import sys
import webkit, gtk
import thread
import fusionyoudao
import fusionicb
import webshot
import record_xclip
import random
import os
from time import sleep
def inputconfig():
    gl.lock=1
    print "切换字典"
    print gl.lock
    gl.prebaseurl = gl.baseurl
    #os.system("touch cache/data.lck")
    conn = sqlite3.connect(gl.datadir)
    c = conn.cursor()
    c1 = conn.cursor()
    c.execute("select value from ItemTable where key = 'dict' ")
    c1.execute("select value from ItemTable where key = 'keyword' ")
    r= c.fetchone()
    r1= c1.fetchone()
    gl.baseurl= "".join(str(r[0]).split('\x00')) #str to string
    #if(gl.prebaseurl !=gl.baseurl):
    #    gl.lock=1
    stext = "".join(str(r1[0]).split('\x00')) 
    c.close()
    c1.close()
    conn.close
    print stext 
    os.system("/bin/echo -e  \'"+ stext  + "\' >> \'"+ gl.historydir + "\'")
def lookup():
    pre_text=""
    text=""
    #监视history.txt变化
    cmd = "tail -f " + gl.historydir 
    if(gl.baseurl==""):
        gl.baseurl=gl.baseurlyoudao
    myfile=os.popen(cmd)
    while True:
        text=myfile.readline().strip('\r\n\x00')
        if (pre_text != text or gl.prebaseurl != gl.baseurl) and text != "" : # 或者不一定对lzt
            pre_text = text
            print gl.lock
            gl.prebaseurl =  gl.baseurl
            url= gl.baseurl + text                           #合成地址
            print url + "kkkkkkkkkkkk"                       #合成地址检测点1 
            r = requests.get(url)                            #获得网页
            #doc = html.document_fromstring(r.text)          
            f_tar=open(gl.origindir,'w+')             #缓存原始网页
            print >>f_tar,r.text
            f_tar.close()
            os.system("echo \'"+ gl.downloadwait + "\' > cache/result.html")         #清空最终缓冲增强程序稳健性
            if(gl.baseurl==gl.baseurlyoudao):
                fusionyoudao.reconstruct()                   #区分聚合
            if(gl.baseurl==gl.baseurlicb):
                fusionicb.reconstruct()
            gl.homeurl="file://" + gl.resultdir #合成最终缓冲访问地址
            window.load(gl.homeurl)                             #加载最终缓冲内容到浏览器
            window.show()                                    #显示结果
            gl.lock=0

def webshow():
    global window
    global Alive
    window = webshot.Window()
    window.load(gl.homeurl)
    window.show() 
    gtk.main() 
    Alive=0

def gettext():
    os.system("xclip -f /dev/null")           #清空剪切板
    os.system("/bin/echo "" > \'"+ gl.historydir + "\'")
    record_xclip.record_dpy.record_enable_context(record_xclip.ctx, record_xclip.record_callback)            
    record_xclip.record_dpy.record_free_context(record_xclip.ctx)
def loadconfig():
    sleep(1)
    cmd = "inotifywait  -m " + gl.datadir
    if os.path.isfile(gl.datadir)==False:
        os.system("cp -rf  \'"+ gl.youdaosqlitedir + "\' \'" + gl.sqlitedir +"\'")
        switch=os.popen(cmd)
    switch=os.popen(cmd)
    gl.lock=0
    while Alive :
        if str(switch.readline()).find("MODIFY"):
            if(gl.lock==0):
                inputconfig()
            

# Main thread
def main():
    
    #运行标志结束 
    global Alive
    Alive=1
    thread.start_new_thread(webshow,())
    sleep(0.1)
    thread.start_new_thread(gettext,())
    sleep(1)
    thread.start_new_thread(lookup,())
    thread.start_new_thread(loadconfig,())
    	
    while Alive:
        sleep(1)
        

    print 'All threads have terminated.'
if __name__ == '__main__':
    main()<commit_msg>inotify bug<commit_after>#encoding=utf-8 
from lxml import html
import popen2
import sqlite3
import requests
import gl
import string
import sys
import webkit, gtk
import thread
import fusionyoudao
import fusionicb
import webshot
import record_xclip
import random
import os
from time import sleep
def inputconfig():
    gl.lock=1
    print "切换字典"
    print gl.lock
    gl.prebaseurl = gl.baseurl
    #os.system("touch cache/data.lck")
    conn = sqlite3.connect(gl.datadir)
    c = conn.cursor()
    c1 = conn.cursor()
    c.execute("select value from ItemTable where key = 'dict' ")
    c1.execute("select value from ItemTable where key = 'keyword' ")
    r= c.fetchone()
    r1= c1.fetchone()
    gl.baseurl= "".join(str(r[0]).split('\x00')) #str to string
    #if(gl.prebaseurl !=gl.baseurl):
    #    gl.lock=1
    stext = "".join(str(r1[0]).split('\x00')) 
    c.close()
    c1.close()
    conn.close
    print stext 
    os.system("/bin/echo -e  \'"+ stext  + "\' >> \'"+ gl.historydir + "\'")
def lookup():
    pre_text=""
    text=""
    #监视history.txt变化
    cmd = "tail -f " + gl.historydir 
    if(gl.baseurl==""):
        gl.baseurl=gl.baseurlyoudao
    myfile=os.popen(cmd)
    while True:
        text=myfile.readline().strip('\r\n\x00')
        if (pre_text != text or gl.prebaseurl != gl.baseurl) and text != "" : # 或者不一定对lzt
            pre_text = text
            print gl.lock
            gl.prebaseurl =  gl.baseurl
            url= gl.baseurl + text                           #合成地址
            print url + "kkkkkkkkkkkk"                       #合成地址检测点1 
            r = requests.get(url)                            #获得网页
            #doc = html.document_fromstring(r.text)          
            f_tar=open(gl.origindir,'w+')             #缓存原始网页
            print >>f_tar,r.text
            f_tar.close()
            os.system("echo \'"+ gl.downloadwait + "\' > cache/result.html")         #清空最终缓冲增强程序稳健性
            if(gl.baseurl==gl.baseurlyoudao):
                fusionyoudao.reconstruct()                   #区分聚合
            if(gl.baseurl==gl.baseurlicb):
                fusionicb.reconstruct()
            gl.homeurl="file://" + gl.resultdir #合成最终缓冲访问地址
            window.load(gl.homeurl)                             #加载最终缓冲内容到浏览器
            window.show()                                    #显示结果
            gl.lock=0

def webshow():
    global window
    global Alive
    window = webshot.Window()
    window.load(gl.homeurl)
    window.show() 
    gtk.main() 
    Alive=0

def gettext():
    os.system("xclip -f /dev/null")           #清空剪切板
    os.system("/bin/echo "" > \'"+ gl.historydir + "\'")
    record_xclip.record_dpy.record_enable_context(record_xclip.ctx, record_xclip.record_callback)            
    record_xclip.record_dpy.record_free_context(record_xclip.ctx)
def loadconfig():
    sleep(1)
    cmd = "inotifywait  -m " + gl.datadir
    if os.path.isfile(gl.datadir)==False:
        os.system("cp -rf  \'"+ gl.youdaosqlitedir + "\' \'" + gl.sqlitedir +"\'")
        switch=os.popen(cmd)
    else:
        switch=os.popen(cmd)
    gl.lock=0
    while Alive :
        if str(switch.readline()).find("MODIFY"):
            if(gl.lock==0):
                inputconfig()
            

# Main thread
def main():
    
    #运行标志结束 
    global Alive
    Alive=1
    thread.start_new_thread(webshow,())
    sleep(0.1)
    thread.start_new_thread(gettext,())
    sleep(1)
    thread.start_new_thread(lookup,())
    thread.start_new_thread(loadconfig,())
    	
    while Alive:
        sleep(1)
        

    print 'All threads have terminated.'
if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.1339706289830978, 'fraction_numerical': 0.008589637018564699, 'mean_word_length': 1.945328437372501, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1582951', 'n_tokens_mistral': 2568, 'n_tokens_neox': 2621, 'n_words': 457}