<commit_before>#!/usr/bin/env python


import sys
sys.path.append('gen-py.twisted')

# from filename.thrift import Service
from memorystructure import MemoryStructure as ms
from memorystructure.ttypes import *

from twisted.internet.defer import inlineCallbacks
from twisted.internet import reactor
from twisted.internet.protocol import ClientCreator

from thrift import Thrift
from thrift.transport import TTwisted
from thrift.protocol import TBinaryProtocol

import pymongo
import time
import json, re

config = json.load(open('config.json', 'r'))

#  curl http://scrapyd.host:6800/schedule.json \
#   -d start_urls=http://www.mongodb.org/,http://blog.mongodb.org/ \
#   -d maxdepth=2 \
#   -d follow_prefixes=s:http|t:80|h:org|h:mongodb \
#   -d nofollow_prefixes=s:http|t:80|h:org|h:mongodb|p:support \
#   -d discover_prefixes=s:http|t:80|h:ly|h:bit,s:http|t:80|h:ly|h:bit \
#   -d "user_agent=Mozilla/5.0 (compatible; hcibot/0.1)"


@inlineCallbacks
def test_memory_structure(client,crawler_page_queue):
    """ 
    Test the Memory structure thrift connection
    """
    
    #pi = ms.PageItem("id","url", "lru", "time", 200, 1, "errorCode", True, True, {"key":"value"})
    #pi2 = ms.PageItem("id2","url2", "lru2", "time2", 400, 2, "errorCode2", False, False, {"key2":"value2"})
    print "### clearIndex"
    yield client.clearIndex()
    print True
    
    
    page_items=crawler_page_queue.find()
    
    # add page with cache
    print "### createCache"
    pages={}
    links={}
    original_link_number=0
    # removing port if 80 :
    strip_lru_port = lambda lru : "|".join([stem for stem in lru.split("|") if stem!="t:80" ])
    # Removing subdomain if www :
    strip_lru_www = lambda lru : "|".join([stem for stem in lru.split("|") if stem!="h:www" ])
    # Removing anchors :
    anchor_regexp = re.compile(r"[rf]")
    strip_lru_anchors = lambda lru : "|".join([stem for stem in lru.split("|") if not anchor_regexp.match(stem) ])
    # Order query parameters alphabetically
    query_regexp = re.compile(r"\|q:([^\|]*)")
    def order_lru_query (lru):
        match = query_regexp.search(lru)
	if match is not None and match.group(1) is not None:
            res = match.group(1).split("&")
            res.sort()
            return lru.replace(match.group(1), "&".join(res))
        return lru
    # Identify links which are nodes
    lru_is_node = lambda lru : (len(lru.split("|")) <= config['precisionLimit'])
    # for get node, we should check exceptions
    get_node_lru = lambda lru : "|".join([stem for stem in lru.split("|")[:config['precisionLimit']] ])

    testlru = "s:http|t:80|h:org|h:mongodb|h:www|p:display|p:DOCS|p:Verifying+Propagation+of+Writes+with+getLastError|q:f=1&sg=3&a=3&b=34&_675=5&loiyy=hdsjk|f:HGYT"
    print testlru + "\n" + order_lru_query(strip_lru_anchors(strip_lru_www(strip_lru_port(testlru))))

    s=time.time()  
 
    for page_item in page_items : 
        # clean LRUs
        print "ORIG : " + page_item["lru"]
        page_item["lru"]=order_lru_query(strip_lru_anchors(strip_lru_www(strip_lru_port(page_item["lru"]))))
        print "RESU : " + page_item["lru"]
        is_node=lru_is_node(page_item["lru"])
        node_lru=page_item["lru"] if is_node else get_node_lru(page_item["lru"])
        
        if not page_item["lru"] in pages : 
            pages[page_item["lru"]]=ms.PageItem(str(page_item["_id"]),page_item["url"],page_item["lru"],str(page_item["timestamp"]),int(page_item["status"]),int(page_item["depth"]), "errorCode", False, is_node,{})

        if "lrulinks" in page_item :
            for index,lrulink in enumerate(page_item["lrulinks"]):
                lrulink=strip_lru_www(strip_lru_port(lrulink))
                target_node=lrulink if lru_is_node(lrulink) else get_node_lru(lrulink)
                original_link_number+=1
                if lrulink not in pages :
                    pages[lrulink]=ms.PageItem(str(page_item["_id"])+"_"+str(index),"",lrulink,str(page_item["timestamp"]),0,-1, "errorCode", False,lru_is_node(lrulink),{})
                
                links[(node_lru,target_node)]=links[(node_lru,target_node)]+1 if (node_lru,target_node) in links else 0
    
    print "processed "+str(len(pages))+" unique pages "+str(original_link_number)+" links "+str(len(links.values()))+" unique links in "+str(time.time()-s)
    
    
    s=time.time()
    cache_id=yield client.createCache(pages.values())
    print "page cache created  : "+str(cache_id)+ " containing "+str(len(pages))+" pages in "+str(time.time()-s)
        
    # index cache
    print "### indexCache"
    s=time.time()
    nb_pages=yield client.indexCache(cache_id)
    print str(nb_pages)+" pages indexed in "+str(time.time()-s)
    
    # add web entity creation rule by default
    print "### saveWebEntityCreationRule default"
    yield client.saveWebEntityCreationRule(WebEntityCreationRule(config['defaultWebEntityCreationRule']['regexp'], config['defaultWebEntityCreationRule']['prefix']))
    wecrs=yield client.getWebEntityCreationRules()
    print "result : "+str(len(wecrs)==1 and wecrs[0].regExp==config['defaultWebEntityCreationRule']['regexp'])
    
    # save node links
    print "### saveNodeLinks"
    s=time.time()
    yield client.saveNodeLinks([NodeLink("id",source,target,weight) for (source,target),weight in links.iteritems()])
    print "inserting NodeLinks in "+str(time.time()-s)
    
    # create web entities from cache
    print "### createWebEntities from cache"
    s=time.time()
    yield client.createWebEntities(cache_id)
    print "web entities created in "+str(time.time()-s)    
    
    print "### getWebEntities + getPagesFromWebEntities"
    s=time.time()    
    wes=yield client.getWebEntities()
    with open("webentities_pages.csv","w") as we_pages_file :
        we_pages_file.write("web entity name, page lru,web entity aliases\n")
        for we in wes : 
            pages = yield client.getPagesFromWebEntity(we.id)
            for page in pages :
                 we_pages_file.write(we.name+","+page.lru+","+",".join(we.LRUSet)+"\n")
        we_pages_file.close()
    print "export done in "+str(time.time()-s)
    
    # generate Web Entity Links
    print "### generate Web Entity Links"
    s=time.time()
    yield client.generateWebEntityLinks()
    print "processed webentity links in "+str(time.time()-s) 
    
    # get webentitynetwork
#    print "### getWebEntityNetwork gexf"
#    s=time.time()
#    network=yield client.getWebEntityNetwork("gexf")
#    print "got the network in "+str(time.time()-s) 
#    with open("hci_web_entities_network.gexf","w") as f : 
#        f.write(network)
    
    
    # createWebEntity function 
#     print "### createWebEntity"
#     we = yield client.createWebEntity("hci wiki",["s:http|h:fr|h:sciences-po|h:medialab|h:jiminy|p:hci"])
#     print str(we.name=="hci wiki") + " webentity "+str(we.id)+" created"
    
    
    # getWebEntity 
#     print "getWebEntity function"
#     we2 = yield client.getWebEntity(we.id)
#     print "result : " + str(we2.id==we.id and we2.name==we.name)
#     print "old : "+we.id +" new :"+we2.id
#     #print "old : "+we.name +" new :"+we2.name
#     print " new :"+we2.name
#     
#     
#     # update webentity
#     new_name="new WE"
#     we.name=new_name
#     we_id=yield client.updateWebEntity(we)
#     we2 = yield client.getWebEntity(we_id)
#     print "result :"+ str(we2.name==new_name)
    reactor.stop()
  

if __name__ == '__main__':

    crawler_queue = pymongo.Connection(config['mongoDB']['host'])[config['mongoDB']['db']][config['mongoDB']['queueCol']]

    client = ClientCreator(reactor,
                      TTwisted.ThriftClientProtocol,
                      ms.Client,
                      TBinaryProtocol.TBinaryProtocolFactory(),
                      ).connectTCP(config['memoryStructure']['IP'], config['memoryStructure']['port'])
    client.addCallback(lambda conn: conn.client)
    client.addCallback(test_memory_structure,crawler_queue)
    
    
    reactor.run()<commit_msg>log errors, clean LRUs & replace unnecessary callback<commit_after>#!/usr/bin/env python


import sys
sys.path.append('gen-py.twisted')

# from filename.thrift import Service
from memorystructure import MemoryStructure as ms
from memorystructure.ttypes import *

from twisted.internet.defer import inlineCallbacks
from twisted.internet import reactor
from twisted.internet.protocol import ClientCreator

from thrift import Thrift
from thrift.transport import TTwisted
from thrift.protocol import TBinaryProtocol

import pymongo, time, json, re

config = json.load(open('config.json', 'r'))

#  curl http://scrapyd.host:6800/schedule.json \
#   -d start_urls=http://www.mongodb.org/,http://blog.mongodb.org/ \
#   -d maxdepth=2 \
#   -d follow_prefixes=s:http|t:80|h:org|h:mongodb \
#   -d nofollow_prefixes=s:http|t:80|h:org|h:mongodb|p:support \
#   -d discover_prefixes=s:http|t:80|h:ly|h:bit,s:http|t:80|h:ly|h:bit \
#   -d "user_agent=Mozilla/5.0 (compatible; hcibot/0.1)"


@inlineCallbacks
def test_memory_structure(conn,crawler_page_queue):
    """ 
    Test the Memory structure thrift connection
    """
    client = conn.client

    #pi = ms.PageItem("id","url", "lru", "time", 200, 1, "errorCode", True, True, {"key":"value"})
    #pi2 = ms.PageItem("id2","url2", "lru2", "time2", 400, 2, "errorCode2", False, False, {"key2":"value2"})
    print "### clearIndex"
    yield client.clearIndex()
    print True
    
    page_items=crawler_page_queue.find()
    
    # add page with cache
    print "### createCache"
    pages={}
    links={}
    original_link_number=0
    # removing port if 80 :
    strip_lru_port = lambda lru : "|".join([stem for stem in lru.split("|") if stem!="t:80" ])
    # Removing subdomain if www :
    strip_lru_www = lambda lru : "|".join([stem for stem in lru.split("|") if stem!="h:www" ])
    # Removing anchors :
    anchor_regexp = re.compile(r"[rf]")
    strip_lru_anchors = lambda lru : "|".join([stem for stem in lru.split("|") if not anchor_regexp.match(stem) ])
    # Order query parameters alphabetically
    query_regexp = re.compile(r"\|q:([^\|]*)")
    def order_lru_query (lru):
        match = query_regexp.search(lru)
	if match is not None and match.group(1) is not None:
            res = match.group(1).split("&")
            res.sort()
            return lru.replace(match.group(1), "&".join(res))
        return lru
    clean_lru = lambda lru : order_lru_query(strip_lru_anchors(strip_lru_www(strip_lru_port(lru))))
    # Identify links which are nodes
    lru_is_node = lambda lru : (len(lru.split("|")) <= config['precisionLimit'])
    # for get node, we should check exceptions
    get_node_lru = lambda lru : "|".join([stem for stem in lru.split("|")[:config['precisionLimit']] ])

#    testlru = "s:http|t:80|h:org|h:mongodb|h:www|p:display|p:DOCS|p:Verifying+Propagation+of+Writes+with+getLastError|q:f=1&sg=3&a=3&b=34&_675=5&loiyy=hdsjk|f:HGYT"
#    print "TEST : " + testlru + "\nTEST : " + clean_lru(testlru)

    s=time.time()  
 
    for page_item in page_items : 
        # clean LRUs
        page_item["lru"] = clean_lru(page_item["lru"])
        is_node=lru_is_node(page_item["lru"])
        node_lru=page_item["lru"] if is_node else get_node_lru(page_item["lru"])
        
        if not page_item["lru"] in pages : 
            pages[page_item["lru"]]=ms.PageItem(str(page_item["_id"]),page_item["url"],page_item["lru"],str(page_item["timestamp"]),int(page_item["status"]),int(page_item["depth"]), "errorCode", False, is_node,{})

        if "lrulinks" in page_item :
            for index,lrulink in enumerate(page_item["lrulinks"]):
                lrulink = clean_lru(lrulink)
                target_node=lrulink if lru_is_node(lrulink) else get_node_lru(lrulink)
                original_link_number+=1
                if lrulink not in pages :
                    pages[lrulink]=ms.PageItem(str(page_item["_id"])+"_"+str(index),"",lrulink,str(page_item["timestamp"]),0,-1, "errorCode", False,lru_is_node(lrulink),{})
                
                links[(node_lru,target_node)]=links[(node_lru,target_node)]+1 if (node_lru,target_node) in links else 0
    
    print "processed "+str(len(pages))+" unique pages "+str(original_link_number)+" links "+str(len(links.values()))+" unique links in "+str(time.time()-s)
    
    
    s=time.time()
    cache_id=yield client.createCache(pages.values())
    print "page cache created  : "+str(cache_id)+ " containing "+str(len(pages))+" pages in "+str(time.time()-s)
        
    # index cache
    print "### indexCache"
    s=time.time()
    nb_pages=yield client.indexCache(cache_id)
    print str(nb_pages)+" pages indexed in "+str(time.time()-s)
    
    # add web entity creation rule by default
    print "### saveWebEntityCreationRule default"
    yield client.saveWebEntityCreationRule(WebEntityCreationRule(config['defaultWebEntityCreationRule']['regexp'], config['defaultWebEntityCreationRule']['prefix']))
    wecrs=yield client.getWebEntityCreationRules()
    print "result : "+str(len(wecrs)==1 and wecrs[0].regExp==config['defaultWebEntityCreationRule']['regexp'])
    
    # save node links
    print "### saveNodeLinks"
    s=time.time()
    yield client.saveNodeLinks([NodeLink("id",source,target,weight) for (source,target),weight in links.iteritems()])
    print "inserting NodeLinks in "+str(time.time()-s)
    
    # create web entities from cache
    print "### createWebEntities from cache"
    s=time.time()
    yield client.createWebEntities(cache_id)
    print "web entities created in "+str(time.time()-s)    
    
    print "### getWebEntities + getPagesFromWebEntities"
    s=time.time()    
    wes=yield client.getWebEntities()
    with open("webentities_pages.csv","w") as we_pages_file :
        we_pages_file.write("web entity name, page lru,web entity aliases\n")
        for we in wes : 
            pages = yield client.getPagesFromWebEntity(we.id)
            for page in pages :
                 we_pages_file.write(we.name+","+page.lru+","+",".join(we.LRUSet)+"\n")
        we_pages_file.close()
    print "export done in "+str(time.time()-s)
    
    # generate Web Entity Links
    print "### generate Web Entity Links"
    s=time.time()
    yield client.generateWebEntityLinks()
    print "processed webentity links in "+str(time.time()-s) 
    
    # get webentitynetwork
#     print "### getWebEntityNetwork gexf"
#     s=time.time()
#     network=yield client.getWebEntityNetwork("gexf")
#     print "got the network in "+str(time.time()-s) 
#     with open("hci_web_entities_network.gexf","w") as f : 
#         f.write(network)
    
    
    # createWebEntity function 
#     print "### createWebEntity"
#     we = yield client.createWebEntity("hci wiki",["s:http|h:fr|h:sciences-po|h:medialab|h:jiminy|p:hci"])
#     print str(we.name=="hci wiki") + " webentity "+str(we.id)+" created"
    
    
    # getWebEntity 
#     print "getWebEntity function"
#     we2 = yield client.getWebEntity(we.id)
#     print "result : " + str(we2.id==we.id and we2.name==we.name)
#     print "old : "+we.id +" new :"+we2.id
#     #print "old : "+we.name +" new :"+we2.name
#     print " new :"+we2.name
#     
#     
    # update webentity
#     new_name="new WE"
#     we.name=new_name
#     we_id=yield client.updateWebEntity(we)
#     we2 = yield client.getWebEntity(we_id)
#     print "result :"+ str(we2.name==new_name)

    reactor.stop()

def errorHandler(failure):
    print failure
    failure.trap(Exception)

if __name__ == '__main__':

    crawler_queue = pymongo.Connection(config['mongoDB']['host'])[config['mongoDB']['db']][config['mongoDB']['queueCol']]

    client = ClientCreator(reactor,
                      TTwisted.ThriftClientProtocol,
                      ms.Client,
                      TBinaryProtocol.TBinaryProtocolFactory(),
                      ).connectTCP(config['memoryStructure']['IP'], config['memoryStructure']['port'])
# moved this in test_memory_struct
#    client.addCallback(lambda conn: conn.client)
    client.addCallback(test_memory_structure,crawler_queue)
    client.addErrback(errorHandler)

    reactor.run()

{'fraction_non_alphanumeric': 0.1509819713772381, 'fraction_numerical': 0.008301840034694257, 'mean_word_length': 3.5393700787401574, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 14, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5132451', 'n_tokens_mistral': 5694, 'n_tokens_neox': 5554, 'n_words': 1036}