<commit_before>#!/usr/bin/python

import sys
import os
import glob
#sys.path.append(os.path.join(os.path.dirname(__file__), "gen-py"))
sys.path.append(os.path.join(os.path.dirname(__file__),"gen-py/thrift_solr/"))
sys.path.append(os.path.dirname(__file__) )

from thrift.transport import TSocket 
from thrift.server import TServer 
#import thrift_solr


import ExtractorService

import sys

import readability

import readability

def extract_with_python_readability( raw_content ):
    doc = readability.Document( raw_content )
    
    return [ u'' + doc.short_title(),
             u'' + doc.summary() ]

class ExtractorHandler:
    def extract_html( self, raw_html ):

        #print raw_html

        #raw_html = raw_html.encode( 'utf-8' )

        ret =  extract_with_python_readability( raw_html )
        #print ret[1]
        return ret

handler = ExtractorHandler()
processor = ExtractorService.Processor(handler)
listening_socket = TSocket.TServerSocket(port=9090)
server = TServer.TThreadPoolServer(processor, listening_socket)

print ("[Server] Started")
server.serve()<commit_msg>Use the TBinaryProtocolAccelerated protocol instead of TBinaryProtocol to improve performance.<commit_after>#!/usr/bin/python

import sys
import os
import glob
#sys.path.append(os.path.join(os.path.dirname(__file__), "gen-py"))
sys.path.append(os.path.join(os.path.dirname(__file__),"gen-py/thrift_solr/"))
sys.path.append(os.path.dirname(__file__) )

from thrift.transport import TSocket 
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.server import TServer 
from thrift.protocol.TBinaryProtocol import TBinaryProtocolAccelerated

#import thrift_solr


import ExtractorService

import sys

import readability

import readability

def extract_with_python_readability( raw_content ):
    doc = readability.Document( raw_content )
    
    return [ u'' + doc.short_title(),
             u'' + doc.summary() ]

class ExtractorHandler:
    def extract_html( self, raw_html ):

        #print raw_html

        #raw_html = raw_html.encode( 'utf-8' )

        ret =  extract_with_python_readability( raw_html )
        #print ret[1]
        return ret

handler = ExtractorHandler()
processor = ExtractorService.Processor(handler)
listening_socket = TSocket.TServerSocket(port=9090)
tfactory = TTransport.TBufferedTransportFactory()
#pfactory = TBinaryProtocol.TBinaryProtocolFactory()
pfactory = TBinaryProtocol.TBinaryProtocolAcceleratedFactory()

server = TServer.TThreadPoolServer(processor, listening_socket, tfactory, pfactory)

print ("[Server] Started")
server.serve()
{'fraction_non_alphanumeric': 0.10107197549770292, 'fraction_numerical': 0.004594180704441042, 'mean_word_length': 4.69281045751634, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4819025', 'n_tokens_mistral': 826, 'n_tokens_neox': 803, 'n_words': 170}