<commit_before>#!/usr/bin/python

import sys
import os
import glob
sys.path.append(os.path.join(os.path.dirname(__file__),"gen-py/thrift_solr/"))
sys.path.append(os.path.dirname(__file__) )

from thrift.transport import TSocket 
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.server import TServer 
from thrift.protocol.TBinaryProtocol import TBinaryProtocolAccelerated
from thrift.server.TProcessPoolServer import TProcessPoolServer

import ExtractorService
import sys
from readability.readability import Document


def extract_with_python_readability( raw_content ):
    doc = Document( raw_content )
    
    return [ u'' + doc.short_title().strip(),
             u'' + doc.summary().strip() ]

class ExtractorHandler:
    def extract_html( self, raw_html ):
        ret = extract_with_python_readability( raw_html )
        return ret

handler = ExtractorHandler()
processor = ExtractorService.Processor(handler)
listening_socket = TSocket.TServerSocket(port=9090)
tfactory = TTransport.TBufferedTransportFactory()
pfactory = TBinaryProtocol.TBinaryProtocolFactory()

# Test the extractor real quick; if it doesn't work, don't proceed to creating the server
test_html = "<html><body><p>Media Cloud</p></body></html>"
extracted_text = handler.extract_html( test_html )
if not extracted_text:
    raise ImportError("'readability' module has been imported, but I'm unable to extract anything with it")

server = TProcessPoolServer(processor, listening_socket, tfactory, pfactory)
server.setNumWorkers( 30 )
print ("[Server] Started")
server.serve()<commit_msg>Start Readability's Thrift server only if running as the main program<commit_after>#!/usr/bin/python

import sys
import os
import glob
sys.path.append(os.path.join(os.path.dirname(__file__),"gen-py/thrift_solr/"))
sys.path.append(os.path.dirname(__file__) )

from thrift.transport import TSocket 
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.server import TServer 
from thrift.protocol.TBinaryProtocol import TBinaryProtocolAccelerated
from thrift.server.TProcessPoolServer import TProcessPoolServer

import ExtractorService
import sys
from readability.readability import Document


def extract_with_python_readability( raw_content ):
    doc = Document( raw_content )
    
    return [ u'' + doc.short_title().strip(),
             u'' + doc.summary().strip() ]

class ExtractorHandler:
    def extract_html( self, raw_html ):
        ret = extract_with_python_readability( raw_html )
        return ret


if __name__ == "__main__":

    handler = ExtractorHandler()
    processor = ExtractorService.Processor(handler)
    listening_socket = TSocket.TServerSocket(port=9090)
    tfactory = TTransport.TBufferedTransportFactory()
    pfactory = TBinaryProtocol.TBinaryProtocolFactory()

    # Test the extractor real quick; if it doesn't work, don't proceed to creating the server
    test_html = "<html><body><p>Media Cloud</p></body></html>"
    extracted_text = handler.extract_html( test_html )
    if not extracted_text:
        raise ImportError("'readability' module has been imported, but I'm unable to extract anything with it")

    server = TProcessPoolServer(processor, listening_socket, tfactory, pfactory)
    server.setNumWorkers( 30 )
    print ("[Server] Started")
    server.serve()
{'fraction_non_alphanumeric': 0.09223445403153824, 'fraction_numerical': 0.0035703659625111574, 'mean_word_length': 5.014311270125224, 'pattern_counts': {'":': 1, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7655438', 'n_tokens_mistral': 986, 'n_tokens_neox': 957, 'n_words': 256}