<commit_before># -*- coding: utf-8 -*-
from logging import getLogger, StreamHandler, DEBUG
import os
from datetime import datetime
#
from serializer import Serializer
from ocrengine import OCREngine
from naivebayes import NaiveBayes
from dataprocessor import DataProcessor, ImageType

logger = getLogger('myapp.tweetbot')
if __name__ == "__main__":
    handler = StreamHandler()
    handler.setLevel(DEBUG)
    logger.setLevel(DEBUG)
    logger.addHandler(handler)

class Ranking(object):
    def __init__(self, config):
        self.ocr = OCREngine()
        self.naivebayes = NaiveBayes()
    def getResult(self, src):
        """
            @param {string} src
            @return {OCRDocument} doucument
        """
        pro = DataProcessor(src, ImageType.RAW)
        if pro.prepare() is None:
            logger.error('image error:{0}'.format(src))
            return None
        batch = pro.batch()
        temp_file_name = pro.save_tempfile(batch)
        
        doucument = self.ocr.recognize(temp_file_name)
        os.remove(temp_file_name)
        with Serializer.open_stream('../temp/corpus.txt', mode='a') as file:
            header = '#' + datetime.now().strftime('%F %T.%f')[:-3] + '\n'
            file.write(header)
            file.write('\n'.join(doucument.names()))
            file.write('\n')
        # ocr corpus data -> NaiveBayes classifier
        # ranking name swap
        for i, name in enumerate(self.naivebayes.predict_all(doucument.names(), doucument.countries)):
            doucument.ranking[i]['name'] = name
        doucument.dump()
        return doucument

def main():
    config = Serializer.load_json('../resource/setting.json')
    r = Ranking(config)
    # benchMark
    ele = ['../backup/hints/201702190825_0565e4fcbc166f00577cbd1f9a76f8c7.png',
        #    '../backup/test/201702191909_ac08ccbbb04f2a1feeb4f8aaa08ae008.png',
        #     '../backup/test/201702192006_2e268d7508c20aa00b22dfd41639d65e.png',
        ] * 2
    for l in ele:
        logger.info(l)
        doucument = r.getResult(l)
        logger.info(doucument.ranking)
if __name__ == "__main__":
    main()<commit_msg>remove save_tempfile<commit_after># -*- coding: utf-8 -*-
from logging import getLogger, StreamHandler, DEBUG
import os
import tempfile
from io import BytesIO
from datetime import datetime
#
from serializer import Serializer
from ocrengine import OCREngine
from naivebayes import NaiveBayes
from dataprocessor import DataProcessor, ImageType

logger = getLogger('myapp.tweetbot')
if __name__ == "__main__":
    handler = StreamHandler()
    handler.setLevel(DEBUG)
    logger.setLevel(DEBUG)
    logger.addHandler(handler)

class Ranking(object):
    def __init__(self, config):
        self.ocr = OCREngine()
        self.naivebayes = NaiveBayes()
    def getResult(self, src):
        """
            @param {string} src
            @return {OCRDocument} doucument
        """
        pro = DataProcessor(src, ImageType.RAW)
        if pro.prepare() is None:
            logger.error('image error:{0}'.format(src))
            return None
        buffer = pro.tobinary(pro.batch())
        temp_file_name = ''
        with tempfile.NamedTemporaryFile(delete=False) as temp:
            temp.write(buffer.getvalue())
            temp_file_name = temp.name
            logger.info(temp_file_name)
        doucument = self.ocr.recognize(temp_file_name)
        os.remove(temp_file_name)
        with Serializer.open_stream('../temp/corpus.txt', mode='a') as file:
            header = '#' + datetime.now().strftime('%F %T.%f')[:-3] + '\n'
            file.write(header)
            file.write('\n'.join(doucument.names()))
            file.write('\n')
        # ocr corpus data -> NaiveBayes classifier
        # ranking name swap
        for i, name in enumerate(self.naivebayes.predict_all(doucument.names(), doucument.countries)):
            doucument.ranking[i]['name'] = name
        doucument.dump()
        return doucument

def main():
    config = Serializer.load_json('../resource/setting.json')
    r = Ranking(config)
    # benchMark
    ele = ['../backup/hints/201702190825_0565e4fcbc166f00577cbd1f9a76f8c7.png',
            '../images/backup/2017-03-09_0525_Ielsord.png',
        #     '../backup/test/201702192006_2e268d7508c20aa00b22dfd41639d65e.png',
        ] * 1
    for l in ele:
        logger.info(l)
        doucument = r.getResult(l)
        logger.info(doucument.ranking)
if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.12270898524810013, 'fraction_numerical': 0.0386678587393831, 'mean_word_length': 2.623481781376518, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2304320', 'n_tokens_mistral': 1556, 'n_tokens_neox': 1442, 'n_words': 279}