<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
text2img converts text into an image
"""
# @author chengdujin
# @contact <EMAIL>
# @created Nov. 15, 2013


import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

import chardet
from config.settings import logger
from PIL import Image, FontFile, ImageFont, ImageDraw
import nltk
import re
import subprocess
import textwrap
import tinysegmenter

# CONSTATNS
from config.settings import CATEGORY_IMAGE_SIZE
from config.settings import DATA_PATH
from config.settings import FONT_PATH_EN
from config.settings import FONT_PATH_IN
from config.settings import FONT_PATH_JA
from config.settings import FONT_PATH_PT
from config.settings import FONT_PATH_TH
from config.settings import FONT_PATH_ZH
from config.settings import IMAGES_LOCAL_DIR
from config.settings import IMAGES_PUBLIC_DIR
from config.settings import TEXT_WIDTH_EN
from config.settings import TEXT_WIDTH_IN
from config.settings import TEXT_WIDTH_JA
from config.settings import TEXT_WIDTH_PT
from config.settings import TEXT_WIDTH_TH
from config.settings import TEXT_WIDTH_ZH
from config.settings import THAI_WORDSEG
from config.settings import THAI_WORDSEG_DICT


class Text2Image:

    """
    Converts text into image
    """

    def __init__(self, language=None, text=None, textimage_relative_path=None, background_color='#000000', font_color='#FFFFFF'):
        if not language or not text or not textimage_relative_path:
            logger.error("Method malformed!")

        self._language = language
        self._text = text
        self._background_color = background_color
        self._font_color = font_color
        self._textimage_relative_path = textimage_relative_path
        self._font_size = 1
        self._image = None
        self._draw = None
        self._font = None

        # danamic loading specific import
        FONT_PATH = 'FONT_PATH_%s' % self._language.upper()
        TEXT_WIDTH = 'TEXT_WIDTH_%s' % self._language.upper()
        self._font_path = eval(FONT_PATH)
        self._text_width = eval(TEXT_WIDTH)

        self._set_background()
        self._add_text_to_image()

    def _set_background(self):
        """
        set the image background
        """
        try:
            #self._image = Image.new("RGB", CATEGORY_IMAGE_SIZE, self._background_color)
            self._image = Image.open("%s/home_bg.png" % DATA_PATH)
            self._draw = ImageDraw.Draw(self._image)
        except Exception as k:
            logger.error(str(k))
            return None

    def _set_font_size(self, line=None):
        """
        set font size on the image
        """
        if not line:
            logger.error("Method malformed!")
            return None

        try:
            img_fraction = 0.83
            self._font = ImageFont.truetype(self._font_path, self._font_size)

            # adjust font size by text
            while self._font.getsize(line)[0] < img_fraction * self._image.size[0]:
                self._font_size = self._font_size + 1
                # print self._font_size, self._font.getsize(line)[0]
                self._font = ImageFont.truetype(
                    self._font_path, self._font_size)
        except Exception as k:
            logger.error(str(k))
            return None

    def _parse_text(self):
        """
        use nltk or other engines to split text into sentences
        """
        try:
            sentences = None

            # convert to appropriate encoding
            if isinstance(self._text, str):
                self._text = self._text.decode(
                    chardet.detect(self._text)['encoding'], 'ignore')

            # special: thai, arabic
            if self._language == 'zh':
                cj_punctuation = u"""-|〃|。|．|\.|!|！|\?|？|、|-|〟|〰|-|-|＊|，|,|-|／|：|-|；|-|-|＿|-|･|‐|-|―|-|…|-|‧|﹏"""
                sentences = re.split(cj_punctuation, self._text)
            if self._language == 'ja':
                segmenter = tinysegmenter.TinySegmenter()
                sentences = segmenter.tokenize(self._text)
                sentences = [sentence for sentence in sentences if sentence.strip()]
            elif self._language == 'th':
                command = 'echo "%s" | %s %s/scw.conf %s' % (
                    str(self._text), THAI_WORDSEG, THAI_WORDSEG_DICT, THAI_WORDSEG_DICT)
                response = subprocess.Popen(
                    command, stdout=subprocess.PIPE, shell=True)
                content, error = response.communicate()
                if not error and content:
                    if 'error' not in content or 'permission' not in content:
                        content = content.strip()
                        paragraphs = [paragraph.strip()
                                      for paragraph in content.split('\n') if paragraph.strip()]
                        for paragraph in paragraphs:
                            # modes[0]: the orginal
                            # modes[1]: phrase seg
                            # modes[2]: basic wordseg
                            # modes[3]: subphrase seg
                            modes = [mode.strip()
                                     for mode in paragraph.split('\t') if mode.strip()]
                            # for mode in modes:
                            #    print  str(mode)
                            # u'|' is very crucial(, instead of '|')
                            sentences = [
                                sentence for sentence in modes[2].split(u'|') if sentence]
            else:  # latin-based
                sentences = nltk.sent_tokenize(self._text)

            # remove void lines
            sentences = [sentence.strip()
                         for sentence in sentences if sentence.strip()]

            if self._language in ['en', 'in', 'pt', 'ja', 'zh', 'th']:
                lines = []
                for sentence in sentences:
                    splits = textwrap.wrap(sentence, self._text_width)
                    # print str(' | '.join(splits))
                    lines.extend(splits)

                sentences = []
                previous_line = None
                for index in xrange(len(lines)):
                    current_line = lines[index]
                    # print str(current_line)
                    if previous_line:
                        # print '  previous', str(previous_line)
                        possible_line = ""
                        if self._language in ['en', 'in', 'pt']:
                            possible_line = u"%s %s" % (previous_line, current_line)
                        else:
                            possible_line = u"%s%s" % (previous_line, current_line)

                        if len(textwrap.wrap(possible_line, self._text_width)) == 1:
                            # print '    possible', str(possible_line)
                            previous_line = possible_line
                        else:
                            # print '    no possible', str(current_line)
                            sentences.append(previous_line)
                            previous_line = current_line
                    else:
                        # print '  no previous', str(current_line)
                        previous_line = current_line
                    # if current_line is the last one, add it to list
                    if index + 1 == len(lines):
                        sentences.append(previous_line)

            # print
            # for s in sentences:
            #    print '*', str(s)
            return sentences
        except Exception as k:
            logger.error(str(k))
            return None

    def _add_text_to_image(self):
        """
        convert text to image
        """
        sentences = self._parse_text()
        # pick up the longest sentence
        # length evaluation should be done in str, not unicode
        longest_sentence = max([(len(str(sentence)), index)
                               for index, sentence in enumerate(sentences)], key=lambda x: x[0])
        # print '[longest sentence]', str(sentences[longest_sentence[1]])
        # longest_sentence: (sentence_length, sentence_index)
        self._set_font_size(sentences[longest_sentence[1]])

        try:
            for count, sentence in enumerate(sentences):
                width, height = self._font.getsize(sentence)
                if ((count + 1.5) * height + 30 + count * 16) > self._image.size[1]:
                    sentence = ". . . . . ."
                    self._draw.text(((self._image.size[0] - 30) / 2 - self._font.getsize(sentence)[0] / 2, (
                        (count - 1) * height) + 30 + height / 2 + count * 8), sentence, fill=self._font_color, font=self._font)
                    break
                else:
                    # print str(sentence)
                    self._draw.text(
                        (40, (count * height) + 30 + count * 8), sentence, fill=self._font_color, font=self._font)

            textimage_local_path = "%s%s" % (
                IMAGES_LOCAL_DIR, self._textimage_relative_path)
            # print textimage_local_path
            self._image.save(textimage_local_path, "PNG")
        except Exception as k:
            logger.error(str(k))

    def get_image(self):
        textimage_public_path = "%s%s" % (<commit_msg>Adjust text2img parameters<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
text2img converts text into an image
"""
# @author chengdujin
# @contact <EMAIL>
# @created Nov. 15, 2013


import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

import chardet
from config.settings import logger
from PIL import Image, FontFile, ImageFont, ImageDraw
import nltk
import re
import subprocess
import textwrap
import tinysegmenter

# CONSTATNS
from config.settings import CATEGORY_IMAGE_SIZE
from config.settings import DATA_PATH
from config.settings import FONT_PATH_EN
from config.settings import FONT_PATH_IN
from config.settings import FONT_PATH_JA
from config.settings import FONT_PATH_PT
from config.settings import FONT_PATH_TH
from config.settings import FONT_PATH_ZH
from config.settings import IMAGES_LOCAL_DIR
from config.settings import IMAGES_PUBLIC_DIR
from config.settings import TEXT_WIDTH_EN
from config.settings import TEXT_WIDTH_IN
from config.settings import TEXT_WIDTH_JA
from config.settings import TEXT_WIDTH_PT
from config.settings import TEXT_WIDTH_TH
from config.settings import TEXT_WIDTH_ZH
from config.settings import THAI_WORDSEG
from config.settings import THAI_WORDSEG_DICT


class Text2Image:

    """
    Converts text into image
    """

    def __init__(self, language=None, text=None, textimage_relative_path=None, background_color='#000000', font_color='#FFFFFF'):
        if not language or not text or not textimage_relative_path:
            logger.error("Method malformed!")

        self._language = language
        self._text = text
        self._background_color = background_color
        self._font_color = font_color
        self._textimage_relative_path = textimage_relative_path
        self._font_size = 1
        self._image = None
        self._draw = None
        self._font = None

        # danamic loading specific import
        FONT_PATH = 'FONT_PATH_%s' % self._language.upper()
        TEXT_WIDTH = 'TEXT_WIDTH_%s' % self._language.upper()
        self._font_path = eval(FONT_PATH)
        self._text_width = eval(TEXT_WIDTH)

        self._set_background()
        self._add_text_to_image()

    def _set_background(self):
        """
        set the image background
        """
        try:
            #self._image = Image.new("RGB", CATEGORY_IMAGE_SIZE, self._background_color)
            self._image = Image.open("%s/home_bg.png" % DATA_PATH)
            self._draw = ImageDraw.Draw(self._image)
        except Exception as k:
            logger.error(str(k))
            return None

    def _set_font_size(self, line=None):
        """
        set font size on the image
        """
        if not line:
            logger.error("Method malformed!")
            return None

        try:
            img_fraction = 0.67
            self._font = ImageFont.truetype(self._font_path, self._font_size)

            # adjust font size by text
            while self._font.getsize(line)[0] < img_fraction * self._image.size[0]:
                self._font_size = self._font_size + 1
                # print self._font_size, self._font.getsize(line)[0]
                self._font = ImageFont.truetype(
                    self._font_path, self._font_size)
        except Exception as k:
            logger.error(str(k))
            return None

    def _parse_text(self):
        """
        use nltk or other engines to split text into sentences
        """
        try:
            sentences = None

            # convert to appropriate encoding
            if isinstance(self._text, str):
                self._text = self._text.decode(
                    chardet.detect(self._text)['encoding'], 'ignore')

            # special: thai, arabic
            if self._language == 'zh':
                cj_punctuation = u"""-|〃|。|．|\.|!|！|\?|？|、|-|〟|〰|-|-|＊|，|,|-|／|：|-|；|-|-|＿|-|･|‐|-|―|-|…|-|‧|﹏"""
                sentences = re.split(cj_punctuation, self._text)
            if self._language == 'ja':
                segmenter = tinysegmenter.TinySegmenter()
                sentences = segmenter.tokenize(self._text)
                sentences = [sentence for sentence in sentences if sentence.strip()]
            elif self._language == 'th':
                command = 'echo "%s" | %s %s/scw.conf %s' % (
                    str(self._text), THAI_WORDSEG, THAI_WORDSEG_DICT, THAI_WORDSEG_DICT)
                response = subprocess.Popen(
                    command, stdout=subprocess.PIPE, shell=True)
                content, error = response.communicate()
                if not error and content:
                    if 'error' not in content or 'permission' not in content:
                        content = content.strip()
                        paragraphs = [paragraph.strip()
                                      for paragraph in content.split('\n') if paragraph.strip()]
                        for paragraph in paragraphs:
                            # modes[0]: the orginal
                            # modes[1]: phrase seg
                            # modes[2]: basic wordseg
                            # modes[3]: subphrase seg
                            modes = [mode.strip()
                                     for mode in paragraph.split('\t') if mode.strip()]
                            # for mode in modes:
                            #    print  str(mode)
                            # u'|' is very crucial(, instead of '|')
                            sentences = [
                                sentence for sentence in modes[2].split(u'|') if sentence]
            else:  # latin-based
                sentences = nltk.sent_tokenize(self._text)

            # remove void lines
            sentences = [sentence.strip()
                         for sentence in sentences if sentence.strip()]

            if self._language in ['en', 'in', 'pt', 'ja', 'zh', 'th']:
                lines = []
                for sentence in sentences:
                    splits = textwrap.wrap(sentence, self._text_width)
                    # print str(' | '.join(splits))
                    lines.extend(splits)

                sentences = []
                previous_line = None
                for index in xrange(len(lines)):
                    current_line = lines[index]
                    # print str(current_line)
                    if previous_line:
                        # print '  previous', str(previous_line)
                        possible_line = ""
                        if self._language in ['en', 'in', 'pt']:
                            possible_line = u"%s %s" % (previous_line, current_line)
                        else:
                            possible_line = u"%s%s" % (previous_line, current_line)

                        if len(textwrap.wrap(possible_line, self._text_width)) == 1:
                            # print '    possible', str(possible_line)
                            previous_line = possible_line
                        else:
                            # print '    no possible', str(current_line)
                            sentences.append(previous_line)
                            previous_line = current_line
                    else:
                        # print '  no previous', str(current_line)
                        previous_line = current_line
                    # if current_line is the last one, add it to list
                    if index + 1 == len(lines):
                        sentences.append(previous_line)

            # print
            # for s in sentences:
            #    print '*', str(s)
            return sentences
        except Exception as k:
            logger.error(str(k))
            return None

    def _add_text_to_image(self):
        """
        convert text to image
        """
        sentences = self._parse_text()
        # pick up the longest sentence
        # length evaluation should be done in str, not unicode
        longest_sentence = max([(len(str(sentence)), index)
                               for index, sentence in enumerate(sentences)], key=lambda x: x[0])
        # print '[longest sentence]', str(sentences[longest_sentence[1]])
        # longest_sentence: (sentence_length, sentence_index)
        self._set_font_size(sentences[longest_sentence[1]])

        try:
            for count, sentence in enumerate(sentences):
                width, height = self._font.getsize(sentence)
                if ((count + 2) * height + 30 + count * 16) > self._image.size[1]:
                    sentence = ". . . . . ."
                    self._draw.text(((self._image.size[0] - 30) / 2 - self._font.getsize(sentence)[0] / 2, (
                        (count - 1) * height) + 30 + height / 2 + count * 8), sentence, fill=self._font_color, font=self._font)
                    break
                else:
                    # print str(sentence)
                    self._draw.text(
                        (40, (count * height) + 30 + count * 8), sentence, fill=self._font_color, font=self._font)

            textimage_local_path = "%s%s" % (
                IMAGES_LOCAL_DIR, self._textimage_relative_path)
            # print textimage_local_path
            self._image.save(textimage_local_path, "PNG")
        except Exception as k:
            logger.error(str(k))

    def get_image(self):
        textimage_public_path = "%s%s" % (
{'fraction_non_alphanumeric': 0.09805177475313584, 'fraction_numerical': 0.006084867894315452, 'mean_word_length': 1.6493212669683257, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4414135', 'n_tokens_mistral': 5247, 'n_tokens_neox': 4945, 'n_words': 1242}