<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
A wrapper for UCK's new interface
"""
# @author chengdujin
# @contact <EMAIL>
# @created Aug. 25, 2013


import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append("..")

from config import hparser
from config import logger
import image_helper
import urllib2

# CONSTANTS
from config import UCK_TIMEOUT
from config import UCK_TRANSCODING_NEW


def _collect_images(content):
    """
    find all images from the content
    """
    return image_helper.find_images(content)


def _transcode(link):
    """
    send link to uck and get the data
    """
    try:
        html = urllib2.urlopen(
            '%s%s' % (UCK_TRANSCODING_NEW, link), timeout=UCK_TIMEOUT).read()
        data = urllib2.unquote(hparser.unescape(html))
        return data
    except Exception as k:
        logger.error(str(k))
        return None


def _extract(link):
    """
    extract title, content and images
    """
    data_string = _transcode(link)
    if data_string:
        data = eval(data_string)
<commit_msg>added syntax checker for new uck output<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-

"""
A wrapper for UCK's new interface
"""
# @author chengdujin
# @contact <EMAIL>
# @created Aug. 25, 2013


import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append("..")

from config import hparser
from config import logger
import image_helper
import urllib2

# CONSTANTS
from config import UCK_TIMEOUT
from config import UCK_TRANSCODING_NEW


def _collect_images(content):
    """
    find all images from the content
    """
    return image_helper.find_images(content)


def _transcode(link):
    """
    send link to uck and get the data
    """
    try:
        html = urllib2.urlopen(
            '%s%s' % (UCK_TRANSCODING_NEW, link), timeout=UCK_TIMEOUT).read()
        data = urllib2.unquote(hparser.unescape(html))
        return data
    except Exception as k:
        logger.error(str(k))
        return None


def _extract(link):
    """
    extract title, content and images
    """
    data_string = _transcode(link)
    if data_string:
        # syntax checker
        try:
            eval(data_string)
        except Exception:
            logger.info('Invalid syntax found for New UCK output')
            return None, None, None
        
        data = eval(data_string)

{'fraction_non_alphanumeric': 0.10597140454163162, 'fraction_numerical': 0.009251471825063078, 'mean_word_length': 2.8432956381260097, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4110453', 'n_tokens_mistral': 785, 'n_tokens_neox': 756, 'n_words': 213}