<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
call baidu uck's api to transcode a web page
"""
# @author chengdujin
# @contact <EMAIL>
# @created Jan 2, 2013


import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

from BeautifulSoup import BeautifulSoup, NavigableString, Tag
from config.settings import hparser
from config.settings import logger
import illustrator
from illustrator import NormalizedImage
import urllib2

# CONSTANTS
from config.settings import UCK_TIMEOUT
from config.settings import UCK_TRANSCODING


# TODO: test the code
# TODO: remove code that sanitize too much
def _sanitize(content=None, referer=None):
    """
    modified uck content to suit news needs
    """
    if not content:
        return None

    try:
        soup = BeautifulSoup(content.decode('utf-8', 'ignore'))
        # remove all <span>
        for span in soup.findAll('span'):
            span.extract()

        # sanitize <a>
        for a in soup.findAll('a'):
            img = a.find('img')
            if img:
                a.replaceWith(img)
            else:  # it might be a simple href
                a.replaceWith(a.text)

        # remove img prefix
        for img in soup.findAll('img'):
            img_source = img.get('src')
            if img_source:
                img_tuple = img_source.rpartition('src=')
                img['src'] = img_tuple[2]
                # call NormalizedImage
                width = height = None
                try:
                    ni = NormalizedImage(image['src'], referer)
                    width, height = ni.get_image_size()
                except Exception as k:
                    logger.info('Problem [%s] for Source [%s]' % (str(k), str(image['src'])))
                    continue
                if width >= 480:
                    img['width'] = '100%'
                    img['height'] = 'auto'

        # clear away useless style
        for style in soup.findAll('div', style='border-top:none;'):
            img = style.find('img')
            if not img:
                if not style.find('p'):
                    style.extract()
            else:
                style.replaceWith(img)

        # remove navigble strings and <div>
        for component in soup.contents:
            if isinstance(component, NavigableString):
                if len(component.string.split()) < 10:
                    component.extract()
            elif isinstance(component, Tag):
                if component.name == 'div':
                    if not component.find('p'):
                        component.extract()

        # filter item
        img_count = 0
        for item in soup.contents:
            if isinstance(item, Tag):
                if item.name == 'img':
                    img_count = img_count + 1
        if img_count == len(soup.contents):
            return None
        else:
            return ''.join([str(item) for item in soup.contents])
    except Exception as k:
        logger.error(str(k))
        return None


def _collect_images(data):
    """
    find all possible images
    1. image_list
    2. images in the new content
    """
    if not data:
        return None

    try:
        images = []
        # first try to find images in image_list
        if 'image_list' in data and data.get('image_list'):
            for image in data.get('image_list'):
                if 'src' in image and image['src']:
                    image_normalized = illustrator.find_image(image['src'].strip())
                    if image_normalized:
                        images.append(image_normalized)

        # then try to find images in the content
        images_from_content = illustrator.find_images(data['content'])
        if images_from_content:
            images.extend(images_from_content)

        # remove duplicated ones
        images = illustrator.dedup_images(images) if images else None
        return images
    except Exception as k:
        logger.error(str(k))
        return None


def _extract(data=None, referer=None):
    """
    extract images and text content
    """
    if not data:
        logger.error('Received no data from UCK server.')
        return None, None, None

    successful = int(data['STRUCT_PAGE_TYPE'])
    if successful == 0:
        logger.info('Cannot interpret the page! status != 1')
        return None, None, None

    try:
        # content
        content = data['content'].replace("\\", "")
        content = _sanitize(content, referer)

        # images
        images = _collect_images(data)
        images = images if images else None

        # title
        title = None
        if 'title' in data:
            title = data['title']

        return title, content, images
    except Exception as k:
        logger.error(str(k))<commit_msg>Add referer to collect_images in baidu_uck<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
call baidu uck's api to transcode a web page
"""
# @author chengdujin
# @contact <EMAIL>
# @created Jan 2, 2013


import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

from BeautifulSoup import BeautifulSoup, NavigableString, Tag
from config.settings import hparser
from config.settings import logger
import illustrator
from illustrator import NormalizedImage
import urllib2

# CONSTANTS
from config.settings import UCK_TIMEOUT
from config.settings import UCK_TRANSCODING


# TODO: test the code
# TODO: remove code that sanitize too much
def _sanitize(content=None, referer=None):
    """
    modified uck content to suit news needs
    """
    if not content:
        return None

    try:
        soup = BeautifulSoup(content.decode('utf-8', 'ignore'))
        # remove all <span>
        for span in soup.findAll('span'):
            span.extract()

        # sanitize <a>
        for a in soup.findAll('a'):
            img = a.find('img')
            if img:
                a.replaceWith(img)
            else:  # it might be a simple href
                a.replaceWith(a.text)

        # remove img prefix
        for img in soup.findAll('img'):
            img_source = img.get('src')
            if img_source:
                img_tuple = img_source.rpartition('src=')
                img['src'] = img_tuple[2]
                # call NormalizedImage
                width = height = None
                try:
                    ni = NormalizedImage(image['src'], referer)
                    width, height = ni.get_image_size()
                except Exception as k:
                    logger.info('Problem [%s] for Source [%s]' % (str(k), str(image['src'])))
                    continue
                if width >= 480:
                    img['width'] = '100%'
                    img['height'] = 'auto'

        # clear away useless style
        for style in soup.findAll('div', style='border-top:none;'):
            img = style.find('img')
            if not img:
                if not style.find('p'):
                    style.extract()
            else:
                style.replaceWith(img)

        # remove navigble strings and <div>
        for component in soup.contents:
            if isinstance(component, NavigableString):
                if len(component.string.split()) < 10:
                    component.extract()
            elif isinstance(component, Tag):
                if component.name == 'div':
                    if not component.find('p'):
                        component.extract()

        # filter item
        img_count = 0
        for item in soup.contents:
            if isinstance(item, Tag):
                if item.name == 'img':
                    img_count = img_count + 1
        if img_count == len(soup.contents):
            return None
        else:
            return ''.join([str(item) for item in soup.contents])
    except Exception as k:
        logger.error(str(k))
        return None


def _collect_images(data=None, referer=None):
    """
    find all possible images
    1. image_list
    2. images in the new content
    """
    if not data:
        return None

    try:
        images = []
        # first try to find images in image_list
        if 'image_list' in data and data.get('image_list'):
            for image in data.get('image_list'):
                if 'src' in image and image['src']:
                    image_normalized = illustrator.find_image(image['src'].strip(), referer)
                    if image_normalized:
                        images.append(image_normalized)

        # then try to find images in the content
        images_from_content = illustrator.find_images(data['content'])
        if images_from_content:
            images.extend(images_from_content)

        # remove duplicated ones
        images = illustrator.dedup_images(images) if images else None
        return images
    except Exception as k:
        logger.error(str(k))
        return None


def _extract(data=None, referer=None):
    """
    extract images and text content
    """
    if not data:
        logger.error('Received no data from UCK server.')
        return None, None, None

    successful = int(data['STRUCT_PAGE_TYPE'])
    if successful == 0:
        logger.info('Cannot interpret the page! status != 1')
        return None, None, None

    try:
        # content
        content = data['content'].replace("\\", "")
        content = _sanitize(content, referer)

        # images
        images = _collect_images(data, referer)
        images = images if images else None

        # title
        title = None
        if 'title' in data:
            title = data['title']

        return title, content, images
    except Exception as k:
        logger.error(str(k))
{'fraction_non_alphanumeric': 0.10006170300287948, 'fraction_numerical': 0.0049362402303578775, 'mean_word_length': 1.7873316136428776, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6176624', 'n_tokens_mistral': 2704, 'n_tokens_neox': 2575, 'n_words': 789}