<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
call baidu uck's api to transcode a web page
"""
# @author chengdujin
# @contact <EMAIL>
# @created Jan 2, 2013


import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

from BeautifulSoup import BeautifulSoup
from BeautifulSoup import NavigableString
from BeautifulSoup import Tag
from data_processor import image_helper
from data_processor import thumbnail
import re
import urllib2

# CONSTANTS
from config import UCK_TIMEOUT
from config import UCK_TRANSCODING


# TODO: test the code
# TODO: remove code that sanitize too much
def _sanitize(content):
    ''''''
    soup = BeautifulSoup(content.decode('utf-8', 'ignore'))
    # remove all <span>
    for span in soup.findAll('span'):
        span.extract()
    # sanitize <a>
    for a in soup.findAll('a'):
        img = a.find('img')
        if img:
            a.replaceWith(img)
        else:  # it might be a simple href
            a.replaceWith(a.text)
    # remove img prefix
    for img in soup.findAll('img'):
        img_source = img.get('src')
        if img_source:<commit_msg>added a line of docstring to _sanitize<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
call baidu uck's api to transcode a web page
"""
# @author chengdujin
# @contact <EMAIL>
# @created Jan 2, 2013


import sys
reload(sys)
sys.setdefaultencoding('UTF-8')
sys.path.append('..')

from BeautifulSoup import BeautifulSoup
from BeautifulSoup import NavigableString
from BeautifulSoup import Tag
from data_processor import image_helper
from data_processor import thumbnail
import re
import urllib2

# CONSTANTS
from config import UCK_TIMEOUT
from config import UCK_TRANSCODING


# TODO: test the code
# TODO: remove code that sanitize too much
def _sanitize(content):
    """
    modified uck content to suit news needs
    """
    soup = BeautifulSoup(content.decode('utf-8', 'ignore'))
    # remove all <span>
    for span in soup.findAll('span'):
        span.extract()
    # sanitize <a>
    for a in soup.findAll('a'):
        img = a.find('img')
        if img:
            a.replaceWith(img)
        else:  # it might be a simple href
            a.replaceWith(a.text)
    # remove img prefix
    for img in soup.findAll('img'):
        img_source = img.get('src')
        if img_source:
{'fraction_non_alphanumeric': 0.10824742268041238, 'fraction_numerical': 0.007731958762886598, 'mean_word_length': 3.1813285457809695, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '745161', 'n_tokens_mistral': 754, 'n_tokens_neox': 739, 'n_words': 236}