<commit_before># /usr/bin/env python3

import logging
import sys
import re
import os
import asyncio

logging.basicConfig(
    stream=sys.stderr, level=logging.INFO,
    format='[%(levelname)s](%(funcName)s/%(lineno)d) %(message)s',
)
logger = logging.getLogger(__name__)


def get_direct_image_url(image_url):
    """
    Used to trasform image html url to image url.
    Support: imgur
    """
    imgur_id = re.findall('http[s]?://imgur.com/((?:[a-zA-Z]|[0-9])+)', image_url)
    if imgur_id:
        image_url = 'http://i.imgur.com/{0}.png'.format(imgur_id[0])
    return image_url


def find_image_urls(article_json):
    # https://stackoverflow.com/questions/169625/regex-to-check-if-valid-url-that-ends-in-jpg-png-or-gif
    # https://stackoverflow.com/questions/6883049/regex-to-find-urls-in-string-in-python
    image_urls = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+\.(?:jpg|gif|png)|http://imgur.com/(?:[a-zA-Z]|[0-9])+', article_json['version'][0]['content'])
    return image_urls


@asyncio.coroutine
def download_image(session, image_url, image_folder, image_name):
    logger.info('download@%s', image_url)
    response = yield from session.request('get', image_url)
    if response.status != 200:
        logger.error('Download image failed: %s', image_url)
        return -1
    image_path = os.path.join(image_folder, image_name)
    image = yield from response.read()
    with open(image_path, 'wb') as f:
        f.write(image)
    yield from response.release()
    return 0


@asyncio.coroutine
def get_articles_of_page(session, forum_name, page_index):
    url = 'https://www.dcard.tw/api/forum/{0}/{1}/'.format(forum_name, page_index)
    response = yield from session.request('get', url)
    if response.status != 200:
        logger.error('Get article list failed: %s', url)
        session.close()
        return None
    articles_json = yield from response.json()
    yield from response.release()
    session.close()
    return articles_json


@asyncio.coroutine
def get_article(session, article_id):
    url = 'https://www.dcard.tw/api/post/all/{0}'.format(article_id)
    response = yield from session.request('get', url)
    if response.status != 200:
        logger.error('Get article content failed: %s', url)
        session.close()
        return None
    article_json = yield from response.json()
    yield from response.release()
    return article_json<commit_msg>Return the same type if error in functions<commit_after># /usr/bin/env python3

import logging
import sys
import re
import os
import asyncio

logging.basicConfig(
    stream=sys.stderr, level=logging.INFO,
    format='[%(levelname)s](%(funcName)s/%(lineno)d) %(message)s',
)
logger = logging.getLogger(__name__)


def get_direct_image_url(image_url):
    """
    Used to trasform image html url to image url.
    Support: imgur
    """
    imgur_id = re.findall('http[s]?://imgur.com/((?:[a-zA-Z]|[0-9])+)', image_url)
    if imgur_id:
        image_url = 'http://i.imgur.com/{0}.png'.format(imgur_id[0])
    return image_url


def find_image_urls(article_json):
    # https://stackoverflow.com/questions/169625/regex-to-check-if-valid-url-that-ends-in-jpg-png-or-gif
    # https://stackoverflow.com/questions/6883049/regex-to-find-urls-in-string-in-python
    image_urls = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+\.(?:jpg|gif|png)|http://imgur.com/(?:[a-zA-Z]|[0-9])+', article_json['version'][0]['content'])
    return image_urls


@asyncio.coroutine
def download_image(session, image_url, image_folder, image_name):
    logger.info('download@%s', image_url)
    response = yield from session.request('get', image_url)
    if response.status != 200:
        logger.error('Download image failed: %s', image_url)
        return -1
    image_path = os.path.join(image_folder, image_name)
    image = yield from response.read()
    with open(image_path, 'wb') as f:
        f.write(image)
    yield from response.release()
    return 0


@asyncio.coroutine
def get_articles_of_page(session, forum_name, page_index):
    url = 'https://www.dcard.tw/api/forum/{0}/{1}/'.format(forum_name, page_index)
    response = yield from session.request('get', url)
    if response.status != 200:
        logger.error('Get article list failed: %s', url)
        session.close()
        return []
    articles_json = yield from response.json()
    yield from response.release()
    session.close()
    return articles_json


@asyncio.coroutine
def get_article(session, article_id):
    url = 'https://www.dcard.tw/api/post/all/{0}'.format(article_id)
    response = yield from session.request('get', url)
    if response.status != 200:
        logger.error('Get article content failed: %s', url)
        session.close()
        return {}
    article_json = yield from response.json()
    yield from response.release()
    return article_json
{'fraction_non_alphanumeric': 0.16568527918781725, 'fraction_numerical': 0.016649746192893403, 'mean_word_length': 3.595149253731343, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4716921', 'n_tokens_mistral': 1762, 'n_tokens_neox': 1707, 'n_words': 322}