<commit_before>"""Actual crawler core code"""

import codecs
from contextlib import closing
import functools
import mimetypes
import os.path as osp
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO
import time

import markdown

from docido_sdk.core import Component, implements
from docido_sdk.crawler import ICrawler
from docido_sdk.toolbox.rate_limits import teb_retry
from docido_sdk.toolbox.date_ext import timestamp_ms
from docido_sdk.toolbox.text import to_unicode
from dateutil import parser

from dpc_trello.trello import TrelloClient, TrelloClientException

UTF8_CODEC = codecs.lookup("utf8")
CREATE_CARD_ACTION = 'createCard'
COMMENT_CARD_ACTION = 'commentCard'

def create_trello_client(token):
    """ Create and return a trello client from a provided oauth token

    :param token: a docido_sdk specified OauthToken

    :return: a trello client instance
    """
    return TrelloClient(
        consumer_key=token.consumer_key,
        token=token.access_token
    )


def date_to_timestamp(str_date):
    """ Convert an str formatted date to an UNIX timestamp

    :param str str_date: An str formatted date

    :return: A valid UNIX timestamp
    :rtype: int
    """
    # As pylint cannot infer parser.parse return type because multiple return
    # types are possible the specific induced error is disabled
    # pylint: disable=no-member
    date = parser.parse(str_date)
    return int(
        time.mktime(date.utctimetuple()) * 1e3 + date.microsecond / 1e3
    )


def pick_preview(previews, full=False):
    """ Given a list of preview will pick the one matching specs or the closest

    The returned previews (if any) will respect the following conditions:
        * Biggest one among those which heights and width are lesser than 300
        * Smallest available one if no suitable candidate for first case is
        found

    :param list previews: A list of trello obtained item previews

    :return: The closest specs matching preview or None if no previews is
    available
    """
    def preview_size(preview):
        """ Compute a size for image by adding height and width so previews can
        easily be filtered and ordered

        :param previwe: A preview to compute size for

        :return: The computed size
        :rtype: int
        """
        return preview['height'] + preview['width']
    if not any(previews):
        return None
    candidates = [
        p for p in previews if p['height'] < 300 and p['width'] < 300
    ]
    if any(candidates):
        preview = max(candidates, key=preview_size)
    else:
        preview = min(previews, key=preview_size)
    if full:
        return preview
    return preview[u'url']


class file_type(object):
    TYPE_CHECK = {
        '3gp': 'video',
        'aaf': 'video',
        'aiff': 'sound',
        'ami': 'document',
        'ape': 'sound',<commit_msg>Pick biggest preview available<commit_after>"""Actual crawler core code"""

import codecs
from contextlib import closing
import functools
import mimetypes
import os.path as osp
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO
import time

import markdown

from docido_sdk.core import Component, implements
from docido_sdk.crawler import ICrawler
from docido_sdk.toolbox.rate_limits import teb_retry
from docido_sdk.toolbox.date_ext import timestamp_ms
from docido_sdk.toolbox.text import to_unicode
from dateutil import parser

from dpc_trello.trello import TrelloClient, TrelloClientException

UTF8_CODEC = codecs.lookup("utf8")
CREATE_CARD_ACTION = 'createCard'
COMMENT_CARD_ACTION = 'commentCard'

def create_trello_client(token):
    """ Create and return a trello client from a provided oauth token

    :param token: a docido_sdk specified OauthToken

    :return: a trello client instance
    """
    return TrelloClient(
        consumer_key=token.consumer_key,
        token=token.access_token
    )


def date_to_timestamp(str_date):
    """ Convert an str formatted date to an UNIX timestamp

    :param str str_date: An str formatted date

    :return: A valid UNIX timestamp
    :rtype: int
    """
    # As pylint cannot infer parser.parse return type because multiple return
    # types are possible the specific induced error is disabled
    # pylint: disable=no-member
    date = parser.parse(str_date)
    return int(
        time.mktime(date.utctimetuple()) * 1e3 + date.microsecond / 1e3
    )


def __pick_preview(previews, full=False):
    """ Given a list of preview will pick the one matching specs or the closest

    The returned previews (if any) will respect the following conditions:
        * Biggest one among those which heights and width are lesser than 300
        * Smallest available one if no suitable candidate for first case is
        found

    :param list previews: A list of trello obtained item previews

    :return: The closest specs matching preview or None if no previews is
    available
    """
    def preview_size(preview):
        """ Compute a size for image by adding height and width so previews can
        easily be filtered and ordered

        :param previwe: A preview to compute size for

        :return: The computed size
        :rtype: int
        """
        return preview['height'] + preview['width']
    if not any(previews):
        return None
    candidates = [
        p for p in previews if p['height'] < 300 and p['width'] < 300
    ]
    if any(candidates):
        preview = max(candidates, key=preview_size)
    else:
        preview = min(previews, key=preview_size)
    if full:
        return preview
    return preview[u'url']

def pick_preview(previews, full=False):
    if not any(previews):
        return None
    preview = max(previews, key=lambda p: p['height'] + p['width'])
    if full:
        return preview
    else:
        return preview['url']



class file_type(object):
    TYPE_CHECK = {
        '3gp': 'video',
        'aaf': 'video',
        'aiff': 'sound',
        'ami': 'document',
        'ape': 'sound',
{'fraction_non_alphanumeric': 0.07309024913380631, 'fraction_numerical': 0.00527965682230655, 'mean_word_length': 3.1322426721199728, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '473170', 'n_tokens_mistral': 1795, 'n_tokens_neox': 1708, 'n_words': 639}