<commit_before>from datetime import datetime
from scrapy.contrib.loader import XPathItemLoader
from scrapy.contrib.loader.processor import Compose, MapCompose, TakeFirst
from scrapy.selector import XmlXPathSelector
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import flatten

from stockcrawler.items import ReportItem


DATE_FORMAT = '%Y-%m-%d'

MAX_PER_SHARE_VALUE = 1000.0


class IntermediateValue(object):
    '''
    Intermediate data that serves as output of input processors, i.e., input
    of output processors. "Intermediate" is shorten as "imd" in later naming.

    '''
    def __init__(self, local_name, value, text, context, node=None):
        self.local_name = local_name
        self.value = value
        self.text = text
        self.context = context
        self.node = node

    def __cmp__(self, other):
        if self.value < other.value:
            return -1
        elif self.value > other.value:
            return 1
        return 0

    def __repr__(self):
        context_id = None
        if self.context:
            context_id = self.context.select('@id')[0].extract()
        return '(%s, %s, %s)' % (self.local_name, self.value, context_id)

    def is_member(self):
        return is_member(self.context)


class ExtractText(object):

    def __call__(self, value):
        if hasattr(value, 'select'):
            return value.select('./text()')[0].extract()
        return unicode(value)


class MatchEndDate(object):

    def __init__(self, data_type=str):
        self.data_type = data_type

    def __call__(self, value, loader_context):
        if not hasattr(value, 'select'):
            return IntermediateValue('', 0.0, '0', None)

        doc_end_date_str = loader_context['end_date']
        doc_type = loader_context['doc_type']
        selector = loader_context['selector']

        context_id = value.select('@contextRef')[0].extract()
        context = selector.select('//*[@id="%s"]' % context_id)[0]

        date = None
        try:
            date = context.select('.//*[local-name()="instant"]/text()')[0].extract()
        except (IndexError, ValueError):
            try:
                start_date_str = context.select('.//*[local-name()="startDate"]/text()')[0].extract()
                end_date_str = context.select('.//*[local-name()="endDate"]/text()')[0].extract()
                start_date = datetime.strptime(start_date_str, DATE_FORMAT)
                end_date = datetime.strptime(end_date_str, DATE_FORMAT)
                delta_days = (end_date - start_date).days
                if doc_type == '10-Q' and delta_days < 120 and delta_days > 60:
                    date = end_date
                elif doc_type == '10-K' and delta_days < 380 and delta_days > 350:
                    date = end_date
            except (IndexError, ValueError):
                pass
        else:
            date = datetime.strptime(date, DATE_FORMAT)

        if date:
            doc_end_date = datetime.strptime(doc_end_date_str, DATE_FORMAT)
            delta_days = (doc_end_date - date).days
            if abs(delta_days) < 30:
                try:
                    text = value.select('./text()')[0].extract()
                    val = self.data_type(text)
                except (IndexError, ValueError):
                    pass
                else:
                    local_name = value.select('local-name()')[0].extract()
                    return IntermediateValue(local_name, val, text, context, value)

        return None


class ImdSumMembersOr(object):

    def __init__(self, second_func=None):
        self.second_func = second_func

    def __call__(self, imd_values):
        members = []
        non_members = []
        for imd_value in imd_values:
            if imd_value.is_member():
                members.append(imd_value)
            else:
                non_members.append(imd_value)

        if members and len(members) == len(imd_values):
            return imd_sum(members)

        if imd_values:
            return self.second_func(non_members)
        return None


def imd_first(imd_values):
    if imd_values:
        return imd_values[0].value
    return None


def imd_max(imd_values):
    if imd_values:
        imd_value = max(imd_values)
        return imd_value.value
    return None


def imd_min(imd_values):
    if imd_values:
        imd_value = min(imd_values)
        return imd_value.value
    return None


def imd_sum(imd_values):
    return sum([v.value for v in imd_values])


def imd_get_revenues(imd_values):
    interest_elems = filter(lambda v: 'interest' in v.local_name.lower(), imd_values)
    if len(interest_elems) == len(imd_values):
        # HACK: An exceptional case for BBT
        # Revenues = InterestIncome + NoninterestIncome
        return imd_sum(imd_values)

    return imd_max(imd_values)


def imd_get_net_income(imd_values):
    values = filter(lambda v: '.' not in v.text, imd_values)
    return imd_min(values)


def imd_get_per_share_value(imd_values):
    if not imd_values:
        return None

    v = imd_values[0]
    value = v.value
    if value > MAX_PER_SHARE_VALUE:
        try:
            decimals = int(v.node.select('@decimals')[0].extract())
        except (AttributeError, IndexError):
            return None
        else:
            value *= pow(10, decimals - 2)
    return value if value <= MAX_PER_SHARE_VALUE else None


def imd_filter_member(imd_values):
    non_members = filter(lambda v: not v.is_member(), imd_values)
    if non_members:
        return non_members
    return imd_values

<commit_msg>Add comment on the hack.<commit_after>from datetime import datetime
from scrapy.contrib.loader import XPathItemLoader
from scrapy.contrib.loader.processor import Compose, MapCompose, TakeFirst
from scrapy.selector import XmlXPathSelector
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import flatten

from stockcrawler.items import ReportItem


DATE_FORMAT = '%Y-%m-%d'

MAX_PER_SHARE_VALUE = 1000.0


class IntermediateValue(object):
    '''
    Intermediate data that serves as output of input processors, i.e., input
    of output processors. "Intermediate" is shorten as "imd" in later naming.

    '''
    def __init__(self, local_name, value, text, context, node=None):
        self.local_name = local_name
        self.value = value
        self.text = text
        self.context = context
        self.node = node

    def __cmp__(self, other):
        if self.value < other.value:
            return -1
        elif self.value > other.value:
            return 1
        return 0

    def __repr__(self):
        context_id = None
        if self.context:
            context_id = self.context.select('@id')[0].extract()
        return '(%s, %s, %s)' % (self.local_name, self.value, context_id)

    def is_member(self):
        return is_member(self.context)


class ExtractText(object):

    def __call__(self, value):
        if hasattr(value, 'select'):
            return value.select('./text()')[0].extract()
        return unicode(value)


class MatchEndDate(object):

    def __init__(self, data_type=str):
        self.data_type = data_type

    def __call__(self, value, loader_context):
        if not hasattr(value, 'select'):
            return IntermediateValue('', 0.0, '0', None)

        doc_end_date_str = loader_context['end_date']
        doc_type = loader_context['doc_type']
        selector = loader_context['selector']

        context_id = value.select('@contextRef')[0].extract()
        context = selector.select('//*[@id="%s"]' % context_id)[0]

        date = None
        try:
            date = context.select('.//*[local-name()="instant"]/text()')[0].extract()
        except (IndexError, ValueError):
            try:
                start_date_str = context.select('.//*[local-name()="startDate"]/text()')[0].extract()
                end_date_str = context.select('.//*[local-name()="endDate"]/text()')[0].extract()
                start_date = datetime.strptime(start_date_str, DATE_FORMAT)
                end_date = datetime.strptime(end_date_str, DATE_FORMAT)
                delta_days = (end_date - start_date).days
                if doc_type == '10-Q' and delta_days < 120 and delta_days > 60:
                    date = end_date
                elif doc_type == '10-K' and delta_days < 380 and delta_days > 350:
                    date = end_date
            except (IndexError, ValueError):
                pass
        else:
            date = datetime.strptime(date, DATE_FORMAT)

        if date:
            doc_end_date = datetime.strptime(doc_end_date_str, DATE_FORMAT)
            delta_days = (doc_end_date - date).days
            if abs(delta_days) < 30:
                try:
                    text = value.select('./text()')[0].extract()
                    val = self.data_type(text)
                except (IndexError, ValueError):
                    pass
                else:
                    local_name = value.select('local-name()')[0].extract()
                    return IntermediateValue(local_name, val, text, context, value)

        return None


class ImdSumMembersOr(object):

    def __init__(self, second_func=None):
        self.second_func = second_func

    def __call__(self, imd_values):
        members = []
        non_members = []
        for imd_value in imd_values:
            if imd_value.is_member():
                members.append(imd_value)
            else:
                non_members.append(imd_value)

        if members and len(members) == len(imd_values):
            return imd_sum(members)

        if imd_values:
            return self.second_func(non_members)
        return None


def imd_first(imd_values):
    if imd_values:
        return imd_values[0].value
    return None


def imd_max(imd_values):
    if imd_values:
        imd_value = max(imd_values)
        return imd_value.value
    return None


def imd_min(imd_values):
    if imd_values:
        imd_value = min(imd_values)
        return imd_value.value
    return None


def imd_sum(imd_values):
    return sum([v.value for v in imd_values])


def imd_get_revenues(imd_values):
    interest_elems = filter(lambda v: 'interest' in v.local_name.lower(), imd_values)
    if len(interest_elems) == len(imd_values):
        # HACK: An exceptional case for BBT
        # Revenues = InterestIncome + NoninterestIncome
        return imd_sum(imd_values)

    return imd_max(imd_values)


def imd_get_net_income(imd_values):
    values = filter(lambda v: '.' not in v.text, imd_values)
    return imd_min(values)


def imd_get_per_share_value(imd_values):
    if not imd_values:
        return None

    v = imd_values[0]
    value = v.value
    if value > MAX_PER_SHARE_VALUE:
        try:
            decimals = int(v.node.select('@decimals')[0].extract())
        except (AttributeError, IndexError):
            return None
        else:
            # HACK: some of LTD's reports have unreasonable large per share value, such as
            # 320000 EPS (and it should be 0.32), so use decimals attribute to scale it down,
            # note that this is NOT a correct way to interpret decimals attribute
            value *= pow(10, decimals - 2)
    return value if value <= MAX_PER_SHARE_VALUE else None


def imd_filter_member(imd_values):
    non_members = filter(lambda v: not v.is_member(), imd_values)
    if non_members:
        return non_members
    return imd_values


{'fraction_non_alphanumeric': 0.1028412062053338, 'fraction_numerical': 0.008279588635175179, 'mean_word_length': 2.3050115207373274, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1425421', 'n_tokens_mistral': 3654, 'n_tokens_neox': 3459, 'n_words': 822}