<commit_before>from datetime import datetime
from inspect import getmembers
import json
import re
from urlparse import urlparse, parse_qsl

from twisted.internet.defer import maybeDeferred

from txtwitter.error import TwitterAPIError
from txtwitter.tests.fake_agent import FakeResponse
from txtwitter.twitter import (
    TWITTER_API_URL, TWITTER_STREAM_URL, TWITTER_USERSTREAM_URL,
    TwitterClient)


USER_MENTION_RE = re.compile(r'@[a-zA-Z0-9_]+')

def now():
    return datetime.utcnow()


def mention_from_match(twitter_data, match):
    user = twitter_data.get_user_by_screen_name(match.group(0)[1:])

    if user is None:
        return None

    return {
        'id_str': user.id_str,
        'id': int(user.id_str),
        'indices': list(match.span(0)),
        'screen_name': user.screen_name,
        'name': user.name,
    }


def extract_user_mentions(twitter_data, text):
    mentions = []
    for match in USER_MENTION_RE.finditer(text):
        mention = mention_from_match(twitter_data, match)
        if mention is not None:
            mentions.append(mention)
    return mentions


def extract_entities(twitter_data, text):
    return {
        'user_mentions': extract_user_mentions(twitter_data, text),
        # TODO: More entities
    }


class FakeStream(object):
    def __init__(self):
        self.resp = FakeResponse(None)
        self._message_types = {}

    def add_message_type(self, message_type, predicate=None):
        if predicate is None:
            predicate = lambda _: True
        self._message_types[message_type] = predicate

    def accepts(self, message_type, data):
        predicate = self._message_types.get(message_type)
        return predicate is not None and predicate(data)

    def deliver(self, data):
        self.resp.deliver_data(json.dumps(data))
        self.resp.deliver_data('\r\n')


class FakeTweet(object):
    def __init__(self, id_str, text, user_id_str, reply_to=None, **kw):
        self.id_str = id_str
        self.text = text
        self.user_id_str = user_id_str
        self.reply_to = reply_to
        self.created_at = kw.pop('created_at', now())
        self.kw = kw

    def __cmp__(self, other):
        return cmp(int(self.id_str), int(other.id_str))
<commit_msg>Change predicates to be compulsory for FakeStream<commit_after>from datetime import datetime
from inspect import getmembers
import json
import re
from urlparse import urlparse, parse_qsl

from twisted.internet.defer import maybeDeferred

from txtwitter.error import TwitterAPIError
from txtwitter.tests.fake_agent import FakeResponse
from txtwitter.twitter import (
    TWITTER_API_URL, TWITTER_STREAM_URL, TWITTER_USERSTREAM_URL,
    TwitterClient)


USER_MENTION_RE = re.compile(r'@[a-zA-Z0-9_]+')

def now():
    return datetime.utcnow()


def mention_from_match(twitter_data, match):
    user = twitter_data.get_user_by_screen_name(match.group(0)[1:])

    if user is None:
        return None

    return {
        'id_str': user.id_str,
        'id': int(user.id_str),
        'indices': list(match.span(0)),
        'screen_name': user.screen_name,
        'name': user.name,
    }


def extract_user_mentions(twitter_data, text):
    mentions = []
    for match in USER_MENTION_RE.finditer(text):
        mention = mention_from_match(twitter_data, match)
        if mention is not None:
            mentions.append(mention)
    return mentions


def extract_entities(twitter_data, text):
    return {
        'user_mentions': extract_user_mentions(twitter_data, text),
        # TODO: More entities
    }


class FakeStream(object):
    def __init__(self):
        self.resp = FakeResponse(None)
        self._message_types = {}

    def add_message_type(self, message_type, predicate):
        self._message_types[message_type] = predicate

    def accepts(self, message_type, data):
        predicate = self._message_types.get(message_type)
        return predicate is not None and predicate(data)

    def deliver(self, data):
        self.resp.deliver_data(json.dumps(data))
        self.resp.deliver_data('\r\n')


class FakeTweet(object):
    def __init__(self, id_str, text, user_id_str, reply_to=None, **kw):
        self.id_str = id_str
        self.text = text
        self.user_id_str = user_id_str
        self.reply_to = reply_to
        self.created_at = kw.pop('created_at', now())
        self.kw = kw

    def __cmp__(self, other):
        return cmp(int(self.id_str), int(other.id_str))

{'fraction_non_alphanumeric': 0.09811827956989247, 'fraction_numerical': 0.002240143369175627, 'mean_word_length': 3.2122641509433962, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2113463', 'n_tokens_mistral': 1475, 'n_tokens_neox': 1433, 'n_words': 317}