<commit_before>from twitter import Twitter
from twitter import OAuth

from django.conf import settings
from generic import GenericAggregator


class TwitterAggregator(GenericAggregator):

    CONSUMER_KEY = settings.EDSA_TWITTER_CONSUMER_KEY
    CONSUMER_SECRET = settings.EDSA_TWITTER_CONSUMER_SECRET
    TOKEN = settings.EDSA_TWITTER_TOKEN
    SECRET = settings.EDSA_TWITTER_SECRET

    def init_connector(self):
        auth = OAuth(self.TOKEN, self.SECRET, self.CONSUMER_KEY,
                     self.CONSUMER_SECRET)
        self.connector = Twitter(auth=auth)

    def search(self, query):
        return self.connector.search.tweets(q=query)<commit_msg>structure the search data<commit_after>from twitter import Twitter
from twitter import OAuth

from django.conf import settings
from generic import GenericAggregator


class Aggregator(GenericAggregator):

    CONSUMER_KEY = settings.EDSA_TWITTER_CONSUMER_KEY
    CONSUMER_SECRET = settings.EDSA_TWITTER_CONSUMER_SECRET
    TOKEN = settings.EDSA_TWITTER_TOKEN
    SECRET = settings.EDSA_TWITTER_SECRET

    def init_connector(self):
        auth = OAuth(self.TOKEN, self.SECRET, self.CONSUMER_KEY,
                     self.CONSUMER_SECRET)
        self.connector = Twitter(auth=auth)

    def search(self, query):
        res = self.connector.search.tweets(q=query)
        datas = []
        for tweet in res['statuses']:
            data = {'social_id': tweet['id_str'],
                    'lang': tweet['lang'],
                    'creation_date': tweet['created_at'],
                    'description': tweet['text'],
                    'author': tweet['user']['name'],
                    }
            datas.append(data)

        return datas
{'fraction_non_alphanumeric': 0.084696261682243, 'fraction_numerical': 0.0, 'mean_word_length': 2.858108108108108, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1562956', 'n_tokens_mistral': 563, 'n_tokens_neox': 519, 'n_words': 101}