<commit_before>import os

from gunicorn.app.base import BaseApplication
from gunicorn.six import iteritems

import multiprocessing
import time
from TwitterAPI import TwitterAPI, TwitterRestPager
from elasticsearch import Elasticsearch

from yaml import load, dump
import json


CONSUMER_KEY = ''
CONSUMER_SECRET = ''
ACCESS_TOKEN_KEY = ''
ACCESS_TOKEN_SECRET = ''

SEARCH_TERM = 'docker'

es = Elasticsearch(os.environ["ES_URL"])

def worker():
    api = TwitterAPI(CONSUMER_KEY,
                     CONSUMER_SECRET,
                     ACCESS_TOKEN_KEY,
                     ACCESS_TOKEN_SECRET)

    pager = TwitterRestPager.TwitterRestPager(api, 'search/tweets', {'q': SEARCH_TERM})
    for item in pager.get_iterator():
        if 'text' in item:
            tweet = {}
            # tweet['coordinates'] = item['coordinates']
            # tweet['@timestamp'] = time.mktime(time.strptime(item['created_at'],"%a %b %d %H:%M:%S +0000 %Y"))
            # tweet['place'] = item['place']
            ts = time.strftime('%Y-%m-%d %H:%M:%S', time.strptime(item['created_at'],'%a %b %d %H:%M:%S +0000 %Y'))
            tweet['@timestamp'] = ts
            tweet['username'] = item['user']['name']
            tweet['handle'] = item['user']['screen_name']
            tweet['lang'] = item['lang']
            tweet['timezone'] = item['user']['time_zone']
            tweet['followers'] = item['user']['followers_count']
            tweet['location'] = item['user']['location']
            tweet['retweeted'] = item['retweeted']
            tweet['text'] = item['text']
            es.index(index="tweets", doc_type="tweet", body=tweet)
    return
<commit_msg>Fix timestamp last try.<commit_after>import os

from gunicorn.app.base import BaseApplication
from gunicorn.six import iteritems

import multiprocessing
import time
from TwitterAPI import TwitterAPI, TwitterRestPager
from elasticsearch import Elasticsearch

from yaml import load, dump
import json


CONSUMER_KEY = ''
CONSUMER_SECRET = ''
ACCESS_TOKEN_KEY = ''
ACCESS_TOKEN_SECRET = ''

SEARCH_TERM = 'docker'

es = Elasticsearch(os.environ["ES_URL"])

def worker():
    api = TwitterAPI(CONSUMER_KEY,
                     CONSUMER_SECRET,
                     ACCESS_TOKEN_KEY,
                     ACCESS_TOKEN_SECRET)

    pager = TwitterRestPager.TwitterRestPager(api, 'search/tweets', {'q': SEARCH_TERM})
    for item in pager.get_iterator():
        if 'text' in item:
            tweet = {}
            # tweet['coordinates'] = item['coordinates']
            # tweet['@timestamp'] = time.mktime(time.strptime(item['created_at'],"%a %b %d %H:%M:%S +0000 %Y"))
            # tweet['place'] = item['place']
            # ts = time.strftime('%Y-%m-%d %H:%M:%S', time.strptime(item['created_at'],'%a %b %d %H:%M:%S +0000 %Y'))

            tweet['@timestamp'] = item['created_at']
            tweet['username'] = item['user']['name']
            tweet['handle'] = item['user']['screen_name']
            tweet['lang'] = item['lang']
            tweet['timezone'] = item['user']['time_zone']
            tweet['followers'] = item['user']['followers_count']
            tweet['location'] = item['user']['location']
            tweet['retweeted'] = item['retweeted']
            tweet['text'] = item['text']
            es.index(index="tweets", doc_type="tweet", body=tweet)
    return

{'fraction_non_alphanumeric': 0.15794196829195334, 'fraction_numerical': 0.004786120251271313, 'mean_word_length': 2.938751472320377, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2958141', 'n_tokens_mistral': 1101, 'n_tokens_neox': 1061, 'n_words': 205}