<commit_before>import os
import threading
from datetime import timedelta
from flask import make_response, current_app, request, render_template, jsonify
from apiclient import discovery, errors
from functools import update_wrapper
from app import app
from copy import deepcopy
from app import cache
from pybars import Compiler
import datetime
import base64
import re

compiler = Compiler()
fullmessageset = []
parsedmessageset = []


def getcachedthreads():
    newcollection = None
    cachedmessagesetids = cache.get('cachedmessagesetids')
    if cachedmessagesetids:
        for emailthreadid in cachedmessagesetids:
            cachedthread = cache.get(emailthreadid['id'])
            if cachedthread:
                parsedmessageset.append(cachedthread)
        newcollection = deepcopy(parsedmessageset)
        parsedmessageset[:] = []
    return newcollection


def rendercollection(newcollection):
    basedir = os.path.abspath(os.path.dirname(__file__))
    templatedir = os.path.join(basedir, 'static/piemail/www/libs/handlebars/templates/email-list.handlebars')
    source = open(templatedir, "r").read().decode('utf-8')
    template = compiler.compile(source)
    output = template(newcollection)
    return output


def getcontext(http_auth=None, retrievebody=None):
    service = discovery.build('gmail', 'v1', http=http_auth)
    results = service.users().threads().list(userId='me', maxResults=50, fields="threads/id", q="in:inbox").execute()
    batch = service.new_batch_http_request(callback=processthreads)
    # cache.set('cachedmessagesetids', results['threads'], timeout=300)  # Cache for 5 minutes
    for thread in results['threads']:
        batch.add(service.users().threads().get(userId='me', id=thread['id'], fields="messages/snippet, "
                                                                                     "messages/internalDate, "
                                                                                     "messages/labelIds, "
                                                                                     "messages/threadId, "
                                                                                     "messages/payload/parts, "
                                                                                     "messages/payload/body, "
                                                                                     "messages/payload/headers"))
    batch.execute()
    for item in fullmessageset:
        t = threading.Thread(target=parse_item, kwargs={"item": item, "retrievebody": retrievebody})
        t.start()
        # parse_item(item, retrievebody)
    newcollection = deepcopy(parsedmessageset)
    fullmessageset[:] = []
    parsedmessageset[:] = []
    return newcollection


def getmessages(http_auth, threadid):
    service = discovery.build('gmail', 'v1', http=http_auth)
    threadid = threadid
    try:
        thread = service.users().threads().get(userId='me', id=threadid).execute()
    except errors.HttpError, error:
        print 'An error occurred: %s' % error
        return jsonify(error)

    batch = service.new_batch_http_request(callback=processmessages)
    for message in thread['messages']:
        batch.add(service.users().messages().get(userId='me', id=message['id']))
    batch.execute()
    for item in fullmessageset:
        # m = threading.Thread(target=parse_item, kwargs={"item": item, "retrievebody": True})
        # m.start()
        parse_item(item, retrievebody=True)
    response = dict()
    response['iserror'] = False
    response['savedsuccess'] = True
    response['currentmessagelist'] = deepcopy(parsedmessageset)
    fullmessageset[:] = []
    parsedmessageset[:] = []
    return response


def processthreads(request_id, response, exception):
    if exception is not None:
        pass
    else:<commit_msg>re-added threading<commit_after>import os
import threading
from datetime import timedelta
from flask import make_response, current_app, request, render_template, jsonify
from apiclient import discovery, errors
from functools import update_wrapper
from app import app
from copy import deepcopy
from app import cache
from pybars import Compiler
import datetime
import base64
import re

compiler = Compiler()
fullmessageset = []
parsedmessageset = []


def getcachedthreads():
    newcollection = None
    cachedmessagesetids = cache.get('cachedmessagesetids')
    if cachedmessagesetids:
        for emailthreadid in cachedmessagesetids:
            cachedthread = cache.get(emailthreadid['id'])
            if cachedthread:
                parsedmessageset.append(cachedthread)
        newcollection = deepcopy(parsedmessageset)
        parsedmessageset[:] = []
    return newcollection


def rendercollection(newcollection):
    basedir = os.path.abspath(os.path.dirname(__file__))
    templatedir = os.path.join(basedir, 'static/piemail/www/libs/handlebars/templates/email-list.handlebars')
    source = open(templatedir, "r").read().decode('utf-8')
    template = compiler.compile(source)
    output = template(newcollection)
    return output


def getcontext(http_auth=None, retrievebody=None):
    service = discovery.build('gmail', 'v1', http=http_auth)
    results = service.users().threads().list(userId='me', maxResults=100, fields="threads/id", q="in:inbox").execute()
    batch = service.new_batch_http_request(callback=processthreads)
    # cache.set('cachedmessagesetids', results['threads'], timeout=300)  # Cache for 5 minutes
    for thread in results['threads']:
        batch.add(service.users().threads().get(userId='me', id=thread['id'], fields="messages/snippet, "
                                                                                     "messages/internalDate, "
                                                                                     "messages/labelIds, "
                                                                                     "messages/threadId, "
                                                                                     "messages/payload/parts, "
                                                                                     "messages/payload/body, "
                                                                                     "messages/payload/headers"))
    batch.execute()
    for item in fullmessageset:
        t = threading.Thread(target=parse_item, kwargs={"item": item, "retrievebody": retrievebody})
        t.start()
        # parse_item(item, retrievebody)
    newcollection = deepcopy(parsedmessageset)
    fullmessageset[:] = []
    parsedmessageset[:] = []
    return newcollection


def getmessages(http_auth, threadid):
    service = discovery.build('gmail', 'v1', http=http_auth)
    threadid = threadid
    try:
        thread = service.users().threads().get(userId='me', id=threadid).execute()
    except errors.HttpError, error:
        print 'An error occurred: %s' % error
        return jsonify(error)

    batch = service.new_batch_http_request(callback=processmessages)
    for message in thread['messages']:
        batch.add(service.users().messages().get(userId='me', id=message['id']))
    batch.execute()
    for item in fullmessageset:
        m = threading.Thread(target=parse_item, kwargs={"item": item, "retrievebody": True})
        m.start()

    response = dict()
    response['iserror'] = False
    response['savedsuccess'] = True
    response['currentmessagelist'] = deepcopy(parsedmessageset)
    fullmessageset[:] = []
    parsedmessageset[:] = []
    return response


def processthreads(request_id, response, exception):
    if exception is not None:
        pass
    else:
{'fraction_non_alphanumeric': 0.10591615515214836, 'fraction_numerical': 0.003003787384092987, 'mean_word_length': 2.3237847222222223, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3040848', 'n_tokens_mistral': 1932, 'n_tokens_neox': 1923, 'n_words': 422}