<commit_before>""" Helper functions for actions that operate on accounts.

These could be methods of ImapAccount, but separating them gives us more
flexibility with calling code, as most don't need any attributes of the
account object other than the ID, to limit the action.
"""
import os
import json

from hashlib import sha256

from sqlalchemy import distinct, func
from sqlalchemy.orm.exc import NoResultFound

from flanker import mime

from inbox.util.misc import or_none
from inbox.util.addr import parse_email_address
from inbox.util.file import mkdirp
from inbox.util.misc import parse_ml_headers

from .tables import Block, Message, ImapUid, UIDValidity, FolderItem, Thread

from ..config import config
from ..log import get_logger
log = get_logger()

def total_stored_data(account_id, session):
    """ Computes the total size of the block data of emails in your
        account's IMAP folders
    """
    subq = session.query(Block) \
            .join(Block.message, Message.imapuids) \
            .filter(ImapUid.imapaccount_id==account_id) \
            .group_by(Message.id, Block.id)
    return session.query(func.sum(subq.subquery().columns.size)).scalar()

def total_stored_messages(account_id, session):
    """ Computes the number of emails in your account's IMAP folders """
    return session.query(Message) \
            .join(Message.imapuids) \
            .filter(ImapUid.imapaccount_id==account_id) \
            .group_by(Message.id).count()

def all_uids(account_id, session, folder_name):
    return [uid for uid, in
            session.query(ImapUid.msg_uid).filter_by(
                imapaccount_id=account_id, folder_name=folder_name)]

def g_msgids(account_id, session, in_=None):
    query = session.query(distinct(Message.g_msgid)).join(ImapUid) \
                .filter(ImapUid.imapaccount_id==account_id)
    if in_:
        query = query.filter(Message.g_msgid.in_(in_))
    return sorted([g_msgid for g_msgid, in query], key=long)
<commit_msg>Add a write lock for ImapAccount.<commit_after>""" Helper functions for actions that operate on accounts.

These could be methods of ImapAccount, but separating them gives us more
flexibility with calling code, as most don't need any attributes of the
account object other than the ID, to limit the action.
"""
import os
import json

from hashlib import sha256

from sqlalchemy import distinct, func
from sqlalchemy.orm.exc import NoResultFound

from flanker import mime

from inbox.util.misc import or_none
from inbox.util.addr import parse_email_address
from inbox.util.file import Lock, mkdirp
from inbox.util.misc import parse_ml_headers

from .tables import Block, Message, ImapUid, UIDValidity, FolderItem, Thread

from ..config import config
from ..log import get_logger
log = get_logger()

def _db_write_lockfile_name(account_id):
    return "/var/lock/inbox_datastore/{0}.lock".format(account_id)

def db_write_lock(account_id):
    """ Protect updating this account's Inbox datastore data.

    Note that you should also use this to wrap any code that _figures
    out_ what to update the datastore with, because outside the lock
    you can't guarantee no one is updating the data behind your back.
    """
    return Lock(_db_write_lockfile_name(account_id), block=True)

def total_stored_data(account_id, session):
    """ Computes the total size of the block data of emails in your
        account's IMAP folders
    """
    subq = session.query(Block) \
            .join(Block.message, Message.imapuids) \
            .filter(ImapUid.imapaccount_id==account_id) \
            .group_by(Message.id, Block.id)
    return session.query(func.sum(subq.subquery().columns.size)).scalar()

def total_stored_messages(account_id, session):
    """ Computes the number of emails in your account's IMAP folders """
    return session.query(Message) \
            .join(Message.imapuids) \
            .filter(ImapUid.imapaccount_id==account_id) \
            .group_by(Message.id).count()

def all_uids(account_id, session, folder_name):
    return [uid for uid, in
            session.query(ImapUid.msg_uid).filter_by(
                imapaccount_id=account_id, folder_name=folder_name)]

def g_msgids(account_id, session, in_=None):
    query = session.query(distinct(Message.g_msgid)).join(ImapUid) \
                .filter(ImapUid.imapaccount_id==account_id)
    if in_:
        query = query.filter(Message.g_msgid.in_(in_))
    return sorted([g_msgid for g_msgid, in query], key=long)

{'fraction_non_alphanumeric': 0.08857652833556448, 'fraction_numerical': 0.0015618027666220438, 'mean_word_length': 3.6552440290758046, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6945491', 'n_tokens_mistral': 1413, 'n_tokens_neox': 1359, 'n_words': 406}