<commit_before>Mailbox = require './mailbox'
Message = require './message'
Promise = require 'bluebird'
ImapScheduler = require '../processes/imap_scheduler'
ImapReporter  = require '../processes/imap_reporter'
log = require('../utils/logging')('models:mailbox_imap')
_ = require 'lodash'
mailutils = require '../utils/jwz_tools'
{Break} = require '../utils/errors'

Mailbox::doASAPWithBox = (gen) ->
    ImapScheduler.instanceFor(@accountID)
    .then (scheduler) => scheduler.doASAPWithBox this, gen

Mailbox::doLaterWithBox = (gen) ->
    ImapScheduler.instanceFor(@accountID)
    .then (scheduler) => scheduler.doLaterWithBox this, gen


Mailbox::imap_fetchMails = (limitByBox) ->
    @imap_refreshStep limitByBox


FETCH_AT_ONCE = 1000
Mailbox::imap_refreshStep = (limitByBox, laststep) ->

    step = null
    box = @
    reporter = null

    @doLaterWithBox (imap, imapbox) ->
        laststep ?= min: imapbox.uidnext + 1

        throw new Break() if laststep.min is 1

        step =
            max: Math.max 1, laststep.min - 1
            min: Math.max 1, laststep.min - FETCH_AT_ONCE

        if limitByBox
            step.min = Math.max 1, laststep.min - limitByBox

        log.info "IMAP REFRESH", box.label, "UID #{step.min}:#{step.max}"

        Promise.all [
            imap.search [['UID', "#{step.min}:#{step.max}"]]
            .then (UIDs) ->
                UIDs.sort().reverse()
                imap.fetchMetadata(UIDs)

            Message.UIDsInRange box.id, step.min, step.max
        ]

    .spread (imapUIDs, cozyIds) ->
        ops =
            toFetch: []
            toRemove: []
            flagsChange: []

        for uid, imapMessage of imapUIDs
            if cozyMessage = cozyIds[uid] # this message is already in cozy
                if _.xor(imapMessage[1], cozyMessage[1]).length
                    id = cozyMessage[0]
                    ops.flagsChange.push id: id, flags: imapMessage[1]

            else # this message isnt in this box in cozy
                # add it to be fetched
                ops.toFetch.push {uid: parseInt(uid), mid: imapMessage[0]}

        for uid, cozyMessage of cozyIds
            unless imapUIDs[uid]
                ops.toRemove.push id = cozyMessage[0]

        return ops

    .tap (ops) ->

        nbTasks = ops.toFetch.length +
                ops.toRemove.length +
                ops.flagsChange.length

        if nbTasks > 0
            reporter = ImapReporter.boxFetch @, nbTasks

    .tap (ops) ->
        Promise.serie ops.toRemove, (id) ->
            Message.removeFromMailbox id, box
            .catch (err) -> reporter.onError err
            .tap -> reporter.addProgress 1
            .delay 100 # let the DS breath

    .tap (ops) ->
        Promise.serie ops.flagsChange, (change) ->
            Message.applyFlagsChanges change.id, change.flags
            .catch (err) -> reporter.onError err
            .tap -> reporter.addProgress 1
            .delay 100 # let the DS breath

    .tap (ops) ->
        Promise.serie ops.toFetch, (msg) ->
            Message.byMessageId box.accountID, msg.mid
            .then (existing) ->
                if existing then existing.addToMailbox box, msg.uid
                else box.imap_fetchOneMail msg.uid
            .catch (err) -> reporter.onError err
            .tap -> reporter.addProgress 1
            .delay 100 # let the DS breath<commit_msg>[quickfix] pass box to client's toast<commit_after>Mailbox = require './mailbox'
Message = require './message'
Promise = require 'bluebird'
ImapScheduler = require '../processes/imap_scheduler'
ImapReporter  = require '../processes/imap_reporter'
log = require('../utils/logging')('models:mailbox_imap')
_ = require 'lodash'
mailutils = require '../utils/jwz_tools'
{Break} = require '../utils/errors'

Mailbox::doASAPWithBox = (gen) ->
    ImapScheduler.instanceFor(@accountID)
    .then (scheduler) => scheduler.doASAPWithBox this, gen

Mailbox::doLaterWithBox = (gen) ->
    ImapScheduler.instanceFor(@accountID)
    .then (scheduler) => scheduler.doLaterWithBox this, gen


Mailbox::imap_fetchMails = (limitByBox) ->
    @imap_refreshStep limitByBox


FETCH_AT_ONCE = 1000
Mailbox::imap_refreshStep = (limitByBox, laststep) ->

    step = null
    box = @
    reporter = null

    @doLaterWithBox (imap, imapbox) ->
        laststep ?= min: imapbox.uidnext + 1

        throw new Break() if laststep.min is 1

        step =
            max: Math.max 1, laststep.min - 1
            min: Math.max 1, laststep.min - FETCH_AT_ONCE

        if limitByBox
            step.min = Math.max 1, laststep.min - limitByBox

        log.info "IMAP REFRESH", box.label, "UID #{step.min}:#{step.max}"

        Promise.all [
            imap.search [['UID', "#{step.min}:#{step.max}"]]
            .then (UIDs) ->
                UIDs.sort().reverse()
                imap.fetchMetadata(UIDs)

            Message.UIDsInRange box.id, step.min, step.max
        ]

    .spread (imapUIDs, cozyIds) ->
        ops =
            toFetch: []
            toRemove: []
            flagsChange: []

        for uid, imapMessage of imapUIDs
            if cozyMessage = cozyIds[uid] # this message is already in cozy
                if _.xor(imapMessage[1], cozyMessage[1]).length
                    id = cozyMessage[0]
                    ops.flagsChange.push id: id, flags: imapMessage[1]

            else # this message isnt in this box in cozy
                # add it to be fetched
                ops.toFetch.push {uid: parseInt(uid), mid: imapMessage[0]}

        for uid, cozyMessage of cozyIds
            unless imapUIDs[uid]
                ops.toRemove.push id = cozyMessage[0]

        return ops

    .tap (ops) =>

        nbTasks = ops.toFetch.length +
                ops.toRemove.length +
                ops.flagsChange.length

        if nbTasks > 0
            reporter = ImapReporter.boxFetch @, nbTasks

    .tap (ops) ->
        Promise.serie ops.toRemove, (id) ->
            Message.removeFromMailbox id, box
            .catch (err) -> reporter.onError err
            .tap -> reporter.addProgress 1
            .delay 100 # let the DS breath

    .tap (ops) ->
        Promise.serie ops.flagsChange, (change) ->
            Message.applyFlagsChanges change.id, change.flags
            .catch (err) -> reporter.onError err
            .tap -> reporter.addProgress 1
            .delay 100 # let the DS breath

    .tap (ops) ->
        Promise.serie ops.toFetch, (msg) ->
            Message.byMessageId box.accountID, msg.mid
            .then (existing) ->
                if existing then existing.addToMailbox box, msg.uid
                else box.imap_fetchOneMail msg.uid
            .catch (err) -> reporter.onError err
            .tap -> reporter.addProgress 1
            .delay 100 # let the DS breath
{'fraction_non_alphanumeric': 0.11243037232483143, 'fraction_numerical': 0.008501905599530929, 'mean_word_length': 1.8947815019092067, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1344580', 'n_tokens_mistral': 2189, 'n_tokens_neox': 2036, 'n_words': 510}