<commit_before>const _ = require('underscore');
const {IMAPConnection} = require('isomorphic-core');
const {Capabilities} = IMAPConnection;
const SyncTask = require('./sync-task')
const MessageProcessor = require('../../message-processor')
const SyncActivity = require('../../shared/sync-activity').default

const MessageFlagAttributes = ['id', 'threadId', 'folderImapUID', 'unread', 'starred', 'folderImapXGMLabels']
const FETCH_ATTRIBUTES_BATCH_SIZE = 1000;
const MIN_MESSAGE_BATCH_SIZE = 30;
const MAX_MESSAGE_BATCH_SIZE = 300;
const BATCH_SIZE_PER_SELECT_SEC = 60;
const GMAIL_INITIAL_PRIORITIZE_COUNT = 1000;


class FetchMessagesInFolderIMAP extends SyncTask {
  constructor({account, folder, uids} = {}) {
    super({account})
    this._imap = null
    this._box = null
    this._db = null
    this._folder = folder;
    this._uids = uids;
    if (!this._folder) {
      throw new Error("FetchMessagesInFolderIMAP requires a category")
    }
  }

  description() {
    return `FetchMessagesInFolderIMAP (${this._folder.name} - ${this._folder.id})`;
  }

  _isFirstSync() {
    return this._folder.syncState.minUID == null;
  }

  async _recoverFromUIDInvalidity(boxUidvalidity) {
    // UID invalidity means the server has asked us to delete all the UIDs for
    // this folder and start from scratch. Instead of deleting all the messages,
    // we just remove the folder ID and UID. We may re-assign the same message
    // the same UID. Otherwise they're eventually garbage collected.
    const {Message} = this._db;
    await Message.update({
      folderId: null,
      folderImapUID: null,
    }, {where: {folderId: this._folder.id}})

    await this._folder.updateSyncState({
      fetchedmax: null,
      fetchedmin: null,
      minUID: null,
      uidvalidity: boxUidvalidity,
    });
  }

  async _updateMessageAttributes(remoteUIDAttributes, localMessageAttributes) {
    const {Label, Thread} = this._db;

    const messageAttributesMap = {};
    for (const msg of localMessageAttributes) {
      messageAttributesMap[msg.folderImapUID] = msg;
    }

    const messagesWithChangedFlags = [];
    const messagesWithChangedLabels = [];

    // Step 1: Identify changed messages and update their attributes in place

    const preloadedLabels = await Label.findAll();
    for (const uid of Object.keys(remoteUIDAttributes)) {
      const msg = messageAttributesMap[uid];
      const attrs = remoteUIDAttributes[uid];

      if (!msg) continue;

      const unread = !attrs.flags.includes('\\Seen');
      const starred = attrs.flags.includes('\\Flagged');
      const xGmLabels = attrs['x-gm-labels'];<commit_msg>Match \Seen and \Flagged system flags in case-insensitive manner<commit_after>const _ = require('underscore');
const {IMAPConnection} = require('isomorphic-core');
const {Capabilities} = IMAPConnection;
const SyncTask = require('./sync-task')
const MessageProcessor = require('../../message-processor')
const SyncActivity = require('../../shared/sync-activity').default

const MessageFlagAttributes = ['id', 'threadId', 'folderImapUID', 'unread', 'starred', 'folderImapXGMLabels']
const FETCH_ATTRIBUTES_BATCH_SIZE = 1000;
const MIN_MESSAGE_BATCH_SIZE = 30;
const MAX_MESSAGE_BATCH_SIZE = 300;
const BATCH_SIZE_PER_SELECT_SEC = 60;
const GMAIL_INITIAL_PRIORITIZE_COUNT = 1000;


class FetchMessagesInFolderIMAP extends SyncTask {
  constructor({account, folder, uids} = {}) {
    super({account})
    this._imap = null
    this._box = null
    this._db = null
    this._folder = folder;
    this._uids = uids;
    if (!this._folder) {
      throw new Error("FetchMessagesInFolderIMAP requires a category")
    }
  }

  description() {
    return `FetchMessagesInFolderIMAP (${this._folder.name} - ${this._folder.id})`;
  }

  _isFirstSync() {
    return this._folder.syncState.minUID == null;
  }

  async _recoverFromUIDInvalidity(boxUidvalidity) {
    // UID invalidity means the server has asked us to delete all the UIDs for
    // this folder and start from scratch. Instead of deleting all the messages,
    // we just remove the folder ID and UID. We may re-assign the same message
    // the same UID. Otherwise they're eventually garbage collected.
    const {Message} = this._db;
    await Message.update({
      folderId: null,
      folderImapUID: null,
    }, {where: {folderId: this._folder.id}})

    await this._folder.updateSyncState({
      fetchedmax: null,
      fetchedmin: null,
      minUID: null,
      uidvalidity: boxUidvalidity,
    });
  }

  async _updateMessageAttributes(remoteUIDAttributes, localMessageAttributes) {
    const {Label, Thread} = this._db;

    const messageAttributesMap = {};
    for (const msg of localMessageAttributes) {
      messageAttributesMap[msg.folderImapUID] = msg;
    }

    const messagesWithChangedFlags = [];
    const messagesWithChangedLabels = [];

    // Step 1: Identify changed messages and update their attributes in place

    const preloadedLabels = await Label.findAll();
    for (const uid of Object.keys(remoteUIDAttributes)) {
      const msg = messageAttributesMap[uid];
      const attrs = remoteUIDAttributes[uid];

      if (!msg) continue;

      const unread = !attrs.flags.some((attr) => {return attr.toUpperCase() === '\\SEEN'});
      const starred = attrs.flags.some((attr) => {return attr.toUpperCase() === '\\FLAGGED'});
      const xGmLabels = attrs['x-gm-labels'];
{'fraction_non_alphanumeric': 0.11532033426183844, 'fraction_numerical': 0.005942432683379758, 'mean_word_length': 3.708041958041958, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7664583', 'n_tokens_mistral': 1645, 'n_tokens_neox': 1592, 'n_words': 423}