<commit_before>
function isContactMeaningful(contact) {
  // some suggestions: http://stackoverflow.com/questions/6317714/apache-camel-mail-to-identify-auto-generated-messages
  const regex = new RegExp(/^(noreply|no-reply|donotreply|mailer|support|webmaster|news(letter)?@)/ig)

  if (!contact.email) {
    return false;
  }
  if (regex.test(contact.email) || contact.email.length > 60) {
    return false
  }
  return true
}

async function extractContacts({db, message}) {
  const {Contact} = db
  let allContacts = [];
  ['to', 'from', 'bcc', 'cc'].forEach((field) => {
    allContacts = allContacts.concat(message[field])
  })

  const meaningfulContacts = allContacts.filter(c => isContactMeaningful(c));
  const contactsDataById = new Map()
  meaningfulContacts.forEach(c => {
    const id = Contact.hash(c)
    const cdata = {
      id,
      name: c.name,
      email: c.email,
      accountId: message.accountId,
    }
    contactsDataById.set(id, cdata)
  })
  const existingContacts = await Contact.findAll({
    where: {
      id: Array.from(contactsDataById.keys()),
    },
  })

  for (const c of contactsDataById.values()) {
    const existing = existingContacts.find(({id}) => id === c.id)
    if (!existing) {
      await Contact.create(c)
    } else {
      const updateRequired = (c.name !== existing.name);
      if (updateRequired) {
        await existing.update(c)
      }
    }
  }

  return message;<commit_msg>[local-sync] Fix UniqueConstraintError inserting contacts from messages<commit_after>const Sequelize = require('sequelize');

function isContactMeaningful(contact) {
  // some suggestions: http://stackoverflow.com/questions/6317714/apache-camel-mail-to-identify-auto-generated-messages
  const regex = new RegExp(/^(noreply|no-reply|donotreply|mailer|support|webmaster|news(letter)?@)/ig)

  if (!contact.email) {
    return false;
  }
  if (regex.test(contact.email) || contact.email.length > 60) {
    return false
  }
  return true
}

async function extractContacts({db, message}) {
  const {Contact} = db
  let allContacts = [];
  ['to', 'from', 'bcc', 'cc'].forEach((field) => {
    allContacts = allContacts.concat(message[field])
  })

  const meaningfulContacts = allContacts.filter(c => isContactMeaningful(c));
  const contactsDataById = new Map()
  meaningfulContacts.forEach(c => {
    const id = Contact.hash(c)
    const cdata = {
      id,
      name: c.name,
      email: c.email,
      accountId: message.accountId,
    }
    contactsDataById.set(id, cdata)
  })

  const existingContacts = await Contact.findAll({
    where: {
      id: Array.from(contactsDataById.keys()),
    },
  })

  for (const c of contactsDataById.values()) {
    const existing = existingContacts.find(({id}) => id === c.id);

    if (!existing) {
      Contact.create(c).catch(Sequelize.ValidationError, (err) => {
        if (err.name !== "SequelizeUniqueConstraintError") {
          console.log('Unknown error inserting contact', err);
          throw err;
        } else {
          // Another message with the same contact was processing concurrently,
          // and beat us to inserting. Since contacts are never deleted within
          // an account, we can safely assume that we can perform an update
          // instead.
          Contact.find({where: {id: c.id}}).then(
            (row) => { row.update(c) });
        }
      });
    } else {
      existing.update(c);
    }
  }
{'fraction_non_alphanumeric': 0.1413551401869159, 'fraction_numerical': 0.005257009345794392, 'mean_word_length': 3.058056872037915, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6743533', 'n_tokens_mistral': 1096, 'n_tokens_neox': 1066, 'n_words': 231}