<commit_before>// Copy ZenProspect contacts with email replies into Close.io leads

'use strict';
if (process.argv.length !== 4) {
  console.log("Usage: node <script> <Close.io general API key> <ZenProspect auth token>");
  process.exit();
}

const closeIoApiKey = process.argv[2];
const zpAuthToken = process.argv[3];

const scriptStartTime = new Date();

const async = require('async');
const request = require('request');

const zpPageSize = 100;

getZPContacts((err, emailContactMap) => {
  if (err) {
    console.log(err);
    return;
  }
  const tasks = [];
  for (const email in emailContactMap) {
    const contact = emailContactMap[email];
    tasks.push(createUpsertCloseLeadFn(contact));
  }
  async.parallel(tasks, (err, results) => {
    if (err) console.log(err);
    log("Script runtime: " + (new Date() - scriptStartTime));
  });
});

function createCloseLead(zpContact, done) {
  const postData = {
    name: zpContact.organization,
    status: 'Contacted',
    contacts: [
      {
        name: zpContact.name,
        title: zpContact.title,
        emails: [{email: zpContact.email}]
      }
    ],
    custom: {
      lastUpdated: new Date(),
      'Lead Origin': 'outbound campaign'
    }
  };
  if (zpContact.phone) {
    postData.contacts[0].phones = [{phone: zpContact.phone}];
  }
  const options = {
    uri: `https://${closeIoApiKey}:X@app.close.io/api/v1/lead/`,
    body: JSON.stringify(postData)
  };
  request.post(options, (error, response, body) => {
    if (error) return done(error);
    const newLead = JSON.parse(body);
    if (newLead.errors || newLead['field-errors']) {
      console.error(`New lead POST error for ${zpContact.name} ${zpContact.organization}`);
      return done(newLead.errors || newLead['field-errors']);
    }
    return done();
  });
}

function updateCloseLead(zpContact, existingLead, done) {
  // console.log(`DEBUG: updateCloseLead ${existingLead.id} ${zpContact.email}`);
  const putData = {
    status: 'Contacted',
    'custom.lastUpdated': new Date(),
    'custom.Lead Origin': 'outbound campaign'
  };
  const options = {
    uri: `https://${closeIoApiKey}:<EMAIL>/api/v1/lead/${existingLead.id}/`,
    body: JSON.stringify(putData)
  };
  request.put(options, (error, response, body) => {
    if (error) return done(error);
    const result = JSON.parse(body);
    if (result.errors || result['field-errors']) {
      return done(`Update existing lead PUT error for ${existingLead.id} ${zpContact.email} ${result.errors || result['field-errors']}`);
    }
    const postData = {
      lead_id: existingLead.id,
      name: zpContact.name,
      title: zpContact.title,
      emails: [{email: zpContact.email}]
    };
    if (zpContact.phone) {
      postData.phones = [{phone: zpContact.phone}];
    }
    const options = {
      uri: `https://${closeIoApiKey}:<EMAIL>/api/v1/contact/`,
      body: JSON.stringify(postData)
    };
    request.post(options, (error, response, body) => {
      if (error) return done(error);
      const result = JSON.parse(body);
      if (result.errors || result['field-errors']) {
        return done(`New Contact POST error for ${existingLead.id} ${zpContact.email} ${result.errors || result['field-errors']}`);
      }
      return done();
    });
  });
}

function createUpsertCloseLeadFn(zpContact) {
  return (done) => {
    // console.log(`DEBUG: createUpsertCloseLeadFn ${zpContact.organization} ${zpContact.email}`);
    const query = `email:${zpContact.email}`;
    const url = `https://${closeIoApiKey}:<EMAIL>.io/api/v1/lead/?query=${encodeURIComponent(query)}`;
    request.get(url, (error, response, body) => {
      if (error) return done(error);
      const data = JSON.parse(body);
      if (data.total_results != 0) return done();
      const query = `name:${zpContact.organization}`;
      const url = `https://${closeIoApiKey}:X@app.close.io/api/v1/lead/?query=${encodeURIComponent(query)}`;
      request.get(url, (error, response, body) => {
        if (error) return done(error);
        const data = JSON.parse(body);
        if (data.total_results === 0) {
          console.log(`DEBUG: Creating lead for ${zpContact.organization} ${zpContact.email}`);
          return createCloseLead(zpContact, done);
        }
        else {
          const existingLead = data.data[0];
          console.log(`DEBUG: Adding ${zpContact.organization} ${zpContact.email} to ${existingLead.id}`);
          return updateCloseLead(zpContact, existingLead, done);
        }
      });
    });
  };
}

function getZPContactsPage(contacts, searchQuery, done) {
  const options = {
    url: `https://www.zenprospect.com/api/v1/contacts/search?${searchQuery}`,
    headers: {
      'Accept': 'application/json'
    }
  };
  request.get(options, (err, response, body) => {
    if (err) return done(err);
    const data = JSON.parse(body);
    for (let contact of data.contacts) {
      let organization = contact.organization_name;
      if (contact.custom_fields && contact.custom_fields.school_name) organization = contact.custom_fields.school_name;
      contacts.push({
        organization: organization,
        name: contact.name,
        title: contact.title,
        email: contact.email,
        phone: contact.phone,
        data: contact
      });
    }
    return done(null, data.pipeline_total);
  });
}

function createGetZPAutoResponderContactsPage(contacts, page) {
  return (done) => {
    // console.log(`DEBUG: Fetching autoresponder page ${page} ${zpPageSize}...`);
    let searchQuery = `codecombat_special_auth_token=${zpAuthToken}&page=${page}&per_page=${zpPageSize}&contact_email_autoresponder=true`;
    getZPContactsPage(contacts, searchQuery, done);
  };
}

function createGetZPRepliedContactsPage(contacts, page) {
  return (done) => {
    // console.log(`DEBUG: Fetching email reply page ${page} ${zpPageSize}...`);
    let searchQuery = `codecombat_special_auth_token=${zpAuthToken}&page=${page}&per_page=${zpPageSize}&contact_email_replied=true`;
    getZPContactsPage(contacts, searchQuery, done);<commit_msg>Add school and district ids to zp lead importing<commit_after>// Copy ZenProspect contacts with email replies into Close.io leads

'use strict';
if (process.argv.length !== 4) {
  console.log("Usage: node <script> <Close.io general API key> <ZenProspect auth token>");
  process.exit();
}

const closeIoApiKey = process.argv[2];
const zpAuthToken = process.argv[3];

const scriptStartTime = new Date();

const async = require('async');
const request = require('request');

const zpPageSize = 100;

getZPContacts((err, emailContactMap) => {
  if (err) {
    console.log(err);
    return;
  }
  const tasks = [];
  for (const email in emailContactMap) {
    const contact = emailContactMap[email];
    tasks.push(createUpsertCloseLeadFn(contact));
  }
  async.parallel(tasks, (err, results) => {
    if (err) console.log(err);
    log("Script runtime: " + (new Date() - scriptStartTime));
  });
});

function createCloseLead(zpContact, done) {
  const postData = {
    name: zpContact.organization,
    status: 'Contacted',
    contacts: [
      {
        name: zpContact.name,
        title: zpContact.title,
        emails: [{email: zpContact.email}]
      }
    ],
    custom: {
      lastUpdated: new Date(),
      'Lead Origin': 'outbound campaign'
    }
  };
  if (zpContact.phone) {
    postData.contacts[0].phones = [{phone: zpContact.phone}];
  }
  if (zpContact.district) {
    postData.custom['demo_nces_district'] = zpContact.district;
    postData.custom['demo_nces_name'] = zpContact.organization;
  }
  if (zpContact.nces_district_id) {
    postData.custom['demo_nces_district_id'] = zpContact.nces_district_id;
  }
  if (zpContact.nces_school_id) {
    postData.custom['demo_nces_id'] = zpContact.nces_school_id;
  }
  const options = {
    uri: `https://${closeIoApiKey}:X@app.close.io/api/v1/lead/`,
    body: JSON.stringify(postData)
  };
  request.post(options, (error, response, body) => {
    if (error) return done(error);
    const newLead = JSON.parse(body);
    if (newLead.errors || newLead['field-errors']) {
      console.error(`New lead POST error for ${zpContact.name} ${zpContact.organization}`);
      return done(newLead.errors || newLead['field-errors']);
    }
    return done();
  });
}

function updateCloseLead(zpContact, existingLead, done) {
  // console.log(`DEBUG: updateCloseLead ${existingLead.id} ${zpContact.email}`);
  const putData = {
    status: 'Contacted',
    'custom.lastUpdated': new Date(),
    'custom.Lead Origin': 'outbound campaign'
  };
  const options = {
    uri: `https://${closeIoApiKey}:X@app.close.io/api/v1/lead/${existingLead.id}/`,
    body: JSON.stringify(putData)
  };
  request.put(options, (error, response, body) => {
    if (error) return done(error);
    const result = JSON.parse(body);
    if (result.errors || result['field-errors']) {
      return done(`Update existing lead PUT error for ${existingLead.id} ${zpContact.email} ${result.errors || result['field-errors']}`);
    }
    const postData = {
      lead_id: existingLead.id,
      name: zpContact.name,
      title: zpContact.title,
      emails: [{email: zpContact.email}]
    };
    if (zpContact.phone) {
      postData.phones = [{phone: zpContact.phone}];
    }
    const options = {
      uri: `https://${closeIoApiKey}:<EMAIL>.close.io/api/v1/contact/`,
      body: JSON.stringify(postData)
    };
    request.post(options, (error, response, body) => {
      if (error) return done(error);
      const result = JSON.parse(body);
      if (result.errors || result['field-errors']) {
        return done(`New Contact POST error for ${existingLead.id} ${zpContact.email} ${result.errors || result['field-errors']}`);
      }
      return done();
    });
  });
}

function createUpsertCloseLeadFn(zpContact) {
  return (done) => {
    // console.log(`DEBUG: createUpsertCloseLeadFn ${zpContact.organization} ${zpContact.email}`);
    const query = `email:${zpContact.email}`;
    const url = `https://${closeIoApiKey}:X@<EMAIL>.close.io/api/v1/lead/?query=${encodeURIComponent(query)}`;
    request.get(url, (error, response, body) => {
      if (error) return done(error);
      const data = JSON.parse(body);
      if (data.total_results != 0) return done();
      const query = `name:${zpContact.organization}`;
      const url = `https://${closeIoApiKey}:<EMAIL>.io/api/v1/lead/?query=${encodeURIComponent(query)}`;
      request.get(url, (error, response, body) => {
        if (error) return done(error);
        const data = JSON.parse(body);
        if (data.total_results === 0) {
          console.log(`DEBUG: Creating lead for ${zpContact.organization} ${zpContact.email}`);
          return createCloseLead(zpContact, done);
        }
        else {
          const existingLead = data.data[0];
          console.log(`DEBUG: Adding ${zpContact.organization} ${zpContact.email} to ${existingLead.id}`);
          return updateCloseLead(zpContact, existingLead, done);
        }
      });
    });
  };
}

function getZPContactsPage(contacts, searchQuery, done) {
  const options = {
    url: `https://www.zenprospect.com/api/v1/contacts/search?${searchQuery}`,
    headers: {
      'Accept': 'application/json'
    }
  };
  request.get(options, (err, response, body) => {
    if (err) return done(err);
    const data = JSON.parse(body);
    for (let contact of data.contacts) {
      const newContact = {
        organization: contact.organization_name,
        name: contact.name,
        title: contact.title,
        email: contact.email,
        phone: contact.phone,
        data: contact
      };
      // Check custom fields, school_name set means organization_name is district name
      if (contact.custom_fields) {
        if (contact.custom_fields.school_name) {
          newContact.district = contact.organization_name;
          newContact.organization = contact.custom_fields.school_name;
          // console.log(`DEBUG: found contact with school name ${newContact.email} ${contact.custom_fields.school_name}`);
        }
        if (contact.custom_fields.nces_district_id) {
          newContact.nces_district_id = contact.custom_fields.nces_district_id;
          // console.log(`DEBUG: found contact with district id ${newContact.email} ${newContact.nces_district_id}`);
        }
        if (contact.custom_fields.nces_school_id) {
          newContact.nces_school_id = contact.custom_fields.nces_school_id;
          // console.log(`DEBUG: found contact with school id ${newContact.email} ${newContact.nces_school_id}`);
        }
      }
      contacts.push(newContact);
    }
    return done(null, data.pipeline_total);
  });
}

function createGetZPAutoResponderContactsPage(contacts, page) {
  return (done) => {
    // console.log(`DEBUG: Fetching autoresponder page ${page} ${zpPageSize}...`);
    let searchQuery = `codecombat_special_auth_token=${zpAuthToken}&page=${page}&per_page=${zpPageSize}&contact_email_autoresponder=true`;
    getZPContactsPage(contacts, searchQuery, done);
  };
}

function createGetZPRepliedContactsPage(contacts, page) {
  return (done) => {
    // console.log(`DEBUG: Fetching email reply page ${page} ${zpPageSize}...`);
    let searchQuery = `codecombat_special_auth_token=${zpAuthToken}&page=${page}&per_page=${zpPageSize}&contact_email_replied=true`;
    getZPContactsPage(contacts, searchQuery, done);
{'fraction_non_alphanumeric': 0.15355833270972086, 'fraction_numerical': 0.002394671855122353, 'mean_word_length': 3.448735019973369, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 37, 'https://': 12, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3731368', 'n_tokens_mistral': 4324, 'n_tokens_neox': 4309, 'n_words': 849}