<commit_before>// This scripts updates candidate data from the CSV of volunteer database
// It prompts on each update

// Open csv file
var parse = require('csv-parse')

const Bookshelf = require('../../bookshelf')
const Candidate = require('../../models/candidate')

// var inputFile = 'data/2018-data-from-volunteers-2018-07-16.csv'
// var inputFile = 'short.csv'

const columnDefinitions = {
  contests: {
    name: 'contests',
    columnNumber: 0,
    dbColumnName: false,
  },
  candidateName: {
    name: 'Candidate Name',
    columnNumber: 1,
    dbColumnName: 'candidate_name',
  },
  party: {
    name: 'Party',
    columnNumber: 2,
    dbColumnName: 'party',
  },
  _emailWebSiteIgnore: {
    name: 'EmailWebsiteIgnore',
    columnNumber: 3,
    dbColumnName: false,
  },
  phoneNumber: {
    name: 'Phone Number',
    columnNumber: 4,
    dbColumnName: 'phone',
  },
  candWebsiteUrl: {
    name: 'cand_website_url',
    columnNumber: 5,
    dbColumnName: 'website',
  },
  candFacebookUrl: {
    name: 'cand_facebook_url',
    columnNumber: 6,
    dbColumnName: 'facebook_url',
  },
  candTwitterUrl: {
    name: 'cand_twitter_url',
    columnNumber: 7,
    dbColumnName: 'twitter_url',
  },
  candInstagramUrl: {
    name: 'cand_instagram_url',
    columnNumber: 8,
    dbColumnName: 'instagram_url',
  },
  candEmailAddress: {
    name: 'cand_email_address',
    columnNumber: 9,
    dbColumnName: 'email_address',
  },
  ccRegistrationNumber: {
    name: 'cc_registration_number',
    columnNumber: 10,
    dbColumnName: 'cc_registration_number',
  },
  fecCCId: {
    name: 'fec_cc_id',
    columnNumber: 11,
    dbColumnName: 'fec_cc_id',
  },
  previousCivilBeatUrl: {
    name: 'civil_beat_url',
    columnNumber: 12,
    dbColumnName: false,
  },
  civilBeatUrl: {
    name: 'cb_url_2018',
    columnNumber: 13,
    dbColumnName: 'civil_beat_url',
  },
  leageWomenVotersUrl: {
    name: 'lwv_url',
    columnNumber: 14,
    dbColumnName: false,
  },
  starAdvertiserUrl: {
    name: 'staradvertiser_url',
    columnNumber: 15,
    dbColumnName: 'staradvertiser_url',
  },
  candidatePhotoUrl: {
    name: 'candidate_photo_url',
    columnNumber: 16,
    dbColumnName: 'candidate_photo_url',
  },
}

const columnDefinitionsByLine = Object.values(columnDefinitions).reduce((result, definition) => {
  result[definition.columnNumber] = definition
  return result
}, {})

var parser = parse({delimiter: ','}, function (err, data) {
  data.reduce((promise, line) => {
    return promise.then(() => {
      return handleCsvLine(line)
    })
  }, new Promise(resolve => resolve()))
    .then(function() {
      console.log("Destroying bookshelf pool so process can exit")
      Bookshelf.knex.destroy()
    })
})

// fs.createReadStream(inputFile).pipe(parser)
process.stdin.pipe(parser)

// Print out each candidate
function handleCsvLine(line) {
  const candidateName = readColumn(columnDefinitions.candidateName, line)
  if (candidateName === 'Candidate Name') return new Promise(resolve => resolve())

  console.log('candidateName', candidateName)
  return Candidate
    .forge({candidate_name: candidateName})
    .fetch()
    .then(foundCandidate => {
      // Compare the attributes in the database
      if (foundCandidate) {
        compareAttributes(foundCandidate, line)
      } else {
        console.log(`Unable to find information for ${candidateName}`)
        return new Promise(resolve => resolve())
      }
    })
}

function readColumn(columnDefinition, line) {
  return line[columnDefinition.columnNumber]
}

// Compare the attributes from the csv and the database
// If they are different then prompt the user if we want to change the data in
// the database
async function compareAttributes(candidate, line) {<commit_msg>Read hard-coded file<commit_after>// This scripts updates candidate data from the CSV of volunteer database
// It prompts on each update

// Open csv file
var parse = require('csv-parse')

const Bookshelf = require('../../bookshelf')
const Candidate = require('../../models/candidate')

var inputFile = 'data/2018-data-from-volunteers-2018-07-16.csv'
// var inputFile = 'short.csv'

const columnDefinitions = {
  contests: {
    name: 'contests',
    columnNumber: 0,
    dbColumnName: false,
  },
  candidateName: {
    name: '<NAME>',
    columnNumber: 1,
    dbColumnName: 'candidate_name',
  },
  party: {
    name: 'Party',
    columnNumber: 2,
    dbColumnName: 'party',
  },
  _emailWebSiteIgnore: {
    name: 'EmailWebsiteIgnore',
    columnNumber: 3,
    dbColumnName: false,
  },
  phoneNumber: {
    name: 'Phone Number',
    columnNumber: 4,
    dbColumnName: 'phone',
  },
  candWebsiteUrl: {
    name: 'cand_website_url',
    columnNumber: 5,
    dbColumnName: 'website',
  },
  candFacebookUrl: {
    name: 'cand_facebook_url',
    columnNumber: 6,
    dbColumnName: 'facebook_url',
  },
  candTwitterUrl: {
    name: 'cand_twitter_url',
    columnNumber: 7,
    dbColumnName: 'twitter_url',
  },
  candInstagramUrl: {
    name: 'cand_instagram_url',
    columnNumber: 8,
    dbColumnName: 'instagram_url',
  },
  candEmailAddress: {
    name: 'cand_email_address',
    columnNumber: 9,
    dbColumnName: 'email_address',
  },
  ccRegistrationNumber: {
    name: 'cc_registration_number',
    columnNumber: 10,
    dbColumnName: 'cc_registration_number',
  },
  fecCCId: {
    name: 'fec_cc_id',
    columnNumber: 11,
    dbColumnName: 'fec_cc_id',
  },
  previousCivilBeatUrl: {
    name: 'civil_beat_url',
    columnNumber: 12,
    dbColumnName: false,
  },
  civilBeatUrl: {
    name: 'cb_url_2018',
    columnNumber: 13,
    dbColumnName: 'civil_beat_url',
  },
  leageWomenVotersUrl: {
    name: 'lwv_url',
    columnNumber: 14,
    dbColumnName: false,
  },
  starAdvertiserUrl: {
    name: 'staradvertiser_url',
    columnNumber: 15,
    dbColumnName: 'staradvertiser_url',
  },
  candidatePhotoUrl: {
    name: 'candidate_photo_url',
    columnNumber: 16,
    dbColumnName: 'candidate_photo_url',
  },
}

const columnDefinitionsByLine = Object.values(columnDefinitions).reduce((result, definition) => {
  result[definition.columnNumber] = definition
  return result
}, {})

var parser = parse({delimiter: ','}, function (err, data) {
  data.reduce((promise, line) => {
    return promise.then(() => {
      return handleCsvLine(line)
    })
  }, new Promise(resolve => resolve()))
    .then(function() {
      console.log("Destroying bookshelf pool so process can exit")
      Bookshelf.knex.destroy()
    })
})

fs.createReadStream(inputFile).pipe(parser)
// process.stdin.pipe(parser)

// Print out each candidate
function handleCsvLine(line) {
  const candidateName = readColumn(columnDefinitions.candidateName, line)
  if (candidateName === 'Candidate Name') return new Promise(resolve => resolve())

  console.log('candidateName', candidateName)
  return Candidate
    .forge({candidate_name: candidateName})
    .fetch()
    .then(foundCandidate => {
      // Compare the attributes in the database
      if (foundCandidate) {
        compareAttributes(foundCandidate, line)
      } else {
        console.log(`Unable to find information for ${candidateName}`)
        return new Promise(resolve => resolve())
      }
    })
}

function readColumn(columnDefinition, line) {
  return line[columnDefinition.columnNumber]
}

// Compare the attributes from the csv and the database
// If they are different then prompt the user if we want to change the data in
// the database
async function compareAttributes(candidate, line) {
{'fraction_non_alphanumeric': 0.1255659121171771, 'fraction_numerical': 0.010652463382157125, 'mean_word_length': 3.5521212121212122, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4997796', 'n_tokens_mistral': 2513, 'n_tokens_neox': 2400, 'n_words': 551}