<commit_before>const braveHapi = require('../brave-hapi')
const json2csv = require('json2csv')
const underscore = require('underscore')

const create = require('./reports.js').create

const publish = async (debug, runtime, method, publisher, endpoint, payload) => {
  const prefix = publisher ? ('/' + encodeURIComponent(publisher)) : ''
  let result

  result = await braveHapi.wreck[method](runtime.config.publishers.url + '/api/publishers/' + prefix + (endpoint || ''),
    { headers: { authorization: 'Bearer ' + runtime.config.publishers.access_token,
      'content-type': 'application/json'
    },
      payload: JSON.stringify(payload),
      useProxyP: true
    })
  if (Buffer.isBuffer(result)) result = JSON.parse(result)

  return result
}

var exports = {}

exports.initialize = async (debug, runtime) => {
  await runtime.queue.create('publishers-bulk-create')
}

exports.workers = {
/* sent by POST /v1/publishers

    { queue            : 'publishers-bulk-create'
    , message          :
      { reportId       : '...'
      , reportURL      : '...'
      , authority      : '...:...'
      , format         : 'json' | 'csv'
      , publishers     :
        [ { publisher  : '...'
          , name       : '...'
          , phone      : '...'
          , show_verification_status
                       : true | false
          }
          ...
        ]
      }
    }
 */
  'publishers-bulk-create':
    async (debug, runtime, payload) => {
      const authority = payload.authority
      const format = payload.format || 'csv'
      const publishers = payload.publishers
      const publishersC = runtime.db.get('publishers', debug)
      let file, result, state

      state = {
        $currentDate: { timestamp: { $type: 'timestamp' } },
        $set: { verified: true, reason: 'bulk loaded', authority: authority }
      }
      for (let entry of publishers) {
        try {
          result = await publish(debug, runtime, 'post', '', '', {
            publisher: underscore.extend({ brave_publisher_id: entry.publisher, verified: true },
                                         underscore.omit(entry, [ 'publisher' ]))
          })
          await publishersC.update({ publisher: entry.publisher }, state, { upsert: true })

          entry.message = result && result.message
        } catch (ex) {
          entry.message = ex.toString()
        }
      }
<commit_msg>send publisher-report on bulk publisher verification<commit_after>const braveHapi = require('../brave-hapi')
const json2csv = require('json2csv')
const underscore = require('underscore')

const create = require('./reports.js').create

const publish = async (debug, runtime, method, publisher, endpoint, payload) => {
  const prefix = publisher ? ('/' + encodeURIComponent(publisher)) : ''
  let result

  result = await braveHapi.wreck[method](runtime.config.publishers.url + '/api/publishers/' + prefix + (endpoint || ''),
    { headers: { authorization: 'Bearer ' + runtime.config.publishers.access_token,
      'content-type': 'application/json'
    },
      payload: JSON.stringify(payload),
      useProxyP: true
    })
  if (Buffer.isBuffer(result)) result = JSON.parse(result)

  return result
}

var exports = {}

exports.initialize = async (debug, runtime) => {
  await runtime.queue.create('publisher-report')
  await runtime.queue.create('publishers-bulk-create')
}

exports.workers = {
/* sent by POST /v1/publishers

    { queue            : 'publishers-bulk-create'
    , message          :
      { reportId       : '...'
      , reportURL      : '...'
      , authority      : '...:...'
      , format         : 'json' | 'csv'
      , publishers     :
        [ { publisher  : '...'
          , name       : '...'
          , phone      : '...'
          , show_verification_status
                       : true | false
          }
          ...
        ]
      }
    }
 */
  'publishers-bulk-create':
    async (debug, runtime, payload) => {
      const authority = payload.authority
      const format = payload.format || 'csv'
      const publishers = payload.publishers
      const publishersC = runtime.db.get('publishers', debug)
      let file, result, state

      state = {
        $currentDate: { timestamp: { $type: 'timestamp' } },
        $set: { verified: true, reason: 'bulk loaded', authority: authority }
      }
      for (let entry of publishers) {
        try {
          result = await publish(debug, runtime, 'post', '', '', {
            publisher: underscore.extend({ brave_publisher_id: entry.publisher, verified: true },
                                         underscore.omit(entry, [ 'publisher' ]))
          })
          await publishersC.update({ publisher: entry.publisher }, state, { upsert: true })

          entry.message = result && result.message

          if (entry.message === 'success') {
            await runtime.queue.send(debug, 'publisher-report', { publisher: entry.publisher, verified: true })
          }
        } catch (ex) {
          entry.message = ex.toString()
        }
      }

{'fraction_non_alphanumeric': 0.14288545382481174, 'fraction_numerical': 0.0011890606420927466, 'mean_word_length': 2.0830788026878437, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5868942', 'n_tokens_mistral': 1423, 'n_tokens_neox': 1399, 'n_words': 311}