<commit_before>// REQUIRES ---------------------------------------------------------------------------------------
const tpl = require('ejs')
const low = require('lowdb')
const lowDbFileAsync = require('lowdb/lib/file-async')
const request = require('request-promise')
const Q = require('q')
const fs = require('fs-extra')
const path = require('path')
const cheerio = require('cheerio')
const sleep = require('system-sleep')
const moment = require('moment')
const Slack = require('slack-node')
const Regex = require('xregexp')
// /REQUIRES --------------------------------------------------------------------------------------

const SLACK_WEBHOOK = process.env.LABS_SLACK_WEBHOOK_URL_DEVPARANA || ''
const dbFile = path.join(__dirname, 'data/db.json')
if (!fs.existsSync(path.dirname(dbFile)) && !fs.mkdirsSync(path.dirname(dbFile))) {
  throw new Error('Error creating data dir.')
}
const db = low(dbFile, { storage: lowDbFileAsync, writeOnChange: true })

const scrapperUrlBase = 'http://empregos.maringa.com/'
const keywords = [
  'desenvolvedor',
  'programador',
  'web',
  'php',
  'frontend',
  'front-end'
]

// blacklist certain words / expressions in title, to filter found Jobs
const blacklist = [
  'torno',
  'cnc',
  'ppcp',
  'usinagem',
  /venda?s/ig,
  /vendedor/ig,
  /servi[\xE7\xC7]os?/ig,
  /ve[\xCD\xED]culos?/ig,
  /manuten[\xC7\xE7][\xC3\xE3]o/ig,
  /neg[\xF3\xD3]cios?/ig,
  // NOTE: if you decide to use regular expression, don't forget the "ig" flags
]

const deferred = Q.defer()
const deferredProcessing = Q.defer()
const deferredSlack = Q.defer()

db.defaults({ jobs: [], settings: {} }).value()

let keywordsProcessed = 0
let websiteToken = ''
let websiteCookies = []
<commit_msg>added new word in the blacklist<commit_after>// REQUIRES ---------------------------------------------------------------------------------------
const tpl = require('ejs')
const low = require('lowdb')
const lowDbFileAsync = require('lowdb/lib/file-async')
const request = require('request-promise')
const Q = require('q')
const fs = require('fs-extra')
const path = require('path')
const cheerio = require('cheerio')
const sleep = require('system-sleep')
const moment = require('moment')
const Slack = require('slack-node')
const Regex = require('xregexp')
// /REQUIRES --------------------------------------------------------------------------------------

const SLACK_WEBHOOK = process.env.LABS_SLACK_WEBHOOK_URL_DEVPARANA || ''
const dbFile = path.join(__dirname, 'data/db.json')
if (!fs.existsSync(path.dirname(dbFile)) && !fs.mkdirsSync(path.dirname(dbFile))) {
  throw new Error('Error creating data dir.')
}
const db = low(dbFile, { storage: lowDbFileAsync, writeOnChange: true })

const scrapperUrlBase = 'http://empregos.maringa.com/'
const keywords = [
  'desenvolvedor',
  'programador',
  'web',
  'php',
  'frontend',
  'front-end'
]

// blacklist certain words / expressions in title, to filter found Jobs
const blacklist = [
  'torno',
  'cnc',
  'ppcp',
  'usinagem',
  'bordado',
  /venda?s/ig,
  /vendedor/ig,
  /servi[\xE7\xC7]os?/ig,
  /ve[\xCD\xED]culos?/ig,
  /manuten[\xC7\xE7][\xC3\xE3]o/ig,
  /neg[\xF3\xD3]cios?/ig
  // NOTE: if you decide to use regular expression, don't forget the "ig" flags
]

const deferred = Q.defer()
const deferredProcessing = Q.defer()
const deferredSlack = Q.defer()

db.defaults({ jobs: [], settings: {} }).value()

let keywordsProcessed = 0
let websiteToken = ''
let websiteCookies = []

{'fraction_non_alphanumeric': 0.2495657209033005, 'fraction_numerical': 0.0052113491603937466, 'mean_word_length': 2.81767955801105, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1802508', 'n_tokens_mistral': 1149, 'n_tokens_neox': 1124, 'n_words': 262}