<commit_before>#
# A script that collects article data from our mongo database, creates a CSV document,
# and sends that data to S3 for consumption by Fulcrum.
#

knox = require 'knox'
mongojs = require 'mongojs'
fs = require 'fs'
path = require 'path'
moment = require 'moment'
{ pluck, object } = require 'underscore'

# Setup environment variables
env = require 'node-env-file'
switch process.env.NODE_ENV
  when 'test' then env path.resolve __dirname, '../.env.test'
  when 'production', 'staging' then ''
  else env path.resolve __dirname, '../.env'

# Connect to database
db = mongojs(process.env.MONGOHQ_URL, ['articles'])

# Setup file naming
filename = "export_" + moment().format('YYYYMMDDhhmmss') + ".csv"
dir = 'scripts/tmp/'

attrs = ['id', 'author_id', 'auction_ids', 'contributing_authors', 'fair_ids', 'featured', 'featured_artist_ids', 'featured_artwork_ids', 'partner_ids', 'primary_featured_artist_ids', 'slugs', 'tags', 'title', 'tier', 'published_at', 'show_ids', 'section_ids', 'thumbnail_image', 'thumbnail_title', 'keywords', 'slug', 'channel_id', 'partner_channel_id']
projections = object attrs, attrs.map -> 1

db.articles.find({ published: true }, projections).toArray (err, articles) ->

  stringify = (arr) ->
    return null unless arr
    str = arr.toString().replace(/"/gi, "'").replace(/\n/gi, "")
    '\"' + str + '\"'

  csv = [ attrs.join(',') ]

  articles.map (a) ->
    published_at = if a.published_at then moment(a.published_at).format('YYYY-MM-DDThh:mm') + "-05:00" else ''
    contributing_authors = pluck(a.contributing_authors, 'name')?.toString()
    row = [<commit_msg>Stringify contributing authors<commit_after>#
# A script that collects article data from our mongo database, creates a CSV document,
# and sends that data to S3 for consumption by Fulcrum.
#

knox = require 'knox'
mongojs = require 'mongojs'
fs = require 'fs'
path = require 'path'
moment = require 'moment'
{ pluck, object } = require 'underscore'

# Setup environment variables
env = require 'node-env-file'
switch process.env.NODE_ENV
  when 'test' then env path.resolve __dirname, '../.env.test'
  when 'production', 'staging' then ''
  else env path.resolve __dirname, '../.env'

# Connect to database
db = mongojs(process.env.MONGOHQ_URL, ['articles'])

# Setup file naming
filename = "export_" + moment().format('YYYYMMDDhhmmss') + ".csv"
dir = 'scripts/tmp/'

attrs = ['id', 'author_id', 'auction_ids', 'contributing_authors', 'fair_ids', 'featured', 'featured_artist_ids', 'featured_artwork_ids', 'partner_ids', 'primary_featured_artist_ids', 'slugs', 'tags', 'title', 'tier', 'published_at', 'show_ids', 'section_ids', 'thumbnail_image', 'thumbnail_title', 'keywords', 'slug', 'channel_id', 'partner_channel_id']
projections = object attrs, attrs.map -> 1

db.articles.find({ published: true }, projections).toArray (err, articles) ->

  stringify = (arr) ->
    return null unless arr
    str = arr.toString().replace(/"/gi, "'").replace(/\n/gi, "")
    '\"' + str + '\"'

  csv = [ attrs.join(',') ]

  articles.map (a) ->
    published_at = if a.published_at then moment(a.published_at).format('YYYY-MM-DDThh:mm') + "-05:00" else ''
    contributing_authors = stringify(pluck(a.contributing_authors, 'name'))
    row = [
{'fraction_non_alphanumeric': 0.16676923076923078, 'fraction_numerical': 0.0036923076923076922, 'mean_word_length': 4.76418439716312, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2951062', 'n_tokens_mistral': 1132, 'n_tokens_neox': 1094, 'n_words': 288}