<commit_before>var fs = require('fs')
var qs = require('qs')
var request = require('request')
var csvWriter = require('csv-write-stream')

var config = require('./config.json')

var headers = ['dplaLink', 'link', 'title', 'sender', 'recipient',
               'year', 'description', 'place', 'date']

var writer = csvWriter({headers: headers})
writer.pipe(fs.createWriteStream('letters.csv'))


// Set our API query parameters
var baseUrl = 'http://api.dp.la/v2/items?'
var queryString = {
  api_key: config.api_key,
  sourceResource: {
    subject: {
      name: '<NAME>'
    }
  },
  page_size: 500,
  page: 1
}

getData()


function getData() {
  // Build the URL string and set options for the request
  var options = {
    url: baseUrl + qs.stringify(queryString, {allowDots: true}),
    json: true
  }

  // Request data from the API
  request(options, (err, res, records) => {
    if (!err && res.statusCode == 200) {
      records.docs.map((l) => {

        letter = {link_dpla: l.hasOwnProperty('@id') ? l['@id'] : l.object['@id']}

        // Metadata contains most of the information we want, but isn't always present
        if (l.originalRecord.hasOwnProperty('metadata')) {

            letter.title = l.originalRecord.metadata.title
            letter.year = l.originalRecord.metadata.date
            letter.link_org = l.originalRecord.metadata['identifier-access']<commit_msg>fix link and dplaLink columns<commit_after>var fs = require('fs')
var qs = require('qs')
var request = require('request')
var csvWriter = require('csv-write-stream')

var config = require('./config.json')

var headers = ['dplaLink', 'link', 'title', 'sender', 'recipient',
               'year', 'description', 'place', 'date']

var writer = csvWriter({headers: headers})
writer.pipe(fs.createWriteStream('letters.csv'))


// Set our API query parameters
var baseUrl = 'http://api.dp.la/v2/items?'
var queryString = {
  api_key: config.api_key,
  sourceResource: {
    subject: {
      name: '<NAME>'
    }
  },
  page_size: 500,
  page: 1
}

getData()


function getData() {
  // Build the URL string and set options for the request
  var options = {
    url: baseUrl + qs.stringify(queryString, {allowDots: true}),
    json: true
  }

  // Request data from the API
  request(options, (err, res, records) => {
    if (!err && res.statusCode == 200) {
      records.docs.map((l) => {

        letter = {dplaLink: l.hasOwnProperty('@id') ? l['@id'] : l.object['@id']}

        // Metadata contains most of the information we want, but isn't always present
        if (l.originalRecord.hasOwnProperty('metadata')) {

            letter.title = l.originalRecord.metadata.title
            letter.year = l.originalRecord.metadata.date
            letter.link = l.originalRecord.metadata['identifier-access']
{'fraction_non_alphanumeric': 0.15373614587057563, 'fraction_numerical': 0.00572041473006793, 'mean_word_length': 3.1949025487256373, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6229750', 'n_tokens_mistral': 897, 'n_tokens_neox': 866, 'n_words': 227}