<commit_before>var pnet = require('pnet')
  , keys = Object.keys;

var PNET_SETLIST_METHOD = 'shows.setlists.get';

exports.scrape = function(date, options, cb){
  var apikey = options.pnetApikey;

  if (null == apikey) {
    process.nextTick(cb.bind(null, new Error('pnetApikey option is required')));
    return
  }

  getShowsForDate(date, apikey, cb);
}

function getShowsForDate(date, apikey, cb){
  var options = {showdate: date, apikey: apikey};

  pnet.get(PNET_SETLIST_METHOD, options, function(err, _, shows){
    if (null != err)
      cb(err);
    else
      cb(null, shows.map(formatShow));
  });
}


/**
 * Shuffle around attributes from the Phish.net API setlist response.
 *
 * It returns a number of attributes specific to its own API as well
 * as a number of attributes that are generic. The idea is to group
 * the Phish.net specific attributes (i.e. venue id, artist id, show id)
 * under `pnet` properties and leave the generic attributes (i.e. date,
 * location, songs) as is.
 *
 * Further, we rename and reformat/regroup some of these attributes to make
 * a bit more sense (i.e. putting venue info into an object titled `venue`
 * rather than leaving it mixed in top-level).
 */

function formatShow(rawshow){
  return {
    artist: rawshow['artist-name'],
    date: rawshow.showdate,
    year: +rawshow.showyear,
    humanDate: rawshow.nicedate,
    friendlyDate: rawshow.mmddyy,
    sets: formatSongs(rawshow.sets),
    encores: formatSongs(rawshow.encores),
    footnotes: rawshow.footnotes,
    pnet: {
      artistId: +rawshow.artist,
      meta: rawshow.meta,
      setlistNotes: rawshow.setlistnotes,
      showId: +rawshow.showid,
      url: rawshow.url
    },
    venue: {
      name: rawshow.venue,
      notes: rawshow.venuenotes,
      location: {
        city: rawshow.city,
        country: rawshow.country,
        state: rawshow.state
      },
      pnet: {
        venueId: +rawshow.venueid
      }
    }
  }
}

function formatSongs(sets){
  return sets.map(function(set){
    return set.map(function(song){
      return keys(song).reduce(function(memo, key){
        ('url' !== key ? memo : memo.pnet)[key] = song[key];
        return memo;
      }, {pnet: {}});
    });
  });
}
<commit_msg>Move venue notes under pnet attribute<commit_after>var pnet = require('pnet')
  , keys = Object.keys;

var PNET_SETLIST_METHOD = 'shows.setlists.get';

exports.scrape = function(date, options, cb){
  var apikey = options.pnetApikey;

  if (null == apikey) {
    process.nextTick(cb.bind(null, new Error('pnetApikey option is required')));
    return
  }

  getShowsForDate(date, apikey, cb);
}

function getShowsForDate(date, apikey, cb){
  var options = {showdate: date, apikey: apikey};

  pnet.get(PNET_SETLIST_METHOD, options, function(err, _, shows){
    if (null != err)
      cb(err);
    else
      cb(null, shows.map(formatShow));
  });
}


/**
 * Shuffle around attributes from the Phish.net API setlist response.
 *
 * It returns a number of attributes specific to its own API as well
 * as a number of attributes that are generic. The idea is to group
 * the Phish.net specific attributes (i.e. venue id, artist id, show id)
 * under `pnet` properties and leave the generic attributes (i.e. date,
 * location, songs) as is.
 *
 * Further, we rename and reformat/regroup some of these attributes to make
 * a bit more sense (i.e. putting venue info into an object titled `venue`
 * rather than leaving it mixed in top-level).
 */

function formatShow(rawshow){
  return {
    artist: rawshow['artist-name'],
    date: rawshow.showdate,
    year: +rawshow.showyear,
    humanDate: rawshow.nicedate,
    friendlyDate: rawshow.mmddyy,
    sets: formatSongs(rawshow.sets),
    encores: formatSongs(rawshow.encores),
    footnotes: rawshow.footnotes,
    pnet: {
      artistId: +rawshow.artist,
      meta: rawshow.meta,
      setlistNotes: rawshow.setlistnotes,
      showId: +rawshow.showid,
      url: rawshow.url
    },
    venue: {
      name: rawshow.venue,
      location: {
        city: rawshow.city,
        country: rawshow.country,
        state: rawshow.state
      },
      pnet: {
        notes: rawshow.venuenotes,
        venueId: +rawshow.venueid
      }
    }
  }
}

function formatSongs(sets){
  return sets.map(function(set){
    return set.map(function(song){
      return keys(song).reduce(function(memo, key){
        ('url' !== key ? memo : memo.pnet)[key] = song[key];
        return memo;
      }, {pnet: {}});
    });
  });
}

{'fraction_non_alphanumeric': 0.12277580071174377, 'fraction_numerical': 0.0, 'mean_word_length': 3.0295698924731185, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1281169', 'n_tokens_mistral': 1499, 'n_tokens_neox': 1476, 'n_words': 409}