<commit_before>
// make a request to lastFM
function lastFM(data, callback){
  return reqwest({
    url:"https://ws.audioscrobbler.com/2.0/",
    data: data,
    type: data.format || 'xml',
    success: function(data){
      if(callback){callback(false, data)}
    },
    error: function(err){
      if(callback){callback(err)}
    }
  })
}

function requestUser(api_key, user){
  return {
    method:'user.getInfo',
    user: user,
    api_key: api_key,
    format: 'json'
  }
}

function normaliseUser(data){
  console.log(data);
  return data.user
}


// Weeks look like they might be pretty
// useless
function requestWeeks(api_key, user){
  return {
    method:'user.getweeklychartlist',
    user: user,
    api_key: api_key,
    format: 'json'
  }
}

function normaliseWeeks(data){
  return {
    name: data.weeklychartlist['@attr'].user,
    weeks: data.weeklychartlist.chart.map(function(chart){
      return {
        from: chart.from,
        to:   chart.to
      }
    })
  };
}

// generate data for a request
function requestData(api_key, user, page){
  return {
    method:'user.getrecenttracks',
    user:user,
    api_key:api_key,
    limit:200,
    page: page || 1
  }
}

// generate a list of request data objects
function requestList(api_key, user, page_count){
  var requests = [];
  for(var page = 1; page <= page_count; page++){
    requests.push(requestData(api_key, user, page))
  }
  return requests
}

// extract the data from the xml response
function extractTracks(doc){

  // probably nicer ways to do this
  var arr = [];
  var track, obj, child;
  var tracks = doc.evaluate('lfm/recenttracks/track', doc, null, XPathResult.ANY_TYPE, null)
  while (track = tracks.iterateNext()){
    obj = {};
    for (var i = track.childNodes.length - 1; i >= 0; i--) {
      child = track.childNodes[i];
      if(child.tagName)
        obj[child.tagName] = child.textContent;
    };
    arr.push(obj)
  }

  return arr;
}

function extractPageCount(doc){
  var recenttracks = doc.evaluate('lfm/recenttracks', doc, null, XPathResult.ANY_TYPE, null).iterateNext()
  return parseInt(recenttracks.getAttribute('totalPages'), 10)
}

// pull out a row of keys
function row(keys, obj){
  return keys.map(function(k){
    return obj[k]
  })
}

// create a csv row from an array
function csv(array){

  // this is not a world class csv generator
  return array.map(function(item){
    return  typeof(item) === 'string' ?
      item.replace(/[\",]/g,'') :
      item;
  }).join(',')
}

// delay a function by millis
function delay(fn, millis){
  return function(){
    var args = [].slice.call(arguments);
    setTimeout.apply(this, [fn,millis].concat(args));
  }
}


/*
var output = [];<commit_msg>pull out user and first track time from response doc<commit_after>
// make a request to lastFM
function lastFM(data, callback){
  return reqwest({
    url:"https://ws.audioscrobbler.com/2.0/",
    data: data,
    type: data.format || 'xml',
    success: function(data){
      if(callback){callback(false, data)}
    },
    error: function(err){
      if(callback){callback(err)}
    }
  })
}

function requestUser(api_key, user){
  return {
    method:'user.getInfo',
    user: user,
    api_key: api_key,
    format: 'json'
  }
}

function normaliseUser(data){
  console.log(data);
  return data.user
}


// Weeks look like they might be pretty
// useless
function requestWeeks(api_key, user){
  return {
    method:'user.getweeklychartlist',
    user: user,
    api_key: api_key,
    format: 'json'
  }
}

function normaliseWeeks(data){
  return {
    name: data.weeklychartlist['@attr'].user,
    weeks: data.weeklychartlist.chart.map(function(chart){
      return {
        from: chart.from,
        to:   chart.to
      }
    })
  };
}

// generate data for a request
function requestData(api_key, user, page){
  return {
    method:'user.getrecenttracks',
    user:user,
    api_key:api_key,
    limit:200,
    page: page || 1
  }
}

// generate a list of request data objects
function requestList(api_key, user, page_count){
  var requests = [];
  for(var page = 1; page <= page_count; page++){
    requests.push(requestData(api_key, user, page))
  }
  return requests
}

// extract the data from the xml response
function extractTracks(doc){

  // probably nicer ways to do this
  var arr = [];
  var track, obj, child;
  var tracks = doc.evaluate('lfm/recenttracks/track', doc, null, XPathResult.ANY_TYPE, null)
  while (track = tracks.iterateNext()){
    obj = {};
    for (var i = track.childNodes.length - 1; i >= 0; i--) {
      child = track.childNodes[i];
      if(child.tagName)
        obj[child.tagName] = child.textContent;
    };
    arr.push(obj)
  }

  return arr;
}

function extractPageCount(doc){
  var recenttracks = doc.evaluate('lfm/recenttracks', doc, null, XPathResult.ANY_TYPE, null).iterateNext()
  return parseInt(recenttracks.getAttribute('totalPages'), 10)
}

function extractUser(doc){
  var recenttracks = doc.evaluate('lfm/recenttracks', doc, null, XPathResult.ANY_TYPE, null).iterateNext()
  return recenttracks.getAttribute('user')
}

function extractFirstTrackTime(doc){
  var date = doc.evaluate('lfm/recenttracks/track/date', doc, null, XPathResult.ANY_TYPE, null).iterateNext()
  return parseInt(date.getAttribute('uts'), 10)
}

// pull out a row of keys
function row(keys, obj){
  return keys.map(function(k){
    return obj[k]
  })
}

// create a csv row from an array
function csv(array){

  // this is not a world class csv generator
  return array.map(function(item){
    return  typeof(item) === 'string' ?
      item.replace(/[\",]/g,'') :
      item;
  }).join(',')
}

// delay a function by millis
function delay(fn, millis){
  return function(){
    var args = [].slice.call(arguments);
    setTimeout.apply(this, [fn,millis].concat(args));
  }
}


/*
var output = [];
{'fraction_non_alphanumeric': 0.14398489788913676, 'fraction_numerical': 0.004118757508151707, 'mean_word_length': 3.2695970695970695, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5352936', 'n_tokens_mistral': 1998, 'n_tokens_neox': 1936, 'n_words': 486}