<commit_before>var feedparser = require('ortoo-feedparser');
var request = require('request');
var fs      = require('fs');
var db = require('./lib/feedDB');
var mediaDir = '';

fs.exists('media/', function(exists) {
  if (exists) {
    mediaDir = 'media/';
  }
});

// Download media at specified URL and set a ticker to indicate activity.
function downloadMedia(mediaUrl, title) {
  var media = request(mediaUrl).pipe(fs.createWriteStream(mediaDir+title));

  console.log('Downloading ' + title);

  // Set 15sec ticker to indicate download activity.
  var waitTick = setInterval(function() {
    console.log("...");
  }, 15000);

  media.on('finish', function() {
    clearInterval(waitTick);
    console.log(title + ' is done!');
  });
}

// Get and return article from array, by date.
function getByDate(date, articles) {
  var article;
  for (var i=0; i< articles.length; i++) {
    // Check if the date of article[i] is a match.
    if (articles[i].pubDate.toISOString().slice(0,10) == date) {
      article = articles[i];
    }
  }
  return article;
}

// Determine media type and set filename accordingly.
function getMediaType(article, meta) {
  var extension = article.enclosures[0].type;
  var date = article.pubDate.toISOString().slice(0,10);
  // Set extension to the filetype in .type, and ensure filename is lower case.
  extension = '.' + extension.slice(extension.indexOf('/')+1, extension.length);
  var title = meta.title + '-' + date + extension;
  title = title.replace(/\s+/g, '').toLowerCase();

  return title;
}

// Default function for module, returning metadata of specified feed.
function podcatcher(feedUrl, cb) {
  feedparser.parseUrl(feedUrl, function(err, meta, articles) {
    if (err) {
      return cb(err, meta);
    } else {
      return cb(null, meta);
    }
  });
}

// Same functionality as podcatcher(), added for consistency with other methods.
podcatcher.getAll = podcatcher;

// Methods from lib/feedDB.js for reading and writing feed entries to and from levelDB
podcatcher.saveFeed = db.save;
podcatcher.readFeed = db.get;

// Get the latest media in the specified podcast stream and download it.
podcatcher.getNewest = function(feedUrl, cb) {

  // Callback to locate the newest media and download it.
  function getArticle(err, meta, articles) {
    var article = articles[0];

    if (meta) {
      return cb(null, meta, article);
    } else if (err) {
      return cb(new Error('Error 500'));
    }
  }

  feedparser.parseUrl(feedUrl, getArticle);
};

podcatcher.downloadNewest = function(feedUrl, cb) {

  // Callback to download media.
  function download(err, meta, article) {
    var fileName = getMediaType(article, meta);
    downloadMedia(article.enclosures[0].url, fileName);

    if (err) {
      return cb(new Error('Error 500'));
    } else {
      return cb(null, meta, article);
    }
  }

  // Get the feed and issue callback.
  this.getNewest(feedUrl, download);
};

// Get the specified article in the specified podcast stream and download it.
podcatcher.getByDate = function(feedUrl, date, cb) {

  // Callback to locate the media at date and download it.
  function getArticle(err, meta, articles) {
    var article = getByDate(date, articles);

    if (meta) {
      return cb(null, meta, article);
    } else if (err) {
      return cb(new Error('Error 500'));
    }
  }

  // Parse feed and download in parser's callback.
  feedparser.parseUrl(feedUrl, getArticle);
};

podcatcher.downloadByDate = function(feedUrl, date, cb) {

  // Callback to download media.
  function download(err, meta, article) {
    var fileName = getMediaType(article, meta);
    downloadMedia(article.enclosures[0].url, fileName);

    if (err) {
      return cb(new Error('Error 500'));
    } else {
      return cb(null, meta, article);
    }
  }

  // Find the feed and issue the callback.
  this.getByDate(feedUrl, date, download);
}

module.exports = podcatcher;
<commit_msg>Removed superfluous db module<commit_after>var feedparser = require('ortoo-feedparser');
var request = require('request');
var fs      = require('fs');
var mediaDir = '';

fs.exists('media/', function(exists) {
  if (exists) {
    mediaDir = 'media/';
  }
});

/*
 * Helper functions for use in identifying and downloading feeds/articles
 */

// Download media at specified URL and set a ticker to indicate activity.
function downloadMedia(mediaUrl, title) {
  var media = request(mediaUrl).pipe(fs.createWriteStream(mediaDir+title));

  console.log('Downloading ' + title);

  // Set 15sec ticker to indicate download activity.
  var waitTick = setInterval(function() {
    console.log("...");
  }, 15000);

  media.on('finish', function() {
    clearInterval(waitTick);
    console.log(title + ' is done!');
  });
}

// Get and return article from array, by date.
function getByDate(date, articles) {
  var article;
  for (var i=0; i< articles.length; i++) {
    // Check if the date of article[i] is a match.
    if (articles[i].pubDate.toISOString().slice(0,10) == date) {
      article = articles[i];
    }
  }
  return article;
}

// Determine media type and set filename accordingly.
function getMediaType(article, meta) {
  var extension = article.enclosures[0].type;
  var date = article.pubDate.toISOString().slice(0,10);
  // Set extension to the filetype in .type, and ensure filename is lower case.
  extension = '.' + extension.slice(extension.indexOf('/')+1, extension.length);
  var title = meta.title + '-' + date + extension;
  title = title.replace(/\s+/g, '').toLowerCase();

  return title;
}

/*
 * Module methods
 */

// Default function for module, returning metadata of specified feed.
function podcatcher(feedUrl, cb) {
  feedparser.parseUrl(feedUrl, function(err, meta, articles) {
    if (err) {
      return cb(err);
    } else {
      return cb(null, meta, articles);
    }
  });
}

// Same functionality as podcatcher(), added for consistency with other methods.
podcatcher.getAll = podcatcher;

// Get the latest media in the specified podcast stream and download it.
podcatcher.getNewest = function(feedUrl, cb) {

  // Callback to locate the newest media and download it.
  function getArticle(err, meta, articles) {
    var article = articles[0];

    if (meta) {
      return cb(null, meta, article);
    } else if (err) {
      return cb(err);
    }
  }

  feedparser.parseUrl(feedUrl, getArticle);
};

podcatcher.downloadNewest = function(feedUrl, cb) {

  // Callback to download media.
  function download(err, meta, article) {
    var fileName = getMediaType(article, meta);
    downloadMedia(article.enclosures[0].url, fileName);

    if (err) {
      return cb(err);
    } else {
      return cb(null, meta, article);
    }
  }

  // Get the feed and issue callback.
  this.getNewest(feedUrl, download);
};

// Get the specified article in the specified podcast stream and download it.
podcatcher.getByDate = function(feedUrl, date, cb) {

  // Callback to locate the media at date and download it.
  function getArticle(err, meta, articles) {
    var article = getByDate(date, articles);

    if (meta) {
      return cb(null, meta, article);
    } else if (err) {
      return cb(err);
    }
  }

  // Parse feed and download in parser's callback.
  feedparser.parseUrl(feedUrl, getArticle);
};

podcatcher.downloadByDate = function(feedUrl, date, cb) {

  // Callback to download media.
  function download(err, meta, article) {
    var fileName = getMediaType(article, meta);
    downloadMedia(article.enclosures[0].url, fileName);

    if (err) {
      return cb(err);
    } else {
      return cb(null, meta, article);
    }
  }

  // Find the feed and issue the callback.
  this.getByDate(feedUrl, date, download);
}

module.exports = podcatcher;

{'fraction_non_alphanumeric': 0.1310032362459547, 'fraction_numerical': 0.006472491909385114, 'mean_word_length': 3.384790011350738, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 39, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2169085', 'n_tokens_mistral': 2436, 'n_tokens_neox': 2367, 'n_words': 722}