<commit_before>var mongoose = require('mongoose');
var feedparser = require('feedparser')
var request = require('request');

var feedSchema = mongoose.Schema({
    _id:              { type: String, required: true, unique: true },
    title:            String,
    description:      String,
    link:             String,
    xmlUrl:           String,
    date:             { type: Date },
    pubDate:          { type: Date },
    author:           String,
    language:         String,
    image:            { title: String, url: String },
    favicon:          String,
    copyright:        String,
    generator:        String,
    categories:       [String]
});

feedSchema.virtual('url').get(function () {
  return this._id;
});

feedSchema.virtual('url').set(function (url) {
  this._id = url;
});

var Feed = mongoose.model('Feed', feedSchema);

var articleSchema = mongoose.Schema({
    title:            String,
    description:      String,
    link:             String,
    origLink:         String,
    date:             { type: Date },
    pubDate:          { type: Date },
    author:           String,
    guid:             String,
    comments:         String,
    image:            { title: String, url: String },
    categories:       [String],
    source:           { title: String, url: String },
    enclosures:       [{ url: String, type: String, length: String }],
    feed:             { type: String, ref: 'Feed' },
});

var Article = mongoose.model('Article', articleSchema);

var timeoutInMs = 10000;
var defaultMongoDbPort = 27017; 
var updateIntervalHandle = null;

function connect(uri, database, port) {
    port = typeof port !== 'undefined' ? port : defaultMongoDbPort;
    mongoose.connect(uri, database, port);
    
    mongoose.connection.on('error', function(e){
        console.error("Could not connect to MongoDB: "+e.message);
        process.exit(1);
    });
    
    mongoose.connection.once('open', function callback () {
        console.log("Connection to MongoDB was successful.");
    });
}

function addFeed(url) {
    Feed.count({ _id: url }, function(error, count) {
        if(error != null) {
            console.error('Failed to check if feed exists in MongoDB: '+error);
        }
        else {
            if(count == 0) {
               requestAndParseFeed(url);
            }
            else if(count == 1) {<commit_msg>Capped the amount of articles in database.<commit_after>var mongoose = require('mongoose');
var feedparser = require('feedparser')
var request = require('request');

var feedSchema = mongoose.Schema({
    _id:              { type: String, required: true, unique: true },
    title:            String,
    description:      String,
    link:             String,
    xmlUrl:           String,
    date:             { type: Date },
    pubDate:          { type: Date },
    author:           String,
    language:         String,
    image:            { title: String, url: String },
    favicon:          String,
    copyright:        String,
    generator:        String,
    categories:       [String]
});

feedSchema.virtual('url').get(function () {
  return this._id;
});

feedSchema.virtual('url').set(function (url) {
  this._id = url;
});

var Feed = mongoose.model('Feed', feedSchema);

var articleSchema = mongoose.Schema({
    title:            String,
    description:      String,
    link:             String,
    origLink:         String,
    date:             { type: Date },
    pubDate:          { type: Date },
    author:           String,
    guid:             String,
    comments:         String,
    image:            { title: String, url: String },
    categories:       [String],
    source:           { title: String, url: String },
    enclosures:       [{ url: String, type: String, length: String }],
    feed:             { type: String, ref: 'Feed' },
}, { capped: capForArticlesInBytes });

var Article = mongoose.model('Article', articleSchema);

var timeoutInMs = 10000;
var defaultMongoDbPort = 27017; 
var updateIntervalHandle = null;
var capForArticlesInBytes = 100000;

function connect(uri, database, port) {
    port = typeof port !== 'undefined' ? port : defaultMongoDbPort;
    mongoose.connect(uri, database, port);
    
    mongoose.connection.on('error', function(e){
        console.error("Could not connect to MongoDB: "+e.message);
        process.exit(1);
    });
    
    mongoose.connection.once('open', function callback () {
        console.log("Connection to MongoDB was successful.");
    });
}

function addFeed(url) {
    Feed.count({ _id: url }, function(error, count) {
        if(error != null) {
            console.error('Failed to check if feed exists in MongoDB: '+error);
        }
        else {
            if(count == 0) {
               requestAndParseFeed(url);
            }
            else if(count == 1) {
{'fraction_non_alphanumeric': 0.12628759785743718, 'fraction_numerical': 0.006592501030078286, 'mean_word_length': 2.0268079800498753, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '25676', 'n_tokens_mistral': 1388, 'n_tokens_neox': 1362, 'n_words': 363}