<commit_before>var cheerio = require('cheerio');
var url = require('url');
var Entities = require('html-entities').AllHtmlEntities;
var entities = new Entities();
var helpers = require('./helpers');
var async = require('async');

// helper to access nested objects with string literals as object key. Example -> Object.byString(someObj, 'part3[0].name');
Object.byString = helpers.byString;

exports = module.exports =  function (feed, posts, callback) {
    var valid = true;
    var ranking = 0;
    var entries = [];

    async.eachSeries(posts,
        function(post, callback) {
            var entry = {
                origin: post.meta.link,
                host: url.parse(post.meta.link).host,
                ranking: ranking++,
                category: feed.category || []
            };

            feed.template.elements.forEach(function (element) {
                var holder;

                if (element.required) {
                    valid = false;
                }

                element.items.forEach(function (item) {
                    var active = false;
                    // get the entry element
                    if (item.scrape) {
                        var $ = cheerio.load(Object.byString(post, item.selector));
                        holder = $(item.scrape.selector).attr(item.scrape.attribute);
                    }
                    else if (item.decode) {
                        holder = entities.decode(Object.byString(post, item.selector));<commit_msg>use semi-private namespace for default parameters<commit_after>var cheerio = require('cheerio');
var url = require('url');
var Entities = require('html-entities').AllHtmlEntities;
var entities = new Entities();
var helpers = require('./helpers');
var async = require('async');

// helper to access nested objects with string literals as object key. Example -> Object.byString(someObj, 'part3[0].name');
Object.byString = helpers.byString;

exports = module.exports =  function (feed, posts, callback) {
    var valid = true;
    var ranking = 0;
    var entries = [];

    async.eachSeries(posts,
        function(post, callback) {
            var entry = {
                _source: feed.name,
                _origin: post.meta.link,
                _host: url.parse(post.meta.link).host,
                _ranking: ranking++
            };

            feed.template.elements.forEach(function (element) {
                var holder;

                if (element.required) {
                    valid = false;
                }

                element.items.forEach(function (item) {
                    var active = false;
                    // get the entry element
                    if (item.scrape) {
                        var $ = cheerio.load(Object.byString(post, item.selector));
                        holder = $(item.scrape.selector).attr(item.scrape.attribute);
                    }
                    else if (item.decode) {
                        holder = entities.decode(Object.byString(post, item.selector));
{'fraction_non_alphanumeric': 0.11495388669301712, 'fraction_numerical': 0.001976284584980237, 'mean_word_length': 1.7188898836168307, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2703926', 'n_tokens_mistral': 797, 'n_tokens_neox': 750, 'n_words': 188}