<commit_before>//feed.js
var xml2js = require('xml2js');
var request = require('request');
var _ = require('underscore');

/**
All you need to do is send a feed URL that can be opened via fs
Options are optional, see xml2js for extensive list
And a callback of course

The returned formats will be structually the same, but you should still check the 'format' property
**/
exports.parseURL = function(feedURL, options, callback){

	var parser = new xml2js.Parser({trim:false, normalize:true, mergeAttrs:true});
	var obj = request.get({uri:feedURL, jar:false, proxy:false, followRedirect:true}, function (err, response, xml) {
		if(err || xml == null){
			if(err){
				callback(err,null);
			}else{
				callback('failed to retrive source', null);
			}
		}else{

			parser.parseString(xml, function(err, jsonDOM){
				if(err){
					callback(err, null);
				}else{
					//do not have error catching right now, need to build
					if(jsonDOM){
						var err, output;
						if(isRSS(jsonDOM)){
							output = formatRSS(jsonDOM);
						}else{
							output = formatATOM(jsonDOM);
						}
						callback(null, output);
					}else{
						callback("failed to parse xml", null);
					}
				}
			});

		}
	});

	//detects if RSS, otherwise assume atom
	function isRSS(json){<commit_msg>fixed some issues with redirects by not allowing redirects<commit_after>//feed.js
var xml2js = require('xml2js');
var _ = require('underscore');

/**
All you need to do is send a feed URL that can be opened via fs
Options are optional, see xml2js for extensive list
And a callback of course

The returned formats will be structually the same, but you should still check the 'format' property
**/
exports.parseURL = function(feedURL, options, callback){

	var parser = new xml2js.Parser({trim:false, normalize:true, mergeAttrs:true});
	try{

		var request = require('request');
		var req = request({uri:feedURL, jar:false, proxy:false, followRedirect:false}, function (err, response, xml) {
			if(err || xml == null){
				if(err){
					callback(err,null);
				}else{
					callback('failed to retrive source', null);
				}
			}else{
				parser.parseString(xml, function(err, jsonDOM){
					if(err){
						callback(err, null);
					}else{
						//do not have error catching right now, need to build
						if(jsonDOM){
							var err, output;
							if(isRSS(jsonDOM)){
								output = formatRSS(jsonDOM);
							}else{
								output = formatATOM(jsonDOM);
							}
							callback(null, output);
						}else{
							callback("failed to parse xml", null);
						}
					}
				});

			}
		});
	}catch(err){
		console.log(err);
	}

	//detects if RSS, otherwise assume atom
	function isRSS(json){
{'fraction_non_alphanumeric': 0.12570356472795496, 'fraction_numerical': 0.00300187617260788, 'mean_word_length': 3.258785942492013, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4838232', 'n_tokens_mistral': 1040, 'n_tokens_neox': 817, 'n_words': 268}