<commit_before>/* Links that are essential :
 * 1. For prev page = http://www.engadget.com/page/n-1/
 * 2. For curr page = http://www.engadget.com/
 * 3. Example of the url : http://www.engadget.com/2015/06/04/apple-watch-reader-review-roundup/
 */
var e = require('express')
var c = require('cheerio')
var fs = require('fs')
var req = require('request')
var post = require('request')
var store = new Array();
var app = e();
var router = e.Router();
var port = process.env.PORT || 8100;

app.use(e.static('assets'));

console.log("Express Serving is starting")
console.log("Starting Scraping")

router.get('/', function(req, res) {
        var data = fs.readFileSync('index.html', 'utf8');
        res.send(data.toString());
});

app.use('/', router);
app.listen(port, function() {
        console.log("Listening on port " + port);
});

req('http://www.reddit.com/r/technology', function(err, res, html) {
                if(!err && res.statusCode == 200) {
                        var l = c.load(html);

                        // Info from Reddit (r/technology)
                        l('div.entry').each(function(i, element) {
                                var content = l(this).children().children().next('.title');
                                var data = content.text();
                                console.log(data);
                        });
                }
                });

req('http://www.gizmodo.in/gadgets', function(err,res, html) {
                if(!err && res.statusCode == 200) {
                        var l = c.load(html);

                        // Headlines from the Gadgets page of Gizmodo
                        l('article.article').each(function (i, element) {
                                var content = l(this).children().children().children('h2');
                                var data = content.text();
                                console.log(data);
                        });
                }
                });

req('http://www.engadget.com/', function(err, res, html) {
                if(!err && res.statusCode == 200) {
                        var l = c.load(html);

                        // Gives all the headlines together
                        l('header.post-header').each(function(i, element) {
                                var content = l(this).children().children().children('.h2');
                                var data = content.text();
                                console.log(data);
                                store.push(data);
                               // console.log(store);
                        });
                        /*
                        // Gives the top 5 current posts on the page
                        l('div').attr('id','carousel').each(function(i, element) {
                                var content = l(this).children().children('.always');
                                console.log(content.text());
                        });

                        // Gives the top 2 current posts on the page
                        l('li.top2').each(function(i, element) {
                                var content = l(this);
                                console.log(content.text());
                       });

                        // Gives the last current post on the page
                        l('li.last').each(function(i, element) {
                                var content = l(this);
                                console.log(content.text());
                       });*/
                }
                });

post('http://localhost:8100/', function(err, res, html) {
        if(!err && res.statusCode == 200) {
                var p = c.load(html);
                p('h3.panel-title').each(function(i, element) {
                        var content = p(this);
                        console.log(content.text());
                });
        }
});


<commit_msg>Refactoring the code, separating the scraper from the ui router<commit_after>/* Links that are essential :
 * 1. For prev page = http://www.engadget.com/page/n-1/
 * 2. For curr page = http://www.engadget.com/
 * 3. Example of the url : http://www.engadget.com/2015/06/04/apple-watch-reader-review-roundup/
 */
var http = require("http"),
    e = require("express"),
    c = require("cheerio"),
    fs = require("fs"),
    req = require("request");

module.exports.router = function() {
    var port = process.env.PORT || 8100,
        app = e(),
        api = e(),
        appr = e.Router(),
        apir = e.Router();

    app.set('title', "Tribble | Content Scraping");
    app.use(e.static('assets'));

    // The APP router
    appr.get('/', function(req, res) {
        var data = fs.readFileSync("index.html", "UTF-8");
        res.send(data.toString());
    });

    appr.get('/:somevalue', function(req, res) {
        var data = req.params.somevalue;
        res.send("Sorry cannot GET/ " + data + " on this page");
    });

    appr.post('/:somevalue', function(req, res) {
        var data = req.params.somevalue;
        res.send("Sorry cannot make a POST/ " + data + " to this page");
    });

    // The API router
    apir.get('/', function(req, res) {
        res.send("The API powering tribble");
    });

    apir.get('/post/:value', function(req, res) {
        var data = req.params.value;
        res.json({
            request     : 'you have requested for ' + data,
            site        : 'yet to come',
            description : 'yet to come',
            timestamp   : 'yet to come' 
        })
    });

    app.use('/', appr);
    app.use('/api', apir);

    htpp.createServer(app).listen(port, function() {
       console.log("Serving at http://localhost:", port); 
    });
}

req('http://www.reddit.com/r/technology', function(err, res, html) {
                if(!err && res.statusCode == 200) {
                        var l = c.load(html);

                        // Info from Reddit (r/technology)
                        l('div.entry').each(function(i, element) {
                                var content = l(this).children().children().next('.title');
                                var data = content.text();
                                console.log(data);
                        });
                }
                });

req('http://www.gizmodo.in/gadgets', function(err,res, html) {
                if(!err && res.statusCode == 200) {
                        var l = c.load(html);

                        // Headlines from the Gadgets page of Gizmodo
                        l('article.article').each(function (i, element) {
                                var content = l(this).children().children().children('h2');
                                var data = content.text();
                                console.log(data);
                        });
                }
                });

req('http://www.engadget.com/', function(err, res, html) {
                if(!err && res.statusCode == 200) {
                        var l = c.load(html);

                        // Gives all the headlines together
                        l('header.post-header').each(function(i, element) {
                                var content = l(this).children().children().children('.h2');
                                var data = content.text();
                                console.log(data);
                                store.push(data);
                               // console.log(store);
                        });
                        /*
                        // Gives the top 5 current posts on the page
                        l('div').attr('id','carousel').each(function(i, element) {
                                var content = l(this).children().children('.always');
                                console.log(content.text());
                        });

                        // Gives the top 2 current posts on the page
                        l('li.top2').each(function(i, element) {
                                var content = l(this);
                                console.log(content.text());
                       });

                        // Gives the last current post on the page
                        l('li.last').each(function(i, element) {
                                var content = l(this);
                                console.log(content.text());
                       });*/
                }
                });


{'fraction_non_alphanumeric': 0.14414521295527347, 'fraction_numerical': 0.008304662474789417, 'mean_word_length': 1.1958843448814795, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 12, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3235424', 'n_tokens_mistral': 2286, 'n_tokens_neox': 2110, 'n_words': 473}