<commit_before>var url = require('url'),
    http = require('http'),
    redis = require('redis'),
    moment = require('moment'),
    twitter = require('twitter'),
    winchatty = require('./winchatty'),
    tweetScanner = require('./tweetScanner');

var redisClient, twitterClient;

function TweetServer(config) {
    if (!(this instanceof TweetServer)) return new TweetServer(config);

    this.config = config;
    redisClient = redis.createClient();
    twitterClient = new twitter(this.config.api_keys);

    redisClient.on('error', function(error) {
        console.log('(TweetServer) redis error: ' + error);
    });

    this.server = http.createServer(function(req, res) {
        var dateStamp, tweetUrl, tweetId,
            parsed = url.parse(req.url, true);

        switch(parsed.pathname) {
            case '/tweet':
                dateStamp = moment().utc().format('YYYY-MM-DD');
                redisClient.zincrby('daily-requests', 1, dateStamp);

                // confirm tweetUrl is a single, valid twitter url
                tweetUrl = tweetScanner(parsed.query.tweetUrl || '');
                tweetUrl = tweetUrl.length === 1 ? tweetUrl[0] : '';

                tweetId = tweetUrl.match(/twitter\.com\/\w+\/status\/(\d+)/i);
                if(tweetId === null || tweetId.length !== 2) {
                    outputResponse(res, 400, '400 Bad Request<br>Could not find tweetId in required argument tweetUrl');
                } else {
                    tweetId = tweetId[1];
                    redisClient.hget('tweet:' + tweetId, 'body', loadedFromCache);
                }
                break;
            default:
                console.log('(TweetServer) bad endpoint');
                outputResponse(res, 404, '404 Not Found<br>Unknown path: ' + parsed.pathname);
        }

        function loadedFromCache(error, response) {
            if(error) {
                outputResponse(res, 500, '500 Internal Server Error');
            } else if(response !== null) {
                console.log('(TweetServer) cache hit for tweet:' + tweetId);
                outputResponse(res, 200, response);
                redisClient.zincrby('daily-served', 1, dateStamp);
                redisClient.zincrby('tweet-requests', 1, 'tweet:' + tweetId);
            } else {
                console.log('(TweetServer) cache miss for tweet:' + tweetId);
                console.log('(TweetServer) validating ' + tweetUrl);
                winchatty.search(tweetId, validatedUsingSearch);
            }
        }

        function validatedUsingSearch(error, response) {
            if(error || response.posts.length === 0) {
                if(error) {
                    console.log('(TweetServer) winchatty module error: ' + JSON.stringify(error));
                }
                outputResponse(res, 400, '400 Bad Request<br>No chatty post found with this Twitter url');
            } else {
                console.log('(TweetServer) retrieving ' + tweetUrl);
                twitterClient.get('statuses/oembed.json', {url: tweetUrl}, loadedFromTwitter);
            }
        }

        function loadedFromTwitter(error, response) {
            redisClient.zincrby('daily-twitter-api', 1, dateStamp);

            if(error || !('html' in response)) {
                console.log('(TweetServer) twitter module error:' + JSON.stringify(error));
                outputResponse(res, 500, '500 Internal Server Error');
            } else {
                var tweetBody = response.html;
                outputResponse(res, 200, tweetBody);

                console.log('(TweetServer) caching tweet:' + tweetId);
                redisClient.hmset('tweet:' + tweetId, 'body', tweetBody, 'url', tweetUrl, 'saved', moment().utc().format());
                redisClient.zincrby('daily-served', 1, dateStamp);
                redisClient.zincrby('tweet-requests', 1, 'tweet:' + tweetId);
            }
        }

        function outputResponse(res, httpCode, body) {
            res.writeHead(httpCode, {'Content-Type': 'text/html'});
            res.end(body);
        }
    });
}

TweetServer.prototype.start = function() {
    var port = this.config.port;
    this.server.listen(port, function() {
        console.log(moment().utc().format() + ' TweetServer is listening on port ' + port);
    });
};

module.exports = TweetServer;<commit_msg>more info on random requests<commit_after>var url = require('url'),
    http = require('http'),
    redis = require('redis'),
    moment = require('moment'),
    twitter = require('twitter'),
    winchatty = require('./winchatty'),
    tweetScanner = require('./tweetScanner');

var redisClient, twitterClient;

function TweetServer(config) {
    if (!(this instanceof TweetServer)) return new TweetServer(config);

    this.config = config;
    redisClient = redis.createClient();
    twitterClient = new twitter(this.config.api_keys);

    redisClient.on('error', function(error) {
        console.log('(TweetServer) redis error: ' + error);
    });

    this.server = http.createServer(function(req, res) {
        var dateStamp, tweetUrl, tweetId,
            parsed = url.parse(req.url, true);

        switch(parsed.pathname) {
            case '/tweet':
                dateStamp = moment().utc().format('YYYY-MM-DD');
                redisClient.zincrby('daily-requests', 1, dateStamp);

                // confirm tweetUrl is a single, valid twitter url
                tweetUrl = tweetScanner(parsed.query.tweetUrl || '');
                tweetUrl = tweetUrl.length === 1 ? tweetUrl[0] : '';

                tweetId = tweetUrl.match(/twitter\.com\/\w+\/status\/(\d+)/i);
                if(tweetId === null || tweetId.length !== 2) {
                    outputResponse(res, 400, '400 Bad Request<br>Could not find tweetId in required argument tweetUrl');
                } else {
                    tweetId = tweetId[1];
                    redisClient.hget('tweet:' + tweetId, 'body', loadedFromCache);
                }
                break;
            default:
                console.log('(TweetServer) bad endpoint: ' + parsed.pathname);
                outputResponse(res, 404, '404 Not Found<br>Unknown path: ' + parsed.pathname);
        }

        function loadedFromCache(error, response) {
            if(error) {
                outputResponse(res, 500, '500 Internal Server Error');
            } else if(response !== null) {
                console.log('(TweetServer) cache hit for tweet:' + tweetId);
                outputResponse(res, 200, response);
                redisClient.zincrby('daily-served', 1, dateStamp);
                redisClient.zincrby('tweet-requests', 1, 'tweet:' + tweetId);
            } else {
                console.log('(TweetServer) cache miss for tweet:' + tweetId);
                console.log('(TweetServer) validating ' + tweetUrl);
                winchatty.search(tweetId, validatedUsingSearch);
            }
        }

        function validatedUsingSearch(error, response) {
            if(error || response.posts.length === 0) {
                if(error) {
                    console.log('(TweetServer) winchatty module error: ' + JSON.stringify(error));
                }
                outputResponse(res, 400, '400 Bad Request<br>No chatty post found with this Twitter url');
            } else {
                console.log('(TweetServer) retrieving ' + tweetUrl);
                twitterClient.get('statuses/oembed.json', {url: tweetUrl}, loadedFromTwitter);
            }
        }

        function loadedFromTwitter(error, response) {
            redisClient.zincrby('daily-twitter-api', 1, dateStamp);

            if(error || !('html' in response)) {
                console.log('(TweetServer) twitter module error:' + JSON.stringify(error));
                outputResponse(res, 500, '500 Internal Server Error');
            } else {
                var tweetBody = response.html;
                outputResponse(res, 200, tweetBody);

                console.log('(TweetServer) caching tweet:' + tweetId);
                redisClient.hmset('tweet:' + tweetId, 'body', tweetBody, 'url', tweetUrl, 'saved', moment().utc().format());
                redisClient.zincrby('daily-served', 1, dateStamp);
                redisClient.zincrby('tweet-requests', 1, 'tweet:' + tweetId);
            }
        }

        function outputResponse(res, httpCode, body) {
            res.writeHead(httpCode, {'Content-Type': 'text/html'});
            res.end(body);
        }
    });
}

TweetServer.prototype.start = function() {
    var port = this.config.port;
    this.server.listen(port, function() {
        console.log(moment().utc().format() + ' TweetServer is listening on port ' + port);
    });
};

module.exports = TweetServer;
{'fraction_non_alphanumeric': 0.13530149321782742, 'fraction_numerical': 0.010714692807477487, 'mean_word_length': 2.068905211612452, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4509188', 'n_tokens_mistral': 2660, 'n_tokens_neox': 2396, 'n_words': 510}