<commit_before>/**
 * Copyright (c) 2012, <NAME>. 
 * Licensed under the Affero General Public License version 3 or later. 
 * See LICENSE for details.  
 */
var superfeedr = require('./superfeedr-client')
    , FeedLoader = require('./feed').FeedLoader;    
    
var Connector = function(db, renderer) {
    this.db = db;
    this.list_limit = 25;
    this.renderer = renderer
}

Connector.prototype.remove_doc = function(socket, doc) {
    console.log('removing '+doc._id+' rev:'+doc._rev);
    this.db.remove(doc._id, doc._rev, function(err, docs) {
        var res = !res;
        return socket.emit('removeDocResult', {success: res, doc:doc});                        
    });       
}   

Connector.prototype.remove_docs = function(socket, data) {
    var max = data.docs.length,
        bulk= JSON.stringify({'docs':data.docs})
        res = true;    
    console.log('removing '+max+' docs');        
    this.db.save(data.docs, function() {
        return socket.emit('removeDocsResult', {success: true, deleted: data.docs, all:data.all});                                 
    });
}    

Connector.prototype.get_index_docs = function(res, startkey) {
    var limit = this.list_limit,
        start = (startkey) ? JSON.parse(startkey) : false,
        renderFunc = this.renderer.renderList,
        query = {
            descending: true,
            limit: limit + 1
        };     
    if(start !== false) {
        query.startkey=start[0];
        query.startkey_docid=start[1];
    }
    this.db.view('sources/all', query, function(err, docs) {  
        if (err) throw new Error(JSON.stringify(err));        
        renderFunc(docs, true, limit, (startkey !== false), res);                     
    });    
}

Connector.prototype.get_source_docs = function(res, skey, startkey) { 
    var limit = this.list_limit,
        start = (startkey) ? JSON.parse(startkey) : false,
        renderFunc = this.renderer.renderList,
        query = {            
            descending: true,
            startkey: [skey,((start) ? start[0]: {})],
            endkey: [skey],
            limit: limit + 1,
            startkey_docid: ((start) ? start[1] : '')
        };
    this.db.view('sources/by_feed_link', query, function(err, docs) {
        if (err) throw new Error(JSON.stringify(err));
        renderFunc(docs, false, limit, (startkey !== false), res);                                      
    });
}

Connector.prototype.clear_source = function(res, skey) { 
    var limit = this.list_limit,
        query = {key: skey},
        db = this.db;
    this.db.view('sources/docids', query, function(err, docs) {
        if (err) throw new Error(JSON.stringify(err));
        var bulk = []
        docs.forEach(function(doc) {
            bulk.push({_id: doc[0], _rev: doc[1], _deleted:true});
        });
        db.save(bulk, function (err, db_res) {         
            return res.json({success: true, skey:skey, sum:bulk.length});                              
        });
    });
}

Connector.prototype.get_toc = function(res, skey) {   
    var query = {
        group: true
    }
    if(skey) {
        query.startkey = [skey,null];
        query.endkey = [skey,{}];
    }
    this.db.view('sources/toc', query, function(err, docs) {        
        if (err) throw new Error(JSON.stringify(err));
        var rows = docs.rows,
            success = !err;       

        function compare(a,b) {
            if (a.value < b.value) return 1;
            if (a.value > b.value) return -1;
            return 0;
        }
        
        if(!skey) rows = rows.sort(compare);
                 
        return res.json({success: success, rows: rows});                      
    });        
}    

Connector.prototype.add_sub = function(req, res, superfeedrClient){
    var db = this.db;
    function isUrl(s) {
        var regexp = /(ftp|http|https):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?/
        return regexp.test(s);
    }
    
    function isNumber(n) {
        return !isNaN(parseFloat(n)) && isFinite(n);
    }

    if(isUrl(req.body.feed_url ) && isNumber(req.body.step)) {
        var new_url = req.body.feed_url,
            handler = (req.body.handler !== undefined) ? req.body.handler : false,  
            feed = new FeedLoader(new_url, function(err, feed) {
            if (err) {
                console.log(new_url+' '+err)
                res.json({success:false, valid: false}); 
            } else {
                if(feed.valid === true) {
                    var response = {
                            success:true, 
                            step: parseInt(req.body.step), 
                            valid: feed.valid, 
                            pubsubhub: ((superfeedrClient) ? feed.pubSubHub: false), 
                            title: feed.title, 
                            handler: handler,
                            saved: false
                        },
                        doc = {
                            type: "subscription", 
                            url: feed.url, 
                            title: feed.title, 
                            skey: feed.skey,
                            creates: ((response.handler=='superfeedr') ? 'xmpp' : 'newsitem'), 
                            next_check: new Date().getTime(),
                            links: []
                        };
                    if(response.step == 1 && superfeedrClient) {
                        res.json(response); 
                    } else if(response.step == 2 || superfeedrClient === false) {                
                        console.log('save subscription');           
                        response.saved = true;
                        if(response.handler == 'superfeedr') {
                            superfeedrClient.subscribe(doc.url, function(notification) {
                                db.save('sub-'+feed.skey, doc, function(couch_err, couch_res) {
                                    res.json(response); <commit_msg>add pubsub hub to new subs when available<commit_after>/**
 * Copyright (c) 2012, <NAME>. 
 * Licensed under the Affero General Public License version 3 or later. 
 * See LICENSE for details.  
 */
var superfeedr = require('./superfeedr-client')
    , FeedLoader = require('./feed').FeedLoader;    
    
var Connector = function(db, renderer) {
    this.db = db;
    this.list_limit = 25;
    this.renderer = renderer
}

Connector.prototype.remove_doc = function(socket, doc) {
    console.log('removing '+doc._id+' rev:'+doc._rev);
    this.db.remove(doc._id, doc._rev, function(err, docs) {
        var res = !res;
        return socket.emit('removeDocResult', {success: res, doc:doc});                        
    });       
}   

Connector.prototype.remove_docs = function(socket, data) {
    var max = data.docs.length,
        bulk= JSON.stringify({'docs':data.docs})
        res = true;    
    console.log('removing '+max+' docs');        
    this.db.save(data.docs, function() {
        return socket.emit('removeDocsResult', {success: true, deleted: data.docs, all:data.all});                                 
    });
}    

Connector.prototype.get_index_docs = function(res, startkey) {
    var limit = this.list_limit,
        start = (startkey) ? JSON.parse(startkey) : false,
        renderFunc = this.renderer.renderList,
        query = {
            descending: true,
            limit: limit + 1
        };     
    if(start !== false) {
        query.startkey=start[0];
        query.startkey_docid=start[1];
    }
    this.db.view('sources/all', query, function(err, docs) {  
        if (err) throw new Error(JSON.stringify(err));        
        renderFunc(docs, true, limit, (startkey !== false), res);                     
    });    
}

Connector.prototype.get_source_docs = function(res, skey, startkey) { 
    var limit = this.list_limit,
        start = (startkey) ? JSON.parse(startkey) : false,
        renderFunc = this.renderer.renderList,
        query = {            
            descending: true,
            startkey: [skey,((start) ? start[0]: {})],
            endkey: [skey],
            limit: limit + 1,
            startkey_docid: ((start) ? start[1] : '')
        };
    this.db.view('sources/by_feed_link', query, function(err, docs) {
        if (err) throw new Error(JSON.stringify(err));
        renderFunc(docs, false, limit, (startkey !== false), res);                                      
    });
}

Connector.prototype.clear_source = function(res, skey) { 
    var limit = this.list_limit,
        query = {key: skey},
        db = this.db;
    this.db.view('sources/docids', query, function(err, docs) {
        if (err) throw new Error(JSON.stringify(err));
        var bulk = []
        docs.forEach(function(doc) {
            bulk.push({_id: doc[0], _rev: doc[1], _deleted:true});
        });
        db.save(bulk, function (err, db_res) {         
            return res.json({success: true, skey:skey, sum:bulk.length});                              
        });
    });
}

Connector.prototype.get_toc = function(res, skey) {   
    var query = {
        group: true
    }
    if(skey) {
        query.startkey = [skey,null];
        query.endkey = [skey,{}];
    }
    this.db.view('sources/toc', query, function(err, docs) {        
        if (err) throw new Error(JSON.stringify(err));
        var rows = docs.rows,
            success = !err;       

        function compare(a,b) {
            if (a.value < b.value) return 1;
            if (a.value > b.value) return -1;
            return 0;
        }
        
        if(!skey) rows = rows.sort(compare);
                 
        return res.json({success: success, rows: rows});                      
    });        
}    

Connector.prototype.add_sub = function(req, res, superfeedrClient){
    var db = this.db;
    function isUrl(s) {
        var regexp = /(ftp|http|https):\/\/(\w+:{0,1}\w*@)?(\S+)(:[0-9]+)?(\/|\/([\w#!:.?+=&%@!\-\/]))?/
        return regexp.test(s);
    }
    
    function isNumber(n) {
        return !isNaN(parseFloat(n)) && isFinite(n);
    }

    if(isUrl(req.body.feed_url ) && isNumber(req.body.step)) {
        var new_url = req.body.feed_url,
            handler = (req.body.handler !== undefined) ? req.body.handler : false,  
            feed = new FeedLoader(new_url, function(err, feed) {
            if (err) {
                console.log(new_url+' '+err)
                res.json({success:false, valid: false}); 
            } else {
                if(feed.valid === true) {
                    var response = {
                            success:true, 
                            step: parseInt(req.body.step), 
                            valid: feed.valid, 
                            pubsubhub: ((superfeedrClient) ? feed.pubSubHub: false), 
                            title: feed.title, 
                            handler: handler,
                            saved: false
                        },
                        doc = {
                            type: "subscription", 
                            url: feed.url, 
                            title: feed.title, 
                            skey: feed.skey,
                            creates: ((response.handler=='superfeedr') ? 'xmpp' : 'newsitem'), 
                            next_check: new Date().getTime(),
                            links: [],
                            pubsub: null
                        };
                    // add push hub    
                    if(feed.pubSubHub) {
                        doc.pubsub = {hub: feed.pubSubHub}
                    }
                    if(response.step == 1 && superfeedrClient) {
                        res.json(response); 
                    } else if(response.step == 2 || superfeedrClient === false) {                
                        console.log('save subscription');           
                        response.saved = true;
                        if(response.handler == 'superfeedr') {
                            superfeedrClient.subscribe(doc.url, function(notification) {
                                db.save('sub-'+feed.skey, doc, function(couch_err, couch_res) {
                                    res.json(response); 
{'fraction_non_alphanumeric': 0.13692709182841298, 'fraction_numerical': 0.00391453270265862, 'mean_word_length': 1.4644292604501608, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5676208', 'n_tokens_mistral': 3596, 'n_tokens_neox': 3489, 'n_words': 694}