<commit_before>'use strict';

const express = require('express');
const bodyParser = require('body-parser');
const request = require('request');
const app = express();
const fs = require('fs');
const https = require('https');
const http = require('http');
const path = require('path');
const nodemailer = require('nodemailer');
var sesTransport = require('nodemailer-ses-transport');
const log = require('winston');
var MongoClient = require('mongodb').MongoClient
    , assert = require('assert');

log.level = 'debug';
log.add(log.transports.File, {filename: 'logfile.log'});

const configFile = '../config/config.json';

var configuration = JSON.parse(
    fs.readFileSync(configFile)
);

var longToken = configuration.longToken;

app.use(bodyParser.urlencoded({extended: false}));
app.use(bodyParser.json());


// for Facebook verification
/**@prod*/
app.get('/webhook/', function (req, res) {
    if (req.query['hub.verify_token'] === configuration.webhookVerification) {
        res.send(req.query['hub.challenge'])
    } else {
        res.send('Error, wrong token')
    }
});


/**@prod, catches ping leads*/
app.post('/webhook/', function (req, res) {

    var leadId = req.body.entry[0].changes[0].value.leadgen_id;
    log.info('Loaded lead ' + leadId);

    if (leadId) {
        loadLeadDetails(leadId, function (data) {
            if (data) {
                var email = '';
                for (var i in data.field_data) {
                    if (data.field_data[i].name === 'email') {
                        email = data.field_data[i].values[0];
                        break;
                    }
                }
                log.info('Lead details: id ' + leadId + ', email: ' + email + ' at ' + new Date().getTime());

                db.collection('lead').save(data, (err, result) => {
                    if (err) return console.log(err);
                    console.log('saved to database');
                });

                sendMail(email);
            }
        });
    }
});

/**
 * test
 */
app.get('/parse', function (req, res) {
    loadLeadDetails('leadid', function (data) {
        if (data) {
            var email = '';
            for (var element in data.field_data) {
                if (data.field_data[element].name === 'email') {
                    email = data.field_data[element].values[0];
                }
            }
            log.info('Email from lead: ', email);
            sendMail(email);
        }
    });
});

app.post('/callback/', function (req, res) {
    var token = req.body.access_token;
    log.info('Getting long-lived token at ' + new Date().getTime());

    //TODO: move to separate function
    var uri = 'https://graph.facebook.com/v2.8/oauth/access_token?grant_type=fb_exchange_token&client_id=' + configuration.fbAuth.clientId
        + '&client_secret=' + configuration.fbAuth.secret + '&fb_exchange_token=' + token;

    var options = {
        url: uri
    };

    request(options, function (error, response, body) {
        if (!error && response.status == 200) {
            log.warn('Long-term access token retrieved: ' + body + ' at ' + new Date().getTime());
        }
    })
});

/**
 * prod
 * @param leadId
 * @param callback
 */
function loadLeadDetails(leadId, callback) {
    log.info('Loading details... for ' + leadId);
    var url = buildUrl(leadId);

    var options = {
        url: url,
        encoding: 'utf-8'
    };

    request(options, function (error, response, body) {
        if (!error && response.statusCode == 200) {
            var data = JSON.parse(body);
            if (data.error) {
                if (data.error.message.includes('Error validating access token')) {
                    log.warn('Session has expired, refreshing token on ' + new Date().getTime());

                    var mailOptions = {
                        from: configuration.smtp.service,
                        to: configuration.smtp.admin, //temporary for test mode
                        subject: 'Session has expired',
                        text: 'Error token at ' + new Date().getTime() + configuration.tokenErrorMail
                    };

                    sendMail(configuration.smtp.admin, mailOptions);

                } else {
                    log.error(data.error.message);
                }
            } else {
                callback(data)
            }
        }
    });
}


//TODO: refactor
function buildUrl(leadId) {
    return 'https://graph.facebook.com/v2.8/' + leadId + '?access_token=' + longToken +
        '&format=json&method=get&pretty=0&suppress_http_code=1';
}

function sendMail(email, options) {
    log.info('sending mail to ' + email);

    var transporter = nodemailer.createTransport(sesTransport({
        accessKeyId: configuration.smtp.accessKey,
        secretAccessKey: configuration.smtp.secret,
        rateLimit: 5, // do not send more than 5 messages in a second
        region: configuration.smtp.region,
        host: configuration.smtp.host
    }));

    var mailOptions = {
        from: configuration.smtp.sender,
        to: configuration.smtp.sender, //temporary for test mode
        subject: 'New lead',
        text: 'With email ' + email
    };

    if (!options) {
        options = mailOptions;
    }

// send mail with defined transport object
    transporter.sendMail(options, function (error, info) {
        if (error) {
            return log.error(error);
        }
        log.info('Message sent: to ' + email + ' ' + info.response);
    });
}

/**@prod*/
app.get('/platform', function (req, res) {
    res.sendFile(path.resolve('../view/platform.html'));
});


var insertDocuments = function(db, callback) {
    // Get the documents collection
    var collection = db.collection('documents');
    // Insert some documents
    collection.insertMany([
        {a : 1}, {a : 2}, {a : 3}
    ], function(err, result) {
        assert.equal(err, null);
        assert.equal(3, result.result.n);
        assert.equal(3, result.ops.length);
        console.log("Inserted 3 documents into the collection");
        callback(result);
    });
}

// Connection URL
var url = 'mongodb://localhost:27017/fbhandler';

// Use connect method to connect to the server
MongoClient.connect(url, function(err, db) {
    assert.equal(null, err);
    console.log("Connected successfully to server");
    insertDocuments(db, function() {
        db.close();
    });
});


// var options = {
//     key: fs.readFileSync('/etc/letsencrypt/live/willingbot.online/privkey.pem'),
//     cert: fs.readFileSync('/etc/letsencrypt/live/willingbot.online/fullchain.pem')
// };
// https.createServer(options, app).listen(8443);

//uncomment for localhost
http.createServer(app).listen(8000);<commit_msg>save and find leads implemented<commit_after>'use strict';

const express = require('express');
const bodyParser = require('body-parser');
const request = require('request');
const app = express();
const fs = require('fs');
const https = require('https');
const http = require('http');
const path = require('path');
const nodemailer = require('nodemailer');
var sesTransport = require('nodemailer-ses-transport');
const log = require('winston');
var MongoClient = require('mongodb').MongoClient
    , assert = require('assert');
var database = '';

log.level = 'debug';
log.add(log.transports.File, {filename: 'logfile.log'});

const configFile = '../config/config.json';

var configuration = JSON.parse(
    fs.readFileSync(configFile)
);

var longToken = configuration.longToken;

app.use(bodyParser.urlencoded({extended: false}));
app.use(bodyParser.json());


// for Facebook verification
/**@prod*/
app.get('/webhook/', function (req, res) {
    if (req.query['hub.verify_token'] === configuration.webhookVerification) {
        res.send(req.query['hub.challenge'])
    } else {
        res.send('Error, wrong token')
    }
});


/**@prod, catches ping leads*/
app.post('/webhook/', function (req, res) {

    var leadId = req.body.entry[0].changes[0].value.leadgen_id;
    log.info('Loaded lead ' + leadId);

    if (leadId) {
        loadLeadDetails(leadId, function (data) {
            if (data) {
                var email = '';
                for (var element in data.field_data) {
                    if (data.field_data[element].name === 'email') {
                        email = data.field_data[element].values[0];
                    }
                }
                insertLeads(database, data, function (result) {
                    if (result) {
                        sendMail(email);
                    }
                });
            }
        });
    }
});

/**
 * test
 */
app.get('/parse', function (req, res) {
    loadLeadDetails('someLeadId', function (data) {
        if (data) {
            var email = '';
            for (var element in data.field_data) {
                if (data.field_data[element].name === 'email') {
                    email = data.field_data[element].values[0];
                }
            }
            log.info('Loaded lead: ', email);

            insertLeads(database, data, function (result) {
                if (result) {
                    sendMail(email);
                }
            });

        }
    });
});

app.post('/callback/', function (req, res) {
    var token = req.body.access_token;
    log.info('Getting long-lived token at ' + new Date().getTime());

    //TODO: move to separate function
    var uri = 'https://graph.facebook.com/v2.8/oauth/access_token?grant_type=fb_exchange_token&client_id=' + configuration.fbAuth.clientId
        + '&client_secret=' + configuration.fbAuth.secret + '&fb_exchange_token=' + token;

    var options = {
        url: uri
    };

    request(options, function (error, response, body) {
        if (!error && response.status == 200) {
            log.warn('Long-term access token retrieved: ' + body + ' at ' + new Date().getTime());
        }
    })
});

/**
 * prod
 * @param leadId
 * @param callback
 */
function loadLeadDetails(leadId, callback) {
    log.info('Loading details... for ' + leadId);
    var url = buildUrl(leadId);

    var options = {
        url: url,
        encoding: 'utf-8'
    };

    request(options, function (error, response, body) {
        if (!error && response.statusCode == 200) {
            var data = JSON.parse(body);
            if (data.error) {
                if (data.error.message.includes('Error validating access token')) {
                    log.warn('Session has expired, refreshing token on ' + new Date().getTime());

                    var mailOptions = {
                        from: configuration.smtp.service,
                        to: configuration.smtp.admin, //temporary for test mode
                        subject: 'Session has expired',
                        text: 'Error token at ' + new Date().getTime() + configuration.tokenErrorMail
                    };
                    sendMail(configuration.smtp.admin, mailOptions);

                } else {
                    log.error(data.error.message);
                }
            } else {
                callback(data)
            }
        }
    });
}


//TODO: refactor
function buildUrl(leadId) {
    return 'https://graph.facebook.com/v2.8/' + leadId + '?access_token=' + longToken +
        '&format=json&method=get&pretty=0&suppress_http_code=1';
}

function sendMail(email, options) {
    log.info('sending mail to ' + email);

    var transporter = nodemailer.createTransport(sesTransport({
        accessKeyId: configuration.smtp.accessKey,
        secretAccessKey: configuration.smtp.secret,
        rateLimit: 5, // do not send more than 5 messages in a second
        region: configuration.smtp.region,
        host: configuration.smtp.host
    }));

    var mailOptions = {
        from: configuration.smtp.sender,
        to: configuration.smtp.sender, //temporary for test mode
        subject: 'New lead',
        text: 'With email ' + email
    };

    if (!options) {
        options = mailOptions;
    }

// send mail with defined transport object
    transporter.sendMail(options, function (error, info) {
        if (error) {
            return log.error(error);
        }
        log.info('Message sent: to ' + email + ' ' + info.response);
    });
}

/**@prod Required to subscribe fo page RTUs */
app.get('/platform', function (req, res) {
    res.sendFile(path.resolve('../view/platform.html'));
});

var insertLeads = function (db, data, callback) {
    var collection = db.collection('leads');
    collection.findOne({'id': data.id}, function (err, lead) {
        if (lead) {
            log.info(data.id + ' Lead exists');
            callback(null)
        } else {
            collection.insert(data, function (err, result) {
                assert.equal(err, null);
                console.log('Lead inserted into document');
                callback(result);
            })
        }
    })
};

// Connection URL
var url = 'mongodb://localhost:27017/fbhandler';

// Use connect method to connect to the server
MongoClient.connect(url, function (err, db) {
    assert.equal(null, err);
    log.info("Connected successfully to server");
    database = db;
    var options = {
        key: fs.readFileSync('/etc/letsencrypt/live/willingbot.online/privkey.pem'),
        cert: fs.readFileSync('/etc/letsencrypt/live/willingbot.online/fullchain.pem')
    };
    https.createServer(options, app).listen(8443);

//uncomment for localhost
//     http.createServer(app).listen(8000);
});



{'fraction_non_alphanumeric': 0.14539532129108676, 'fraction_numerical': 0.0051821143026354755, 'mean_word_length': 2.1838321942022154, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2836919', 'n_tokens_mistral': 4025, 'n_tokens_neox': 3810, 'n_words': 870}