<commit_before>'use strict';

var Twitter = require('twitter');
var workersUtils = require('../utils');

var SERVER = 'https://api.mtwitter.com/bgd/jdmc/1.0/';
var CONSUMER_KEY = 'EAJ3wyvzBNBSvW3Yq9UIX65ZX';
var CONSUMER_SECRET = '<KEY>';
var ACCESS_TOKEN_KEY = '<KEY>';
var ACCESS_TOKEN_SECRET = '<KEY>';
var client = new Twitter({
  consumer_key: CONSUMER_KEY,
  consumer_secret: CONSUMER_SECRET,
  access_token_key: ACCESS_TOKEN_KEY,
  access_token_secret: ACCESS_TOKEN_SECRET,
});
var twitterJobs = {
  A_TWITTER_SYNC: twitterSyncJob,
  A_TWITTER_SIGNUP: pairTwitterFriends,
  A_TWITTER_LOGIN: pairTwitterFriends,
};

// Doc: https://dev.twitter.com/rest/reference/get/search/tweets

module.exports = twitterJobs;

function twitterSyncJob(context, event) {
  return workersUtils.getCurrentTrips(context)
  .then(function handleTwitterUsers(tripsEvents) {
    return Promise.all(tripsEvents.map(function(tripEvent) {
      return new Promise(function(resolve, reject) {
        client.get(
          'search/tweets', {
            q: '#' + tripEvent.trip.hash,
            lang: 'fr',
          },
          function twitterSearchHandler(err, tweets, response) {
            if(err) {
              reject(err);
            }
            //context.logger.debug(JSON.stringify(tweets, null, 2));
            resolve(Promise.all(tweets.statuses.map(function(status) {
              return context.db.collection('users').findOne({
                'auth.twitter.id': status.user.id + '',
              }).then(function(author) {
                if(!author) {
                  context.logger.debug('No user found for ', status.user._id)
                  return;
                }
                return context.db.collection('events').findOneAndUpdate({
                  'twitter.id': status.id,
                }, {
                  $set: {
                    twitter: {
                      id: status.id,
                      text: status.text,
                      geo: status.geo,
                    },
                    'contents.text': status.text,
                    'contents.geo': status.geo,
                    'contents.profile_image': status.profile_image_url_https,
                    'contents.entities': status.entities,
                  },
                  $setOnInsert: {
                    'contents.date': new Date(status.created_at),
                    'contents.trip_id': tripEvent._id,
                    'contents.type': 'twitter-status',
                    trip: tripEvent.trip,
                  },
                }, {
                  upsert: true,
                  returnOriginal: false,
                });<commit_msg>feat(twitter): add row content of the tweet<commit_after>'use strict';

var Twitter = require('twitter');
var workersUtils = require('../utils');

var SERVER = 'https://api.mtwitter.com/bgd/jdmc/1.0/';
var CONSUMER_KEY = 'EAJ3wyvzBNBSvW3Yq9UIX65ZX';
var CONSUMER_SECRET = '<KEY>';
var ACCESS_TOKEN_KEY = '<KEY>';
var ACCESS_TOKEN_SECRET = '<KEY>';
var client = new Twitter({
  consumer_key: CONSUMER_KEY,
  consumer_secret: CONSUMER_SECRET,
  access_token_key: ACCESS_TOKEN_KEY,
  access_token_secret: ACCESS_TOKEN_SECRET,
});
var twitterJobs = {
  A_TWITTER_SYNC: twitterSyncJob,
  A_TWITTER_SIGNUP: pairTwitterFriends,
  A_TWITTER_LOGIN: pairTwitterFriends,
};

// Doc: https://dev.twitter.com/rest/reference/get/search/tweets

module.exports = twitterJobs;

function twitterSyncJob(context, event) {
  return workersUtils.getCurrentTrips(context)
  .then(function handleTwitterUsers(tripsEvents) {
    return Promise.all(tripsEvents.map(function(tripEvent) {
      return new Promise(function(resolve, reject) {
        client.get(
          'search/tweets', {
            q: '#' + tripEvent.trip.hash,
            lang: 'fr',
          },
          function twitterSearchHandler(err, tweets, response) {
            if(err) {
              reject(err);
            }
            //context.logger.debug(JSON.stringify(tweets, null, 2));
            resolve(Promise.all(tweets.statuses.map(function(status) {
              return context.db.collection('users').findOne({
                'auth.twitter.id': status.user.id + '',
              }).then(function(author) {
                if(!author) {
                  context.logger.debug('No user found for ', status.user._id)
                  return;
                }
                return context.db.collection('events').findOneAndUpdate({
                  'twitter.id': status.id,
                }, {
                  $set: {
                    twitter: {
                      id: status.id,
                      text: status.text,
                      geo: status.geo,
                    },
                    'contents.text': status.text,
                    'contents.geo': status.geo,
                    'contents.profile_image': status.profile_image_url_https,
                    'contents.entities': status.entities,
                    'contents.raw': status,
                  },
                  $setOnInsert: {
                    'contents.date': new Date(status.created_at),
                    'contents.trip_id': tripEvent._id,
                    'contents.type': 'twitter-status',
                    trip: tripEvent.trip,
                  },
                }, {
                  upsert: true,
                  returnOriginal: false,
                });
{'fraction_non_alphanumeric': 0.11866913123844731, 'fraction_numerical': 0.0029574861367837337, 'mean_word_length': 1.7635342185903984, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6873842', 'n_tokens_mistral': 1621, 'n_tokens_neox': 1528, 'n_words': 252}