<commit_before>'use strict';

const Bluebird = require('bluebird');
const Nodegram = require('nodegram');

const dbg = require('debug')('anamorph:instagramDataFetcher');

module.exports = class InstagramDataFetcher {
  constructor(tokens) {
    this.tokens = tokens;

    this.data = {
      numberOfUserPublications: '',
      numberOfUserFollowers: [],
      numberOfUserFollows: [],
      averageOfGetLikes: [],
      averageOfGetComments: [],
      averageOfTagsForPostPublication: [],
      frequency: [],
    };
    this.datePosts = [];
  }

  fetch() {
    dbg('Fetching instagram data');

    this.gram = new Nodegram({ accessToken: this.tokens.accessToken });

    this.gram.getAuthUrl();
    return this.fetchNumberOfUserPublications()
      .then(() => this.fetchNumberOfUserFollowers())
      .then(() => this.fetchNumberOfUserFollows())
      .then(() => this.fetchAverageOfGetLikes())
      .then(() => this.fetchAverageOfGetComments())
      .then(() => this.fetchAverageTagsForPostPublication())
      .then(() => this.fetchPostsFrequency())
      .then(() => this.data)
      .catch(err => dbg(`Error: ${err.message}`));
  }

  fetchNumberOfUserPublications() {
    dbg('Retrieving number of publications');

    return this.gram.get('/users/self/', {})
      .then(res => {
        this.data.numberOfUserPublications = res.counts.media;
          dbg(`Number of user publications: ${this.data.numberOfUserPublications}`);
      })
      .catch(err => dbg(`Error: ${err.message}`));
  }

  fetchNumberOfUserFollowers() {
    dbg('Retrieving number of followers');

    return this.gram.get('/users/self/', {})
      .then(res => {
        this.data.numberOfUserFollowers = res.counts.followed_by;
          dbg(`Number of user followers: ${this.data.numberOfUserFollowers}`);
      })
      .catch(err => dbg(`Error: ${err.message}`));
  }

  fetchNumberOfUserFollows() {
    dbg('Retrieving number of follows');

    return this.gram.get('/users/self/', {})
      .then(res => {
        this.data.numberOfUserFollows = res.counts.follows;
          dbg(`Number of user follows: ${this.data.numberOfUserFollows}`);
      })
      .catch(err => dbg(`Error: ${err.message}`));
  }

  fetchAverageOfGetLikes() {
    dbg('Retrieving average of likes per publication');
    let numberOfPublications = 0;
    let numberOfLikes = 0;

    return this.gram.get('/users/self/media/recent', {})
      .then(res => {
        numberOfPublications += res.length;
        res.forEach(res => numberOfLikes += res.likes.count);

        this.data.averageOfGetLikes = Math.round(numberOfLikes / numberOfPublications);
          dbg(`Average of likes: ${this.data.averageOfGetLikes}`);
      })
      .catch(err => dbg(`Error: ${err.message}`));
  }

  fetchAverageOfGetComments() {
    dbg('Retrieving average of comments per publication');

    let numberOfPublications = 0;
    let numberOfComments = 0;

    return this.gram.get('/users/self/media/recent', {})
      .then(res => {
        numberOfPublications += res.length;
        res.forEach(res => numberOfComments += res.comments.count);

        this.data.averageOfGetComments = Math.round(numberOfComments / numberOfPublications);
          dbg(`Average of comments: ${this.data.averageOfGetComments}`);
      })
      .catch(err => dbg(`Error: ${err.message}`));
  }

  fetchAverageTagsForPostPublication() {
    dbg('Retrieving average of tags used per publication');
    let numberOfPublications = 0;
    let numberOfTags = 0;

    return this.gram.get('/users/self/media/recent', {})
      .then(res => {
        numberOfPublications += res.length;
        res.forEach(res => numberOfTags += res.tags.length);

        this.data.averageOfTagsForPostPublication = Math.round(numberOfTags / numberOfPublications);
          dbg(`Average of tags for post publication: ${this.data.averageOfTagsForPostPublication}`);
      })
      .catch(err => dbg(`Error: ${err.message}`));
  }

<commit_msg>[Instagram] - Added most used hashtags<commit_after>'use strict';

const Bluebird = require('bluebird');
const Nodegram = require('nodegram');

const dbg = require('debug')('anamorph:instagramDataFetcher');

module.exports = class InstagramDataFetcher {
  constructor(tokens) {
    this.tokens = tokens;

    this.data = {
      numberOfUserPublications: '',
      numberOfUserFollowers: [],
      numberOfUserFollows: [],
      averageOfGetLikes: [],
      averageOfGetComments: [],
      averageOfTagsForPostPublication: [],
      frequency: [],
      mostUsedWords: [],
    };
    this.datePosts = [];
  }

  fetch() {
    dbg('Fetching instagram data');

    this.gram = new Nodegram({ accessToken: this.tokens.accessToken });

    this.gram.getAuthUrl();
    return this.fetchNumberOfUserPublications()
      .then(() => this.fetchNumberOfUserFollowers())
      .then(() => this.fetchNumberOfUserFollows())
      .then(() => this.fetchAverageOfGetLikes())
      .then(() => this.fetchAverageOfGetComments())
      .then(() => this.fetchAverageTagsForPostPublication())
      .then(() => this.fetchPostsFrequency())
      .then(() => this.data)
      .catch(err => dbg(`Error: ${err.message}`));
  }

  fetchNumberOfUserPublications() {
    dbg('Retrieving number of publications');

    return this.gram.get('/users/self/', {})
      .then(res => {
        this.data.numberOfUserPublications = res.counts.media;
          dbg(`Number of user publications: ${this.data.numberOfUserPublications}`);
      })
      .catch(err => dbg(`Error: ${err.message}`));
  }

  fetchNumberOfUserFollowers() {
    dbg('Retrieving number of followers');

    return this.gram.get('/users/self/', {})
      .then(res => {
        this.data.numberOfUserFollowers = res.counts.followed_by;
          dbg(`Number of user followers: ${this.data.numberOfUserFollowers}`);
      })
      .catch(err => dbg(`Error: ${err.message}`));
  }

  fetchNumberOfUserFollows() {
    dbg('Retrieving number of follows');

    return this.gram.get('/users/self/', {})
      .then(res => {
        this.data.numberOfUserFollows = res.counts.follows;
          dbg(`Number of user follows: ${this.data.numberOfUserFollows}`);
      })
      .catch(err => dbg(`Error: ${err.message}`));
  }

  fetchAverageOfGetLikes() {
    dbg('Retrieving average of likes per publication');
    let numberOfPublications = 0;
    let numberOfLikes = 0;

    return this.gram.get('/users/self/media/recent', {})
      .then(res => {
        numberOfPublications += res.length;
        res.forEach(res => numberOfLikes += res.likes.count);

        this.data.averageOfGetLikes = Math.round(numberOfLikes / numberOfPublications);
          dbg(`Average of likes: ${this.data.averageOfGetLikes}`);
      })
      .catch(err => dbg(`Error: ${err.message}`));
  }

  fetchAverageOfGetComments() {
    dbg('Retrieving average of comments per publication');

    let numberOfPublications = 0;
    let numberOfComments = 0;

    return this.gram.get('/users/self/media/recent', {})
      .then(res => {
        numberOfPublications += res.length;
        res.forEach(res => numberOfComments += res.comments.count);

        this.data.averageOfGetComments = Math.round(numberOfComments / numberOfPublications);
          dbg(`Average of comments: ${this.data.averageOfGetComments}`);
      })
      .catch(err => dbg(`Error: ${err.message}`));
  }

  fetchAverageTagsForPostPublication() {
    dbg('Retrieving average of tags used per publication');
    let numberOfPublications = 0;
    let numberOfTags = 0;
    var wordList = [];

    return this.gram.get('/users/self/media/recent', {})
      .then(res => {
          numberOfPublications += res.length;
          res.forEach((res => {
            numberOfTags += res.tags.length;

            res.tags.forEach((result => {
              wordList.push(result);
            }));
          }));

          this.data.averageOfTagsForPostPublication = Math.round(numberOfTags / numberOfPublications);
          dbg(`Average of tags for post publication: ${this.data.averageOfTagsForPostPublication}`);


          //Get the most used hashtags
          var wordAssociation = this.getWordFrequency(wordList);
          for(var i=0; i<3; i++){
            this.data.mostUsedWords.push(wordAssociation[i]);
          }
          dbg('Most used hashtags:');
          dbg(this.data.mostUsedWords);

        })
      .catch(err => dbg(`Error: ${err.message}`));
  }

  //Get word frequency for used hashtags and sort them in array (bigger to smaller)
  getWordFrequency(wordList){
    var a = [], b = [], prev;

    wordList.sort();
    for ( var i = 0; i < wordList.length; i++ ) {
      if ( wordList[i] !== prev ) {
        a.push(wordList[i]);
        b.push(1);
      } else {
        b[b.length-1]++;
      }
      prev = wordList[i];
    }

    //Associate the words with their occurrence
    var associated = a.reduce(function (previous, key, index) {
      previous[key] = b[index];
      return previous
    }, {});

    //Sort and reverse the associated array
    var tupleArray = [];
    for (var key in associated) tupleArray.push([key, associated[key]]);
    tupleArray.sort(function (a, b) { return a[1] - b[1] }).reverse();

    return tupleArray;
  }

{'fraction_non_alphanumeric': 0.1528939081333478, 'fraction_numerical': 0.0020631990444130743, 'mean_word_length': 2.9544869042507513, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '801669', 'n_tokens_mistral': 2894, 'n_tokens_neox': 2777, 'n_words': 514}