<commit_before>'use strict';

var fs = require('fs');
var async = require('async');
var ics = require('ics-parser');
var Anyfetch = require('anyfetch');

function generateIdentifier(baseIdentifier, event) {
  var identifier = event.type;

  if(event.uid) {
    identifier = event.uid;
  }
  else {
    if(event.summary) {
      identifier += "-" + event.summary;
    }
    if(event.start) {
      identifier += "-" + event.start;
    }
    if(event.end) {
      identifier += "-" + event.end;
    }
  }

  return baseIdentifier + " - " + (new Buffer(identifier).toString('base64'));
}

module.exports = function(path, document, changes, finalCb) {
  fs.readFile(path, function(err, data) {
    if(err) {
      finalCb(err);
    }
    var events = ics(data.toString());

    var anyfetch = new Anyfetch(document.access_token);
    anyfetch.setApiUrl(finalCb.apiUrl);

    async.each(events, function(event, cb) {
      if(event.type === "VEVENT") {
        var eventDocument = {
          document_type: "event",
          actions: document.actions,
          creation_date: new Date(document.creation_date),
          modification_date: new Date(document.modification_date),
          identifier: generateIdentifier(document.identifier, event),
          metadata: {},
          user_access: document.user_access
        };

        if(event.geo) {
          event.geolocation = event.geo.latitude + ";" + event.geo.longitude;
        }

        ['startDate', 'endDate', 'name', 'description', 'attendee', 'location', 'geolocation', 'organizer'].forEach(function(param) {
          if(event[param]) {
            eventDocument.metadata[param] = event[param];
          }
        });
        if(eventDocument.metadata.description) {
          // remove the default description
          eventDocument.metadata.description = eventDocument.metadata.description.replace(/[^\n]+https?:\/\/www\.google\.com\/calendar\/event\?action=VIEW.*/, '');
        }
        eventDocument.metadata.attendee = eventDocument.metadata.attendee || [];

        if(eventDocument.metadata.organizer) {
          eventDocument.metadata.attendee.push(eventDocument.metadata.organizer);
        }

        for(var i = 0; i < eventDocument.metadata.attendee.length; i += 1) {
          for(var j = i + 1; j < eventDocument.metadata.attendee.length; j += 1) {
            if((eventDocument.metadata.attendee[i].name && eventDocument.metadata.attendee[i].name === eventDocument.metadata.attendee[j].name) ||
              (eventDocument.metadata.attendee[i].mail && eventDocument.metadata.attendee[i].mail === eventDocument.metadata.attendee[j].mail)) {
              eventDocument.metadata.attendee[i].name = eventDocument.metadata.attendee[i].name || eventDocument.metadata.attendee[j].name;
              eventDocument.metadata.attendee[i].mail = eventDocument.metadata.attendee[i].mail || eventDocument.metadata.attendee[j].mail;

              eventDocument.metadata.attendee.splice(j, 1);
            }
          }
        }

        anyfetch.postDocument(eventDocument, cb);
      }
      else {
        cb();<commit_msg>Removing the metadata field if its empty<commit_after>'use strict';

var fs = require('fs');
var async = require('async');
var ics = require('ics-parser');
var Anyfetch = require('anyfetch');

function generateIdentifier(baseIdentifier, event) {
  var identifier = event.type;

  if(event.uid) {
    identifier = event.uid;
  }
  else {
    if(event.summary) {
      identifier += "-" + event.summary;
    }
    if(event.start) {
      identifier += "-" + event.start;
    }
    if(event.end) {
      identifier += "-" + event.end;
    }
  }

  return baseIdentifier + " - " + (new Buffer(identifier).toString('base64'));
}

module.exports = function(path, document, changes, finalCb) {
  fs.readFile(path, function(err, data) {
    if(err) {
      finalCb(err);
    }
    var events = ics(data.toString());

    var anyfetch = new Anyfetch(document.access_token);
    anyfetch.setApiUrl(finalCb.apiUrl);

    async.each(events, function(event, cb) {
      if(event.type === "VEVENT") {
        var eventDocument = {
          document_type: "event",
          actions: document.actions,
          creation_date: new Date(document.creation_date),
          modification_date: new Date(document.modification_date),
          identifier: generateIdentifier(document.identifier, event),
          metadata: {},
          user_access: document.user_access
        };

        if(event.geo) {
          event.geolocation = event.geo.latitude + ";" + event.geo.longitude;
        }

        ['startDate', 'endDate', 'name', 'description', 'attendee', 'location', 'geolocation', 'organizer'].forEach(function(param) {
          if(event[param]) {
            eventDocument.metadata[param] = event[param];
          }
        });

        if(eventDocument.metadata.description) {
          // remove the default description
          eventDocument.metadata.description = eventDocument.metadata.description.replace(/[^\n]+https?:\/\/www\.google\.com\/calendar\/event\?action=VIEW.*/, '');

          if(eventDocument.metadata.description.length === 0) {
            delete eventDocument.metadata.description;
          }
        }
        eventDocument.metadata.attendee = eventDocument.metadata.attendee || [];

        if(eventDocument.metadata.organizer) {
          eventDocument.metadata.attendee.push(eventDocument.metadata.organizer);
        }

        for(var i = 0; i < eventDocument.metadata.attendee.length; i += 1) {
          for(var j = i + 1; j < eventDocument.metadata.attendee.length; j += 1) {
            if((eventDocument.metadata.attendee[i].name && eventDocument.metadata.attendee[i].name === eventDocument.metadata.attendee[j].name) ||
              (eventDocument.metadata.attendee[i].mail && eventDocument.metadata.attendee[i].mail === eventDocument.metadata.attendee[j].mail)) {
              eventDocument.metadata.attendee[i].name = eventDocument.metadata.attendee[i].name || eventDocument.metadata.attendee[j].name;
              eventDocument.metadata.attendee[i].mail = eventDocument.metadata.attendee[i].mail || eventDocument.metadata.attendee[j].mail;

              eventDocument.metadata.attendee.splice(j, 1);
            }
          }
        }

        anyfetch.postDocument(eventDocument, cb);
      }
      else {
        cb();
{'fraction_non_alphanumeric': 0.14548600188738597, 'fraction_numerical': 0.0023592324630386913, 'mean_word_length': 2.716540035067212, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '288271', 'n_tokens_mistral': 1898, 'n_tokens_neox': 1854, 'n_words': 283}