<commit_before>'use strict';

var request = require('request');

var ONLY_PDF = true;

var defineUrlFromFile = function(file) {
  if(file.exportLinks) { // Gdocs ! Yay !
    console.log("Google docs type");
    var el = file.exportLinks;
    if(el["text/html"] && !ONLY_PDF) {
      return {
        url: el["text/html"],
        type: "document"
      };
    } else if(el["application/pdf"]) {
      return {
        url: el["application/pdf"],
        type: "file"
      }
    } else if(el["text/plain"]) {
      return {
        url: el["text/plain"],
        type: "document"
      };
    }
  }
  console.log("Standard file");
  return {
    url: file.downloadUrl,
    type: "file"
  }
};

/**
 * Run the task of uploading a document to anyfetch.
 * This function will be used as a queue
 * @see https://github.com/caolan/async#queueworker-concurrency
 *
 * @param {Object} task Task param
 * @param {AnyFetchClient} anyfetchClient The anyfetch client
 * @param {String}
 * @param {Function} cb Callback once task has been processed.
 */
module.exports = function(task, anyfetchClient, authToken, cb) {
  console.log(" ");
  console.log("====== " + task.id + " ======");
  if(task.deleted) {
    console.log("DELETE");
    // File has been removed
    return anyfetchClient.deletedocument(task.id, anyfetchClient, cb);
  }
  
  var fInfos = defineUrlFromFile(task);

  var isHTML = false;

  // Upload file onto AnyFetch
  var pathExt = "";
  if(task.exportLinks) {
    if(task.exportLinks["text/html"] && !ONLY_PDF) {
      pathExt = ".html";
      isHTML = true;
    } else {
      pathExt = ".pdf";
    }
  }

  var document = {
    identifier: task.id,
    actions: {
      'show': task.alternateLink,
    },
    metadatas: {
      title: task.title,
      path: "/" + task.id + "/" + task.title + pathExt,
    },
    document_type: fInfos.type,
    user_access: [anyfetchClient.accessToken]
  };<commit_msg>Fix spaces<commit_after>'use strict';

var request = require('request');

var ONLY_PDF = true;

var defineUrlFromFile = function(file) {
  if(file.exportLinks) { // Gdocs ! Yay !
    console.log("Google docs type");
    var el = file.exportLinks;
    if(el["text/html"] && !ONLY_PDF) {
      return {
        url: el["text/html"],
        type: "document"
      };
    } else if(el["application/pdf"]) {
      return {
        url: el["application/pdf"],
        type: "file"
      }
    } else if(el["text/plain"]) {
      return {
        url: el["text/plain"],
        type: "document"
      };
    }
  }
  console.log("Standard file");
  return {
    url: file.downloadUrl,
    type: "file"
  }
};

/**
 * Run the task of uploading a document to anyfetch.
 * This function will be used as a queue
 * @see https://github.com/caolan/async#queueworker-concurrency
 *
 * @param {Object} task Task param
 * @param {AnyFetchClient} anyfetchClient The anyfetch client
 * @param {String}
 * @param {Function} cb Callback once task has been processed.
 */
module.exports = function(task, anyfetchClient, authToken, cb) {
  console.log(" ");
  console.log("====== " + task.id + " ======");
  if(task.deleted) {
    console.log("DELETE");
    // File has been removed
    return anyfetchClient.deletedocument(task.id, anyfetchClient, cb);
  }

  var fInfos = defineUrlFromFile(task);

  var isHTML = false;

  // Upload file onto AnyFetch
  var pathExt = "";
  if(task.exportLinks) {
    if(task.exportLinks["text/html"] && !ONLY_PDF) {
      pathExt = ".html";
      isHTML = true;
    } else {
      pathExt = ".pdf";
    }
  }

  var document = {
    identifier: task.id,
    actions: {
      'show': task.alternateLink,
    },
    metadatas: {
      title: task.title,
      path: "/" + task.id + "/" + task.title + pathExt,
    },
    document_type: fInfos.type,
    user_access: [anyfetchClient.accessToken]
  };
{'fraction_non_alphanumeric': 0.15730337078651685, 'fraction_numerical': 0.0, 'mean_word_length': 2.6318785578747628, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2001133', 'n_tokens_mistral': 1275, 'n_tokens_neox': 1259, 'n_words': 295}