<commit_before>/**
 * Manage files associated with datasets. When uploading a file creates a new dataset
 * @param collectionId
 * @param data
 * @param field - JSON data key
 * @param idField - HTML id for the template (has to be 'edition')
 */

function addDataset(collectionId, data, field, idField) {
  var downloadExtensions, pageType;
  var uriUpload;
  var lastIndex;
  if (data[field]) {
    lastIndex = data[field].length;
  } else {
    lastIndex = 0;
  }
  var uploadedNotSaved = {uploaded: false, saved: false, editionUri: ""};
  $(".workspace-edit").scrollTop(Florence.globalVars.pagePos);

  //Add
  if (data.timeseries) {
    downloadExtensions = /\.csdb$/;
    pageType = 'timeseries_dataset';
  } else {
    downloadExtensions = /\.csv$|.xls$|.zip$/;
    pageType = 'dataset';
  }

  $('#add-' + idField).one('click', function () {
    var position = $(".workspace-edit").scrollTop();
    Florence.globalVars.pagePos = position + 200;
    $('#sortable-' + idField).append(
      '<div id="' + lastIndex + '" class="edit-section__item">' +
      '  <form id="UploadForm">' +
      '    <textarea class="auto-size" placeholder="Period (E.g. 2015, August to December 2010, etc." type="text"             id="edition"></textarea>' +
      '    <textarea class="auto-size" placeholder="Label (E.g. Final, Revised, etc.)" type="text" id="version"></textarea>' +
      '    <input type="file" title="Select a file and click Submit" name="files">' +
      '    <br>' +
      '    <button type="submit" form="UploadForm" value="submit">Submit</button>' +
      '    <button class="btn-page-cancel" id="file-cancel">Cancel</button>' +
      '  </form>' +
      '  <div id="response"></div>' +
      '  <ul id="list"></ul>' +
      '</div>');

    $('#file-cancel').one('click', function (e) {
      e.preventDefault();
      $('#' + lastIndex).remove();
      //Check files uploaded and delete them
      if (uploadedNotSaved.uploaded === true) {
        data[field].splice(-1, 1);
        deleteContent(Florence.collection.id, uploadedNotSaved.editionUri,
          onSuccess = function () {
          },
          onError = function (error) {
            handleApiError(error);
          }
        );
      }
      addDataset(collectionId, pageData, 'datasets', 'edition');
    });

    $('#UploadForm').submit(function (e) {
      e.preventDefault();
      e.stopImmediatePropagation();

      var formdata = new FormData();

      function showUploadedItem(source) {
        $('#list').append(source);
      }

      var pageTitle = this[0].value;
      var pageTitleTrimmed = pageTitle.replace(/[^A-Z0-9]+/ig, "").toLowerCase();

      var versionLabel = this[1].value;

      var file = this[2].files[0];
      if (!file) {
        sweetAlert("Please select a file to upload");
        return;
      }

      document.getElementById("response").innerHTML = "Uploading . . .";

      var fileNameNoSpace = file.name.replace(/[^a-zA-Z0-9\.]/g, "").toLowerCase();
      uriUpload = data.uri + '/' + pageTitleTrimmed + '/' + fileNameNoSpace;
      var safeUriUpload = checkPathSlashes(uriUpload);

      if (data[field] && data[field].length > 0) {
        $(data[field]).each(function (i, filesUploaded) {
          if (filesUploaded.file == safeUriUpload) {
            sweetAlert('This file already exists');
            $('#' + lastIndex).remove();
            addDataset(collectionId, pageData, 'datasets', 'edition');
            return;
          }
        });
      }

      if (!!file.name.match(downloadExtensions)) {
        showUploadedItem(fileNameNoSpace);
        if (formdata) {
          formdata.append("name", file);
        }
      } else {
        sweetAlert('This file type is not supported');
        $('#' + lastIndex).remove();
        addDataset(collectionId, pageData, 'datasets', 'edition');
        return;<commit_msg>Solve bugs on detecting duplicate files<commit_after>/**
 * Manage files associated with datasets. When uploading a file creates a new dataset
 * @param collectionId
 * @param data
 * @param field - JSON data key
 * @param idField - HTML id for the template (has to be 'edition')
 */

function addDataset(collectionId, data, field, idField) {
  var downloadExtensions, pageType;
  var uriUpload;
  var lastIndex;
  if (data[field]) {
    lastIndex = data[field].length;
  } else {
    lastIndex = 0;
  }
  var uploadedNotSaved = {uploaded: false, saved: false, editionUri: ""};
  $(".workspace-edit").scrollTop(Florence.globalVars.pagePos);

  //Add
  if (data.timeseries) {
    downloadExtensions = /\.csdb$/;
    pageType = 'timeseries_dataset';
  } else {
    downloadExtensions = /\.csv$|.xls$|.zip$/;
    pageType = 'dataset';
  }

  $('#add-' + idField).one('click', function () {
    var position = $(".workspace-edit").scrollTop();
    Florence.globalVars.pagePos = position + 200;
    $('#sortable-' + idField).append(
      '<div id="' + lastIndex + '" class="edit-section__item">' +
      '  <form id="UploadForm">' +
      '    <textarea class="auto-size" placeholder="Period (E.g. 2015, August to December 2010, etc." type="text"             id="edition"></textarea>' +
      '    <textarea class="auto-size" placeholder="Label (E.g. Final, Revised, etc.)" type="text" id="version"></textarea>' +
      '    <input type="file" title="Select a file and click Submit" name="files">' +
      '    <br>' +
      '    <button type="submit" form="UploadForm" value="submit">Submit</button>' +
      '    <button class="btn-page-cancel" id="file-cancel">Cancel</button>' +
      '  </form>' +
      '  <div id="response"></div>' +
      '  <ul id="list"></ul>' +
      '</div>');

    $('#file-cancel').one('click', function (e) {
      e.preventDefault();
      $('#' + lastIndex).remove();
      //Check files uploaded and delete them
      if (uploadedNotSaved.uploaded === true) {
        data[field].splice(-1, 1);
        deleteContent(Florence.collection.id, uploadedNotSaved.editionUri,
          onSuccess = function () {
          },
          onError = function (error) {
            handleApiError(error);
          }
        );
      }
      addDataset(collectionId, pageData, 'datasets', 'edition');
    });

    $('#UploadForm').submit(function (e) {
      e.preventDefault();
      e.stopImmediatePropagation();

      var formdata = new FormData();

      function showUploadedItem(source) {
        $('#list').append(source);
      }

      var pageTitle = this[0].value;
      var pageTitleTrimmed = pageTitle.replace(/[^A-Z0-9]+/ig, "").toLowerCase();

      var versionLabel = this[1].value;

      var file = this[2].files[0];
      if (!file) {
        sweetAlert("Please select a file to upload");
        return;
      }

      document.getElementById("response").innerHTML = "Uploading . . .";

      var fileNameNoSpace = file.name.replace(/[^a-zA-Z0-9\.]/g, "").toLowerCase();
      uriUpload = data.uri + '/' + pageTitleTrimmed + '/' + fileNameNoSpace;
      var safeUriUpload = checkPathSlashes(uriUpload);

      if (data[field] && data[field].length > 0) {
        $(data[field]).each(function (i, filesUploaded) {
          if (filesUploaded.file === safeUriUpload || filesUploaded.file === fileNameNoSpace) {
            sweetAlert('This file already exists');
            $('#' + lastIndex).remove();
            addDataset(collectionId, pageData, 'datasets', 'edition');
            formdata = false;   // if not present the existing file was being overwritten
            return;
          }
        });
      }

      if (!!file.name.match(downloadExtensions)) {
        showUploadedItem(fileNameNoSpace);
        if (formdata) {
          formdata.append("name", file);
        }
      } else {
        sweetAlert('This file type is not supported');
        $('#' + lastIndex).remove();
        addDataset(collectionId, pageData, 'datasets', 'edition');
        return;
{'fraction_non_alphanumeric': 0.16685728809251493, 'fraction_numerical': 0.00584572372601347, 'mean_word_length': 2.63342566943675, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2763768', 'n_tokens_mistral': 2421, 'n_tokens_neox': 2374, 'n_words': 513}