<commit_before>
define(['q', 'lodash', 'xml2json', 'request', 'fs', 'server/sources/ymlHerokuConfig'], function (Q, _, xml2json, request, fs, configReader) {

  var config = configReader.create('gocd');

  var PIPELINE_API_BASE = '/go/api/pipelines/' + config.get().pipeline;
  var PIPELINE_FEED_ENDPOINT = PIPELINE_API_BASE + '/stages.xml';

  var SAMPLES_PATH = 'server/sources/gocd/sample/';

  var pipelineFeedEtag;

  function resolveAndPromiseSampleFile(path, convertToJson) {
    var defer = Q.defer();

    try {
      var fileContents = fs.readFileSync(path);

      if(convertToJson === true) {
        defer.resolve(xml2json.toJson(fileContents, {
          object: true, sanitize: false
        }));
      } else {
        defer.resolve(fileContents);
      }
    } catch (err) {
      console.log('ERROR reading file', path, err);
      defer.reject();
    }

    return defer.promise;
  }

  var get = function(next) {
    if (config.get().sampleIt()) {
      return getSample(next);
    } else {
      var defer = Q.defer();

      var url = next ? config.addCredentialsToUrl(next) : config.get().url;

      var loggableUrl = next ? next : config.get().loggableUrl;
      console.log('Requesting', loggableUrl + PIPELINE_FEED_ENDPOINT);

      request({ method: 'GET', url: url + PIPELINE_FEED_ENDPOINT, headers: {'If-None-Match': pipelineFeedEtag } },
        function (error, response, body) {
          pipelineFeedEtag = response.headers.etag;

          var json = xml2json.toJson(body, {
            object: true, sanitize: false
          });
          defer.resolve(json);
        }
      );

      return defer.promise;
    }

  };

  function getSample(next) {
    var source = next ? next : SAMPLES_PATH + 'pipeline-stages.xml';
    return resolveAndPromiseSampleFile(source, true);
  }

  var getPipelineRunDetails = function(buildNumber) {
    if (config.get().sampleIt()) {
      return getSamplePipelineRunDetails(buildNumber);
    } else {
      var defer = Q.defer();
<commit_msg>Removing etags for now<commit_after>
define(['q', 'lodash', 'xml2json', 'request', 'fs', 'server/sources/ymlHerokuConfig'], function (Q, _, xml2json, request, fs, configReader) {

  var config = configReader.create('gocd');

  var PIPELINE_API_BASE = '/go/api/pipelines/' + config.get().pipeline;
  var PIPELINE_FEED_ENDPOINT = PIPELINE_API_BASE + '/stages.xml';

  var SAMPLES_PATH = 'server/sources/gocd/sample/';

  var pipelineFeedEtag;

  function resolveAndPromiseSampleFile(path, convertToJson) {
    var defer = Q.defer();

    try {
      var fileContents = fs.readFileSync(path);

      if(convertToJson === true) {
        defer.resolve(xml2json.toJson(fileContents, {
          object: true, sanitize: false
        }));
      } else {
        defer.resolve(fileContents);
      }
    } catch (err) {
      console.log('ERROR reading file', path, err);
      defer.reject();
    }

    return defer.promise;
  }

  var get = function(next) {
    if (config.get().sampleIt()) {
      return getSample(next);
    } else {
      var defer = Q.defer();

      var url = next ? config.addCredentialsToUrl(next) : config.get().url;

      var loggableUrl = next ? next : config.get().loggableUrl;
      console.log('Requesting', loggableUrl + PIPELINE_FEED_ENDPOINT);

      request({ method: 'GET', url: url + PIPELINE_FEED_ENDPOINT}, //, headers: {'If-None-Match': pipelineFeedEtag } },
        function (error, response, body) {
          pipelineFeedEtag = response.headers.etag;

          var json = xml2json.toJson(body, {
            object: true, sanitize: false
          });
          defer.resolve(json);
        }
      );

      return defer.promise;
    }

  };

  function getSample(next) {
    var source = next ? next : SAMPLES_PATH + 'pipeline-stages.xml';
    return resolveAndPromiseSampleFile(source, true);
  }

  var getPipelineRunDetails = function(buildNumber) {
    if (config.get().sampleIt()) {
      return getSamplePipelineRunDetails(buildNumber);
    } else {
      var defer = Q.defer();

{'fraction_non_alphanumeric': 0.13564356435643565, 'fraction_numerical': 0.0019801980198019802, 'mean_word_length': 2.859598853868195, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2539578', 'n_tokens_mistral': 1318, 'n_tokens_neox': 1269, 'n_words': 261}