<commit_before>vde.Vis.Pipeline = (function() {
  var pipeline = function(source) {
    this.name = 'pipeline_' + (vg.keys(vde.Vis.pipelines).length+1);
    this.displayName = this.name;

    this.source = source;
    this.transforms = [];

    this.forkName = null;
    this.forkIdx  = null;

    this.scales = {};

    vde.Vis.pipelines[this.name] = this;

    return this;
  };

  var prototype = pipeline.prototype;

  prototype.spec = function() {
    var self = this;
    var specs = [{
      name: this.name,
      source: this.source,
      transform: []
    }];

    vde.Vis.Callback.run('pipeline.pre_spec', this, {spec: specs});

    var spec = 0;
    this.transforms.forEach(function(t, i) {
      if(t.forkPipeline) {
        spec++;
        self.forkName || (self.forkName = self.name + '_' + t.type);
        self.forkIdx = i;

        specs.push({
          name: self.forkName,
          source: self.source,
          transform: vg.duplicate(specs[spec-1].transform || [])
        }); 
      }

      var s = t.spec();
      if(s) specs[spec].transform.push(s);
    });

    vde.Vis.Callback.run('pipeline.post_spec', this, {spec: specs});

    return specs;
  };

  prototype.values = function(sliceBeg, sliceEnd) {
    var values = vg.duplicate(vde.Vis._data[this.source].values).map(vg.data.ingest);
    this.transforms.slice(sliceBeg, sliceEnd).forEach(function(t) {
      if(t.isVisual) return;

      values = t.transform(values);
    });

    return values;
  };

  prototype.schema = function(sliceBeg, sliceEnd) {
    var self = this, 
        fields = [], seenFields = {};
    var values = vg.duplicate(vde.Vis._data[this.source].values).map(vg.data.ingest);

    var buildFields = function(pipeline) {
      [(values[0] || {}).data, values[0], (values.values || [])[0]].forEach(function(v, i) {
        vg.keys(v).forEach(function(k) {
          if(i != 0 && ['data', 'values', 'keys'].indexOf(k) != -1) return;
          if(seenFields[k]) return;

          var field = new vde.Vis.Field(k);
          field.raw = (i == 0);
          field.pipelineName = pipeline;
          field.type = vg.isNumber(v[k]) ? 'linear' : 'ordinal';

          fields.push(field);
          seenFields[k] = true;
        });
      });
    };

    // Build fields once before we apply any transforms
    buildFields(this.name);

    var pipelineName = this.name;
    this.transforms.slice(sliceBeg, sliceEnd).forEach(function(t) {
      if(t.forkPipeline) pipelineName = self.forkName;
      if(t.isVisual) return;

      values = t.transform(values);
      buildFields(pipelineName);
    });

    return [fields, values];
  };

  // Given a spec, find a pre-existing scale that matches,
  // or if none do, build a new scale. 
  prototype.scale = function(spec, defaultSpec, displayName) {
    for(var scaleName in this.scales) {
      if(this.scales[scaleName].equals(spec))
        return this.scales[scaleName];
    }
<commit_msg>Use data parsings, if available, to determine field.type<commit_after>vde.Vis.Pipeline = (function() {
  var pipeline = function(source) {
    this.name = 'pipeline_' + (vg.keys(vde.Vis.pipelines).length+1);
    this.displayName = this.name;

    this.source = source;
    this.transforms = [];

    this.forkName = null;
    this.forkIdx  = null;

    this.scales = {};

    vde.Vis.pipelines[this.name] = this;

    return this;
  };

  var prototype = pipeline.prototype;

  prototype.spec = function() {
    var self = this;
    var specs = [{
      name: this.name,
      source: this.source,
      transform: []
    }];

    vde.Vis.Callback.run('pipeline.pre_spec', this, {spec: specs});

    var spec = 0;
    this.transforms.forEach(function(t, i) {
      if(t.forkPipeline) {
        spec++;
        self.forkName || (self.forkName = self.name + '_' + t.type);
        self.forkIdx = i;

        specs.push({
          name: self.forkName,
          source: self.source,
          transform: vg.duplicate(specs[spec-1].transform || [])
        }); 
      }

      var s = t.spec();
      if(s) specs[spec].transform.push(s);
    });

    vde.Vis.Callback.run('pipeline.post_spec', this, {spec: specs});

    return specs;
  };

  prototype.values = function(sliceBeg, sliceEnd) {
    var values = vg.duplicate(vde.Vis._data[this.source].values).map(vg.data.ingest);
    this.transforms.slice(sliceBeg, sliceEnd).forEach(function(t) {
      if(t.isVisual) return;

      values = t.transform(values);
    });

    return values;
  };

  prototype.schema = function(sliceBeg, sliceEnd) {
    var self = this, 
        fields = [], seenFields = {};
    var values = vg.duplicate(vde.Vis._data[this.source].values).map(vg.data.ingest);

    var buildFields = function(pipeline) {
      var parse = vde.Vis._data[self.source].format.parse;
      [(values[0] || {}).data, values[0], (values.values || [])[0]].forEach(function(v, i) {
        vg.keys(v).forEach(function(k) {
          if(i != 0 && ['data', 'values', 'keys'].indexOf(k) != -1) return;
          if(seenFields[k]) return;

          var field = new vde.Vis.Field(k);
          field.raw = (i == 0);
          field.pipelineName = pipeline;
          if(parse[k]) field.type = (parse[k] == 'date') ? 'time' : (parse[k] == 'number') ? 'linear' : 'ordinal';
          else field.type = vg.isNumber(v[k]) ? 'linear' : 'ordinal';

          fields.push(field);
          seenFields[k] = true;
        });
      });
    };

    // Build fields once before we apply any transforms
    buildFields(this.name);

    var pipelineName = this.name;
    this.transforms.slice(sliceBeg, sliceEnd).forEach(function(t) {
      if(t.forkPipeline) pipelineName = self.forkName;
      if(t.isVisual) return;

      values = t.transform(values);
      buildFields(pipelineName);
    });

    return [fields, values];
  };

  // Given a spec, find a pre-existing scale that matches,
  // or if none do, build a new scale. 
  prototype.scale = function(spec, defaultSpec, displayName) {
    for(var scaleName in this.scales) {
      if(this.scales[scaleName].equals(spec))
        return this.scales[scaleName];
    }

{'fraction_non_alphanumeric': 0.1602815979043877, 'fraction_numerical': 0.0029469548133595285, 'mean_word_length': 2.5068886337543055, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6237464', 'n_tokens_mistral': 2099, 'n_tokens_neox': 1995, 'n_words': 390}