<commit_before>import * as vlFieldDef from '../fielddef';
import * as util from '../util';
import {Model} from './Model';
import {FieldDef} from '../schema/fielddef.schema';
import {StackProperties} from './stack';

import {MAXBINS_DEFAULT} from '../bin';
import {Channel, X, Y, ROW, COLUMN} from '../channel';
import {SOURCE, STACKED, LAYOUT, SUMMARY} from '../data';
import * as time from './time';
import {NOMINAL, ORDINAL, QUANTITATIVE, TEMPORAL} from '../type';

/**
 * Create Vega's data array from a given encoding.
 *
 * @param  {Encoding} encoding
 * @return {Array} Array of Vega data.
 *                 This always includes a "source" data table.
 *                 If the encoding contains aggregate value, this will also create
 *                 aggregate table as well.
 */
export function compileData(model: Model) {
  var def = [source.def(model)];

  const summaryDef = summary.def(model);
  if (summaryDef) {
    def.push(summaryDef);
  }

  // TODO add "having" filter here

  // append non-positive filter at the end for the data table
  filterNonPositiveForLog(def[def.length - 1], model);

  // add stats for layout calculation
  const statsDef = layout.def(model);
  if(statsDef) {
    def.push(statsDef);
  }

  // Stack
  const stackDef = model.stack();
  if (stackDef) {
    def.push(stack.def(model, stackDef));
  }

  return def;
}

// TODO: Consolidate all Vega interface
interface VgData {
  name: string;
  source?: string;
  values?: any;
  format?: any;
  url?: any;
  transform?: any;
}

export namespace source {
  export function def(model: Model): VgData {
    var source:VgData = {name: SOURCE};

    // Data source (url or inline)
    if (model.hasValues()) {
      source.values = model.data().values;
      source.format = {type: 'json'};
    } else {
      source.url = model.data().url;
      source.format = {type: model.data().formatType};
    }

    // Set data's format.parse if needed
    var parse = formatParse(model);
    if (parse) {
      source.format.parse = parse;
    }

    source.transform = transform(model);
    return source;
  }

  function formatParse(model: Model) {
    var parse;

    model.forEach(function(fieldDef: FieldDef) {
      if (fieldDef.type === TEMPORAL) {
        parse = parse || {};
        parse[fieldDef.field] = 'date';
      } else if (fieldDef.type === QUANTITATIVE) {
        if (vlFieldDef.isCount(fieldDef)) return;
        parse = parse || {};
        parse[fieldDef.field] = 'number';
      }
    });

    return parse;
  }

  /**
   * Generate Vega transforms for the source data table.  This can include
   * transforms for time unit, binning and filtering.
   */
  export function transform(model: Model) {
    // null filter comes first so transforms are not performed on null values
    // time and bin should come before filter so we can filter by time and bin
    return nullFilterTransform(model).concat(
      formulaTransform(model),
      timeTransform(model),
      binTransform(model),
      filterTransform(model)
    );
  }

  export function timeTransform(model: Model) {
    return model.reduce(function(transform, fieldDef: FieldDef, channel: Channel) {
      if (fieldDef.type === TEMPORAL && fieldDef.timeUnit) {
        var field = model.field(channel, {nofn: true, datum: true});

        transform.push({
          type: 'formula',
          field: model.field(channel),
          expr: time.formula(fieldDef.timeUnit, field)
        });
      }
      return transform;
    }, []);
  }

  export function binTransform(model: Model) {
    return model.reduce(function(transform, fieldDef: FieldDef, channel: Channel) {
      const bin = model.bin(channel);
      if (bin) {
        transform.push({
          type: 'bin',
          field: fieldDef.field,
          output: {
            start: model.field(channel, {binSuffix: '_start'}),
            end: model.field(channel, {binSuffix: '_end'})
          },
          maxbins: typeof bin === 'boolean' ? MAXBINS_DEFAULT : bin.maxbins
        });
        // temporary fix for adding missing `bin_mid` from the bin transform
        transform.push({
          type: 'formula',
          field: model.field(channel, {binSuffix: '_mid'}),
          expr: '(' + model.field(channel, {datum: true, binSuffix: '_start'}) +
                '+' + model.field(channel, {datum: true, binSuffix: '_end'}) + ') / 2'
        });
      }
      return transform;
    }, []);
  }

  /**
   * @return {Array} An array that might contain a filter transform for filtering null value based on filterNul config
   */
  export function nullFilterTransform(model: Model) {
    const filterNull = model.config('filterNull');
    const filteredFields = util.keys(model.reduce(function(filteredFields, fieldDef: FieldDef) {
      if (fieldDef.field && fieldDef.field !== '*' && filterNull[fieldDef.type]) {
        filteredFields[fieldDef.field] = true;
      }
      return filteredFields;
    }, {}));<commit_msg>Use the new `bin_mid` from vg 2.4.0  (fixes #695)<commit_after>import * as vlFieldDef from '../fielddef';
import * as util from '../util';
import {Model} from './Model';
import {FieldDef} from '../schema/fielddef.schema';
import {StackProperties} from './stack';

import {MAXBINS_DEFAULT} from '../bin';
import {Channel, X, Y, ROW, COLUMN} from '../channel';
import {SOURCE, STACKED, LAYOUT, SUMMARY} from '../data';
import * as time from './time';
import {NOMINAL, ORDINAL, QUANTITATIVE, TEMPORAL} from '../type';

/**
 * Create Vega's data array from a given encoding.
 *
 * @param  {Encoding} encoding
 * @return {Array} Array of Vega data.
 *                 This always includes a "source" data table.
 *                 If the encoding contains aggregate value, this will also create
 *                 aggregate table as well.
 */
export function compileData(model: Model) {
  var def = [source.def(model)];

  const summaryDef = summary.def(model);
  if (summaryDef) {
    def.push(summaryDef);
  }

  // TODO add "having" filter here

  // append non-positive filter at the end for the data table
  filterNonPositiveForLog(def[def.length - 1], model);

  // add stats for layout calculation
  const statsDef = layout.def(model);
  if(statsDef) {
    def.push(statsDef);
  }

  // Stack
  const stackDef = model.stack();
  if (stackDef) {
    def.push(stack.def(model, stackDef));
  }

  return def;
}

// TODO: Consolidate all Vega interface
interface VgData {
  name: string;
  source?: string;
  values?: any;
  format?: any;
  url?: any;
  transform?: any;
}

export namespace source {
  export function def(model: Model): VgData {
    var source:VgData = {name: SOURCE};

    // Data source (url or inline)
    if (model.hasValues()) {
      source.values = model.data().values;
      source.format = {type: 'json'};
    } else {
      source.url = model.data().url;
      source.format = {type: model.data().formatType};
    }

    // Set data's format.parse if needed
    var parse = formatParse(model);
    if (parse) {
      source.format.parse = parse;
    }

    source.transform = transform(model);
    return source;
  }

  function formatParse(model: Model) {
    var parse;

    model.forEach(function(fieldDef: FieldDef) {
      if (fieldDef.type === TEMPORAL) {
        parse = parse || {};
        parse[fieldDef.field] = 'date';
      } else if (fieldDef.type === QUANTITATIVE) {
        if (vlFieldDef.isCount(fieldDef)) return;
        parse = parse || {};
        parse[fieldDef.field] = 'number';
      }
    });

    return parse;
  }

  /**
   * Generate Vega transforms for the source data table.  This can include
   * transforms for time unit, binning and filtering.
   */
  export function transform(model: Model) {
    // null filter comes first so transforms are not performed on null values
    // time and bin should come before filter so we can filter by time and bin
    return nullFilterTransform(model).concat(
      formulaTransform(model),
      timeTransform(model),
      binTransform(model),
      filterTransform(model)
    );
  }

  export function timeTransform(model: Model) {
    return model.reduce(function(transform, fieldDef: FieldDef, channel: Channel) {
      if (fieldDef.type === TEMPORAL && fieldDef.timeUnit) {
        var field = model.field(channel, {nofn: true, datum: true});

        transform.push({
          type: 'formula',
          field: model.field(channel),
          expr: time.formula(fieldDef.timeUnit, field)
        });
      }
      return transform;
    }, []);
  }

  export function binTransform(model: Model) {
    return model.reduce(function(transform, fieldDef: FieldDef, channel: Channel) {
      const bin = model.bin(channel);
      if (bin) {
        transform.push({
          type: 'bin',
          field: fieldDef.field,
          output: {
            start: model.field(channel, {binSuffix: '_start'}),
            mid: model.field(channel, {binSuffix: '_mid'}),
            end: model.field(channel, {binSuffix: '_end'})
          },
          maxbins: typeof bin === 'boolean' ? MAXBINS_DEFAULT : bin.maxbins
        });
      }
      return transform;
    }, []);
  }

  /**
   * @return {Array} An array that might contain a filter transform for filtering null value based on filterNul config
   */
  export function nullFilterTransform(model: Model) {
    const filterNull = model.config('filterNull');
    const filteredFields = util.keys(model.reduce(function(filteredFields, fieldDef: FieldDef) {
      if (fieldDef.field && fieldDef.field !== '*' && filterNull[fieldDef.type]) {
        filteredFields[fieldDef.field] = true;
      }
      return filteredFields;
    }, {}));
{'fraction_non_alphanumeric': 0.13170175256662864, 'fraction_numerical': 0.0009333195063776833, 'mean_word_length': 2.699271192942079, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3701836', 'n_tokens_mistral': 2910, 'n_tokens_neox': 2826, 'n_words': 831}