<commit_before>'use strict';

import { List } from 'immutable';
import * as d3 from 'd3';
import * as Q from 'q';
import * as Qajax from 'qajax';
import { ImmutableClass, ImmutableInstance, isInstanceOf, arraysEqual } from 'higher-object';
import { $, Expression, ExpressionJS, Executor, basicExecutorFactory, Dataset, Datum, Attributes, AttributeInfo, ChainExpression } from 'plywood';
import { upperCaseFirst, listsEqual } from '../../utils/general';
import { Dimension, DimensionJS } from '../dimension/dimension';
import { Measure, MeasureJS } from '../measure/measure';

function getSplitsDescription(ex: Expression): string {
  var splits: string[] = [];
  ex.forEach((ex) => {
    if (ex instanceof ChainExpression) {
      ex.actions.forEach((action) => {
        if (action.action === 'split') splits.push(action.expression.toString());
      });
    }
  });
  return splits.join(';');
}

function queryUrlExecutorFactory(url: string): Executor {
  return (ex: Expression) => {
    return Qajax({
      method: "POST",
      url: url + '?by=' + getSplitsDescription(ex),
      data: {
        dataset: 'wiki',
        expression: ex.toJS()
      }
    })
      .then(Qajax.filterSuccess)
      .then(Qajax.toJSON)
      .then((dataJS) => Dataset.fromJS(dataJS));
  };
}

function dimensionPreference(dimension: Dimension): number {
  if (dimension.type === 'TIME') return 0;
  return 1;
}

function measurePreference(measure: Measure): number {
  if (measure.name === 'count') return 0;
  return 1;
}

interface DimensionsMetrics {
  dimensions: List<Dimension>;
  measures: List<Measure>;
  timeAttribute: Expression;
  defaultSortOn: string;
}

function makeDimensionsMetricsFromAttributes(attributes: Attributes): DimensionsMetrics {
  var dimensionArray: Dimension[] = [];
  var measureArray: Measure[] = [];
  var timeAttribute = $('time');
  var defaultSortOn: string = null;

  if (!attributes['count']) {
    measureArray.push(new Measure({
      name: 'count',
      title: 'Count',
      expression: $('main').count(),
      format: Measure.DEFAULT_FORMAT
    }));
    defaultSortOn = 'count';
  }

  for (let k in attributes) {
    let type = attributes[k].type;
    switch (type) {
      case 'TIME':
      case 'STRING':
        dimensionArray.push(new Dimension({
          name: k,
          title: upperCaseFirst(k),
          expression: $(k),
          type: type,
          sortOn: null
        }));
        break;

      case 'NUMBER':
        measureArray.push(new Measure({
          name: k,
          title: upperCaseFirst(k),
          expression: $('main').sum($(k)),
          format: Measure.DEFAULT_FORMAT
        }));
        break;

      default:
        throw new Error('bad type ' + type);
    }
  }

  dimensionArray.sort((a, b) => {
    var prefDiff = dimensionPreference(a) - dimensionPreference(b);
    if (prefDiff) return prefDiff;
    return a.title.localeCompare(b.title);
  });

  measureArray.sort((a, b) => {
    var prefDiff = measurePreference(a) - measurePreference(b);
    if (prefDiff) return prefDiff;
    return a.title.localeCompare(b.title);
  });

  if (!defaultSortOn && measureArray.length) {
    defaultSortOn = measureArray[0].name;
  }

  return {
    dimensions: List(dimensionArray),
    measures: List(measureArray),
    timeAttribute,
    defaultSortOn
  };
}


export interface DataSourceValue {
  name: string;
  title: string;
  source: string;
  metadataLoaded: boolean;
  loadError: string;

  dimensions?: List<Dimension>;
  measures?: List<Measure>;
  timeAttribute?: Expression;
  defaultSortOn?: string;

  executor?: Executor;
}

export interface DataSourceJS {
  name: string;
  title: string;
  source: string;

  dimensions?: DimensionJS[];
  measures?: MeasureJS[];
  timeAttribute?: ExpressionJS;
  defaultSortOn?: string;
}

var check: ImmutableClass<DataSourceValue, DataSourceJS>;
export class DataSource implements ImmutableInstance<DataSourceValue, DataSourceJS> {

  public name: string;
  public title: string;
  public source: string;
  public metadataLoaded: boolean;
  public loadError: string;

  public dimensions: List<Dimension>;
  public measures: List<Measure>;
  public timeAttribute: Expression;
  public defaultSortOn: string;

  public executor: Executor;

  static isDataSource(candidate: any): boolean {
    return isInstanceOf(candidate, DataSource);
  }

  static fromQueryURL(name: string, title: string, source: string, attributes: Attributes): DataSource {
    var dm = makeDimensionsMetricsFromAttributes(attributes);
    return new DataSource({
      name,
      title,
      source,
      metadataLoaded: true,
      loadError: null,
      dimensions: dm.dimensions,
      measures: dm.measures,
      timeAttribute: dm.timeAttribute,
      defaultSortOn: dm.defaultSortOn,
      executor: queryUrlExecutorFactory(source)
    });
  }

  static fromDataFileURL(name: string, title: string, source: string): DataSource {
    return new DataSource({
      name,
      title,<commit_msg>one less magic constant<commit_after>'use strict';

import { List } from 'immutable';
import * as d3 from 'd3';
import * as Q from 'q';
import * as Qajax from 'qajax';
import { ImmutableClass, ImmutableInstance, isInstanceOf, arraysEqual } from 'higher-object';
import { $, Expression, ExpressionJS, Executor, basicExecutorFactory, Dataset, Datum, Attributes, AttributeInfo, ChainExpression } from 'plywood';
import { upperCaseFirst, listsEqual } from '../../utils/general';
import { Dimension, DimensionJS } from '../dimension/dimension';
import { Measure, MeasureJS } from '../measure/measure';

function getSplitsDescription(ex: Expression): string {
  var splits: string[] = [];
  ex.forEach((ex) => {
    if (ex instanceof ChainExpression) {
      ex.actions.forEach((action) => {
        if (action.action === 'split') splits.push(action.expression.toString());
      });
    }
  });
  return splits.join(';');
}

function queryUrlExecutorFactory(name: string, url: string): Executor {
  return (ex: Expression) => {
    return Qajax({
      method: "POST",
      url: url + '?by=' + getSplitsDescription(ex),
      data: {
        dataset: name,
        expression: ex.toJS()
      }
    })
      .then(Qajax.filterSuccess)
      .then(Qajax.toJSON)
      .then((dataJS) => Dataset.fromJS(dataJS));
  };
}

function dimensionPreference(dimension: Dimension): number {
  if (dimension.type === 'TIME') return 0;
  return 1;
}

function measurePreference(measure: Measure): number {
  if (measure.name === 'count') return 0;
  return 1;
}

interface DimensionsMetrics {
  dimensions: List<Dimension>;
  measures: List<Measure>;
  timeAttribute: Expression;
  defaultSortOn: string;
}

function makeDimensionsMetricsFromAttributes(attributes: Attributes): DimensionsMetrics {
  var dimensionArray: Dimension[] = [];
  var measureArray: Measure[] = [];
  var timeAttribute = $('time');
  var defaultSortOn: string = null;

  if (!attributes['count']) {
    measureArray.push(new Measure({
      name: 'count',
      title: 'Count',
      expression: $('main').count(),
      format: Measure.DEFAULT_FORMAT
    }));
    defaultSortOn = 'count';
  }

  for (let k in attributes) {
    let type = attributes[k].type;
    switch (type) {
      case 'TIME':
      case 'STRING':
        dimensionArray.push(new Dimension({
          name: k,
          title: upperCaseFirst(k),
          expression: $(k),
          type: type,
          sortOn: null
        }));
        break;

      case 'NUMBER':
        measureArray.push(new Measure({
          name: k,
          title: upperCaseFirst(k),
          expression: $('main').sum($(k)),
          format: Measure.DEFAULT_FORMAT
        }));
        break;

      default:
        throw new Error('bad type ' + type);
    }
  }

  dimensionArray.sort((a, b) => {
    var prefDiff = dimensionPreference(a) - dimensionPreference(b);
    if (prefDiff) return prefDiff;
    return a.title.localeCompare(b.title);
  });

  measureArray.sort((a, b) => {
    var prefDiff = measurePreference(a) - measurePreference(b);
    if (prefDiff) return prefDiff;
    return a.title.localeCompare(b.title);
  });

  if (!defaultSortOn && measureArray.length) {
    defaultSortOn = measureArray[0].name;
  }

  return {
    dimensions: List(dimensionArray),
    measures: List(measureArray),
    timeAttribute,
    defaultSortOn
  };
}


export interface DataSourceValue {
  name: string;
  title: string;
  source: string;
  metadataLoaded: boolean;
  loadError: string;

  dimensions?: List<Dimension>;
  measures?: List<Measure>;
  timeAttribute?: Expression;
  defaultSortOn?: string;

  executor?: Executor;
}

export interface DataSourceJS {
  name: string;
  title: string;
  source: string;

  dimensions?: DimensionJS[];
  measures?: MeasureJS[];
  timeAttribute?: ExpressionJS;
  defaultSortOn?: string;
}

var check: ImmutableClass<DataSourceValue, DataSourceJS>;
export class DataSource implements ImmutableInstance<DataSourceValue, DataSourceJS> {

  public name: string;
  public title: string;
  public source: string;
  public metadataLoaded: boolean;
  public loadError: string;

  public dimensions: List<Dimension>;
  public measures: List<Measure>;
  public timeAttribute: Expression;
  public defaultSortOn: string;

  public executor: Executor;

  static isDataSource(candidate: any): boolean {
    return isInstanceOf(candidate, DataSource);
  }

  static fromQueryURL(name: string, title: string, source: string, attributes: Attributes): DataSource {
    var dm = makeDimensionsMetricsFromAttributes(attributes);
    return new DataSource({
      name,
      title,
      source,
      metadataLoaded: true,
      loadError: null,
      dimensions: dm.dimensions,
      measures: dm.measures,
      timeAttribute: dm.timeAttribute,
      defaultSortOn: dm.defaultSortOn,
      executor: queryUrlExecutorFactory(name, source)
    });
  }

  static fromDataFileURL(name: string, title: string, source: string): DataSource {
    return new DataSource({
      name,
      title,
{'fraction_non_alphanumeric': 0.1267198404785643, 'fraction_numerical': 0.0013958125623130609, 'mean_word_length': 3.428697571743929, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3386455', 'n_tokens_mistral': 2979, 'n_tokens_neox': 2912, 'n_words': 757}