<commit_before>import * as utils from "base/utils";
import Model from "base/model";

/*!
 * HOOK MODEL
 */

const Marker = Model.extend({

  getClassDefaults() {
    const defaults = {
      select: [],
      highlight: [],
      superHighlight: [],
      opacityHighlightDim: 0.1,
      opacitySelectDim: 0.3,
      opacityRegular: 1,
      allowSelectMultiple: true,
      skipFilter: false
    };
    return utils.deepExtend(this._super(), defaults);
  },

  init(name, value, parent, binds, persistent) {
    const _this = this;

    this._type = "marker";
    this._visible = [];

    this._super(name, value, parent, binds, persistent);
    this.on("readyOnce", () => {
      const exceptions = { exceptType: "time" };
      const allDimensions = _this._getAllDimensions(exceptions);
      _this._multiDim = allDimensions.length > 1;
    });
    this.on("change", "space", this.updateSpaceReferences.bind(this));
    this.updateSpaceReferences();
  },

  updateSpaceReferences() {
    utils.forEach(this.getSpace(), dimensionModel => {
      // make reference to dimension
      this._space[dimensionModel] = this.getClosestModel(dimensionModel);
    });
  },

  setSpace(newSpace) {
    this.space = this._root.dimensionManager.getDimensionModelsForSpace(this._space, newSpace);
  },

  getAvailableSpaces() {
    const spaces = new Map();
    utils.forEach(this._root._data, dataSource => {
      if (dataSource._type !== "data") return;

      const indicatorsDB = dataSource.getConceptprops();

      dataSource.keyAvailability.forEach((space, str) => {
        if (space.length == this.space.length) { // only same dimension as marker already has for now. Supported dimensions might later depend on tool.
          spaces.set(str, space.map(dimension => indicatorsDB[dimension]));
        }
      });
    });
    return spaces;
  },

  getAvailableData() {
    const data = [];

    if (d3.keys(this._space).length === 0) return utils.warn("getAvailableData() is trying to access missing _space items of marker '" + this._name + "' which likely haven't been resoled in time");
    const dimensions = this.space.map(dim => this._space[dim].dim);

    utils.forEach(this._root._data, dataSource => {
      if (dataSource._type !== "data") return;

      const indicatorsDB = dataSource.getConceptprops();

      dataSource.dataAvailability.datapoints.forEach(kvPair => {
        if (dimensions.length == kvPair.key.size && dimensions.every(dim => kvPair.key.has(dim))) {
          data.push({
            key: kvPair.key,
            value: indicatorsDB[kvPair.value],
            dataSource
          });
        }
      });

      // get all available entity properties for current marker space
      const entitiesAvailability = [];
      dataSource.dataAvailability.entities.forEach(kvPair => {
        if (kvPair.value == null) return;
        dimensions.forEach(dim => {
          if (kvPair.key.has(dim) && kvPair.value.indexOf("is--") === -1) {
            data.push({
              key: Array.from(kvPair.key).map(concept => indicatorsDB[concept]),
              value: indicatorsDB[kvPair.value],
              dataSource
            });
          }
        });
      });

    });

    // just first dataModel, can lead to problems if first data source doesn't contain dim-concept
    const firstDataModel = this._root.dataManager.getDataModels().values().next().value;
    dimensions.forEach(dim => data.push({
      key: [firstDataModel.getConceptprops(dim)],
      value: firstDataModel.getConceptprops(dim),
      dataSource: firstDataModel
    }));
    data.push({
      key: [firstDataModel.getConceptprops("_default")],
      value: firstDataModel.getConceptprops("_default"),
      dataSource: firstDataModel
    });

    return data;
  },


  getAvailableConcept({ index: index = 0, type: type = null, includeOnlyIDs: includeOnlyIDs = [], excludeIDs: excludeIDs = [] } = { }) {
    if (!type && includeOnlyIDs.length == 0 && excludeIDs.length == 0) {
      return null;
    }

    const filtered = this.getAvailableData().filter(f =>
      (!type || !f.value.concept_type || f.value.concept_type === type)
      && (includeOnlyIDs.length == 0 || includeOnlyIDs.indexOf(f.value.concept) !== -1)
      && (excludeIDs.length == 0 || excludeIDs.indexOf(f.value.concept) == -1)
    );
    return filtered[index] || filtered[filtered.length - 1];
  },
<commit_msg>Ignore null dimensions when doing marker.getAvailableData(). Fixes popbyage errors on startup<commit_after>import * as utils from "base/utils";
import Model from "base/model";

/*!
 * HOOK MODEL
 */

const Marker = Model.extend({

  getClassDefaults() {
    const defaults = {
      select: [],
      highlight: [],
      superHighlight: [],
      opacityHighlightDim: 0.1,
      opacitySelectDim: 0.3,
      opacityRegular: 1,
      allowSelectMultiple: true,
      skipFilter: false
    };
    return utils.deepExtend(this._super(), defaults);
  },

  init(name, value, parent, binds, persistent) {
    const _this = this;

    this._type = "marker";
    this._visible = [];

    this._super(name, value, parent, binds, persistent);
    this.on("readyOnce", () => {
      const exceptions = { exceptType: "time" };
      const allDimensions = _this._getAllDimensions(exceptions);
      _this._multiDim = allDimensions.length > 1;
    });
    this.on("change", "space", this.updateSpaceReferences.bind(this));
    this.updateSpaceReferences();
  },

  updateSpaceReferences() {
    utils.forEach(this.getSpace(), dimensionModel => {
      // make reference to dimension
      this._space[dimensionModel] = this.getClosestModel(dimensionModel);
    });
  },

  setSpace(newSpace) {
    this.space = this._root.dimensionManager.getDimensionModelsForSpace(this._space, newSpace);
  },

  getAvailableSpaces() {
    const spaces = new Map();
    utils.forEach(this._root._data, dataSource => {
      if (dataSource._type !== "data") return;

      const indicatorsDB = dataSource.getConceptprops();

      dataSource.keyAvailability.forEach((space, str) => {
        if (space.length == this.space.length) { // only same dimension as marker already has for now. Supported dimensions might later depend on tool.
          spaces.set(str, space.map(dimension => indicatorsDB[dimension]));
        }
      });
    });
    return spaces;
  },

  getAvailableData() {
    const data = [];

    if (d3.keys(this._space).length === 0) return utils.warn("getAvailableData() is trying to access missing _space items of marker '" + this._name + "' which likely haven't been resoled in time");
    const dimensions = this.space.map(dim => this._space[dim].dim);

    utils.forEach(this._root._data, dataSource => {
      if (dataSource._type !== "data") return;

      const indicatorsDB = dataSource.getConceptprops();

      dataSource.dataAvailability.datapoints.forEach(kvPair => {
        if (dimensions.length == kvPair.key.size && dimensions.every(dim => kvPair.key.has(dim))) {
          data.push({
            key: kvPair.key,
            value: indicatorsDB[kvPair.value],
            dataSource
          });
        }
      });

      // get all available entity properties for current marker space
      const entitiesAvailability = [];
      dataSource.dataAvailability.entities.forEach(kvPair => {
        if (kvPair.value == null) return;
        dimensions.forEach(dim => {
          if (kvPair.key.has(dim) && kvPair.value.indexOf("is--") === -1) {
            data.push({
              key: Array.from(kvPair.key).map(concept => indicatorsDB[concept]),
              value: indicatorsDB[kvPair.value],
              dataSource
            });
          }
        });
      });

    });

    // just first dataModel, can lead to problems if first data source doesn't contain dim-concept
    const firstDataModel = this._root.dataManager.getDataModels().values().next().value;
    dimensions
      .filter(dim => dim != null)
      .forEach(dim => data.push({
        key: [firstDataModel.getConceptprops(dim)],
        value: firstDataModel.getConceptprops(dim),
        dataSource: firstDataModel
      }));
    data.push({
      key: [firstDataModel.getConceptprops("_default")],
      value: firstDataModel.getConceptprops("_default"),
      dataSource: firstDataModel
    });

    return data;
  },


  getAvailableConcept({ index: index = 0, type: type = null, includeOnlyIDs: includeOnlyIDs = [], excludeIDs: excludeIDs = [] } = { }) {
    if (!type && includeOnlyIDs.length == 0 && excludeIDs.length == 0) {
      return null;
    }

    const filtered = this.getAvailableData().filter(f =>
      (!type || !f.value.concept_type || f.value.concept_type === type)
      && (includeOnlyIDs.length == 0 || includeOnlyIDs.indexOf(f.value.concept) !== -1)
      && (excludeIDs.length == 0 || excludeIDs.indexOf(f.value.concept) == -1)
    );
    return filtered[index] || filtered[filtered.length - 1];
  },

{'fraction_non_alphanumeric': 0.14099539377598022, 'fraction_numerical': 0.003819795528592293, 'mean_word_length': 2.833763996554694, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6160508', 'n_tokens_mistral': 2652, 'n_tokens_neox': 2529, 'n_words': 548}