<commit_before>/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

import {runAsyncTask, updateMessage} from './async';
import {DataPoint, DataSet, DatasetMetadata, PointMetadata, MetadataInfo, ColumnStats} from './data';

/** Maximum number of colors supported in the color map. */
const NUM_COLORS_COLOR_MAP = 20;

const METADATA_MSG_ID = 'metadata';
const TENSORS_MSG_ID = 'tensors';

/** Information associated with a tensor. */
export interface TensorInfo {
  /** Name of the tensor. */
  name: string;
  /** The shape of the tensor. */
  shape: [number, number];
  /** The path to the metadata file associated with the tensor. */
  metadataFile: string;
}

/** Information for the model checkpoint. */
export interface CheckpointInfo {
  tensors: {[name: string]: TensorInfo};
  checkpointFile: string;
}

/** Interface between the data storage and the UI. */
export interface DataProvider {
  /** Returns a list of run names that have embedding config files. */
  retrieveRuns(callback: (runs: string[]) => void): void;

  /**
   * Returns info about the checkpoint: number of tensors, their shapes,
   * and their associated metadata files.
   */
  retrieveCheckpointInfo(run: string, callback: (d: CheckpointInfo) => void): void;

  /** Fetches and returns the tensor with the specified name. */
  retrieveTensor(run: string, tensorName: string, callback: (ds: DataSet) => void);

  /**
   * Fetches the metadata for the specified tensor and merges it with the
   * specified data source.
   */
  retrieveMetadata(run: string, tensorName: string,
      callback: (r: MetadataInfo) => void): void;

  /**
   * Returns the name of the tensor that should be fetched by default.
   * Used in demo mode to load a tensor when the app starts. Returns null if no
   * default tensor exists.
   */
  getDefaultTensor(run: string, callback: (tensorName: string) => void): void;
}

/**
 * Data provider that loads data provided by a python server (usually backed
 * by a checkpoint file).
 */
class ServerDataProvider implements DataProvider {
  /** Prefix added to the http requests when asking the server for data. */
  static DEFAULT_ROUTE_PREFIX = 'data';
  private routePrefix: string;

  constructor(routePrefix: string) {
    this.routePrefix = routePrefix;
  }

  retrieveRuns(callback: (runs: string[]) => void): void {
    let msgId = updateMessage('Fetching runs...');
    d3.json(`${this.routePrefix}/runs`, (err, runs) => {
      updateMessage(null, msgId);
      callback(runs);
    });
  }

  retrieveCheckpointInfo(run: string, callback: (d: CheckpointInfo) => void)
      : void {
    let msgId = updateMessage('Fetching checkpoint info...');
    d3.json(`${this.routePrefix}/info?run=${run}`, (err, checkpointInfo) => {
      updateMessage(null, msgId);
      callback(checkpointInfo);
    });
  }

  retrieveTensor(run: string, tensorName: string, callback: (ds: DataSet) => void) {
    // Get the tensor.
    updateMessage('Fetching tensor values...', TENSORS_MSG_ID);
    d3.text(
        `${this.routePrefix}/tensor?run=${run}&name=${tensorName}`,
        (err: Error, tsv: string) => {
          if (err) {
            console.error(err);
            return;
          }
          parseTensors(tsv).then(dataPoints => {
            callback(new DataSet(dataPoints));<commit_msg>Cache the checkpoint info requests.<commit_after>/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

import {runAsyncTask, updateMessage} from './async';
import {DataPoint, DataSet, DatasetMetadata, PointMetadata, MetadataInfo, ColumnStats} from './data';

/** Maximum number of colors supported in the color map. */
const NUM_COLORS_COLOR_MAP = 20;

const METADATA_MSG_ID = 'metadata';
const TENSORS_MSG_ID = 'tensors';

/** Information associated with a tensor. */
export interface TensorInfo {
  /** Name of the tensor. */
  name: string;
  /** The shape of the tensor. */
  shape: [number, number];
  /** The path to the metadata file associated with the tensor. */
  metadataFile: string;
}

/** Information for the model checkpoint. */
export interface CheckpointInfo {
  tensors: {[name: string]: TensorInfo};
  checkpointFile: string;
}

/** Interface between the data storage and the UI. */
export interface DataProvider {
  /** Returns a list of run names that have embedding config files. */
  retrieveRuns(callback: (runs: string[]) => void): void;

  /**
   * Returns info about the checkpoint: number of tensors, their shapes,
   * and their associated metadata files.
   */
  retrieveCheckpointInfo(run: string, callback: (d: CheckpointInfo) => void): void;

  /** Fetches and returns the tensor with the specified name. */
  retrieveTensor(run: string, tensorName: string, callback: (ds: DataSet) => void);

  /**
   * Fetches the metadata for the specified tensor and merges it with the
   * specified data source.
   */
  retrieveMetadata(run: string, tensorName: string,
      callback: (r: MetadataInfo) => void): void;

  /**
   * Returns the name of the tensor that should be fetched by default.
   * Used in demo mode to load a tensor when the app starts. Returns null if no
   * default tensor exists.
   */
  getDefaultTensor(run: string, callback: (tensorName: string) => void): void;
}

/**
 * Data provider that loads data provided by a python server (usually backed
 * by a checkpoint file).
 */
class ServerDataProvider implements DataProvider {
  /** Prefix added to the http requests when asking the server for data. */
  static DEFAULT_ROUTE_PREFIX = 'data';
  private routePrefix: string;
  private runCheckpointInfoCache: {[run: string]: CheckpointInfo} = {};

  constructor(routePrefix: string) {
    this.routePrefix = routePrefix;
  }

  retrieveRuns(callback: (runs: string[]) => void): void {
    let msgId = updateMessage('Fetching runs...');
    d3.json(`${this.routePrefix}/runs`, (err, runs) => {
      updateMessage(null, msgId);
      callback(runs);
    });
  }

  retrieveCheckpointInfo(run: string, callback: (d: CheckpointInfo) => void)
      : void {
    if (run in this.runCheckpointInfoCache) {
      callback(this.runCheckpointInfoCache[run]);
      return;
    }

    let msgId = updateMessage('Fetching checkpoint info...');
    d3.json(`${this.routePrefix}/info?run=${run}`, (err, checkpointInfo) => {
      updateMessage(null, msgId);
      this.runCheckpointInfoCache[run] = checkpointInfo;
      callback(checkpointInfo);
    });
  }

  retrieveTensor(run: string, tensorName: string, callback: (ds: DataSet) => void) {
    // Get the tensor.
    updateMessage('Fetching tensor values...', TENSORS_MSG_ID);
    d3.text(
        `${this.routePrefix}/tensor?run=${run}&name=${tensorName}`,
        (err: Error, tsv: string) => {
          if (err) {
            console.error(err);
            return;
          }
          parseTensors(tsv).then(dataPoints => {
            callback(new DataSet(dataPoints));
{'fraction_non_alphanumeric': 0.13182604398319744, 'fraction_numerical': 0.0032122559920929085, 'mean_word_length': 3.7118742724097786, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2402817', 'n_tokens_mistral': 2263, 'n_tokens_neox': 2215, 'n_words': 812}