<commit_before>import React from "react";
import _fp, {
  filter,
  groupBy,
  set,
  sortBy,
  partition,
  first,
  get,
  compact,
} from "lodash/fp";
import NarrowContainer from "~/components/layout/NarrowContainer";
import DataTable from "~/components/visualizations/table/DataTable";
import { getOfficialMetadataFields } from "~/api/metadata";
import { getGroupIndex } from "./constants";
import { Dropdown } from "~ui/controls/dropdowns";
import cs from "./metadata_dictionary.scss";
import { getAllHostGenomes } from "../../../api";

const map = _fp.map.convert({ cap: false });

const dictionaryHeaders = {
  name: "Name",
  description: "Description",
  examples: "Examples",
};

const getExamplesForHostGenome = (field, hostGenomeId) =>
  compact(get(hostGenomeId, field.examples) || get("all", field.examples)).join(
    ", "
  ) || "--";

class MetadataDictionary extends React.Component {
  state = {
    officialFields: null,
    hostGenomes: null,
    currentHostGenome: null,
  };

  async componentDidMount() {
    let [officialFields, hostGenomes] = await Promise.all([
      getOfficialMetadataFields(),
      getAllHostGenomes(),
    ]);

    officialFields = map(
      field =>
        set("options", field.options ? field.options.join(", ") : "", field),
      officialFields
    );

    this.setState({
      officialFields,
      hostGenomes,
      currentHostGenome: get("id", first(hostGenomes)),
    });
  }

  processFields = fields => {
    const processedFields = map(
      field =>
        set(
          "examples",
          getExamplesForHostGenome(field, this.state.currentHostGenome),
          field
        ),
      fields
    );

    // Move required fields to the front, and alphabetically sort both sets.
    const [requiredFields, nonrequiredFields] = partition(
      "is_required",
      processedFields
    );

    return [
      ...map(
        field =>
          set("name", <div className={cs.required}>{field.name}*</div>, field),
        sortBy("name", requiredFields)
      ),
      ...sortBy("name", nonrequiredFields),
    ];
  };

  // Get the field groups for the current host genome. Sort by pre-defined order in constants.
  getFieldGroups = () => {
    if (!this.state.officialFields) return null;

    const fieldsForHostGenome = filter(
      field => field.host_genome_ids.includes(this.state.currentHostGenome),
      this.state.officialFields
    );

    const groups = map(
      (fields, name) => ({
        name,
        fields: this.processFields(fields),
      }),
      groupBy("group", fieldsForHostGenome)
    );

    return sortBy(group => getGroupIndex(group.name), groups);
  };

  handleHostGenomeChange = id => {
    this.setState({
      currentHostGenome: id,
    });
  };

  getHostGenomeOptions = () =>
    sortBy(
      "text",
      this.state.hostGenomes.map(hostGenome => ({
        text: hostGenome.name,
        value: hostGenome.id,
      }))
    );

  getColumnWidth = column => (column === "name" ? 250 : "");

  render() {
    const fieldGroupsToDisplay = this.getFieldGroups();
    return (
      <NarrowContainer className={cs.metadataDictionary}>
        <div className={cs.title}>Metadata Dictionary</div>
        {!fieldGroupsToDisplay && <div>Loading...</div>}
        {fieldGroupsToDisplay && (
          <div>
            <div>* = Required</div>
            <Dropdown
              className={cs.hostGenome}
              label="Host Organism"
              options={this.getHostGenomeOptions()}
              value={this.state.currentHostGenome}
              onChange={this.handleHostGenomeChange}
              rounded
            />
          </div>
        )}
        {fieldGroupsToDisplay &&
          map(
            fieldGroup => (
              <div className={cs.metadataGroup} key={fieldGroup.name}>
                <div className={cs.groupName}>{fieldGroup.name} Fields</div>
                <DataTable
                  data={fieldGroup.fields}
                  headers={dictionaryHeaders}
                  columns={["name", "description", "examples"]}
                  getColumnWidth={this.getColumnWidth}
                />
              </div>
            ),
            fieldGroupsToDisplay
          )}
      </NarrowContainer>
    );
  }
}

export default MetadataDictionary;
<commit_msg>Filter out user created host genomes from metadata dict (#3119)<commit_after>import React from "react";
import _fp, {
  filter,
  groupBy,
  set,
  sortBy,
  partition,
  first,
  get,
  compact,
} from "lodash/fp";
import NarrowContainer from "~/components/layout/NarrowContainer";
import DataTable from "~/components/visualizations/table/DataTable";
import { getOfficialMetadataFields } from "~/api/metadata";
import { getGroupIndex } from "./constants";
import { Dropdown } from "~ui/controls/dropdowns";
import cs from "./metadata_dictionary.scss";
import { getAllHostGenomes } from "../../../api";

const map = _fp.map.convert({ cap: false });

const dictionaryHeaders = {
  name: "Name",
  description: "Description",
  examples: "Examples",
};

const getExamplesForHostGenome = (field, hostGenomeId) =>
  compact(get(hostGenomeId, field.examples) || get("all", field.examples)).join(
    ", "
  ) || "--";

class MetadataDictionary extends React.Component {
  state = {
    officialFields: null,
    hostGenomes: null,
    currentHostGenome: null,
  };

  async componentDidMount() {
    let [officialFields, hostGenomes] = await Promise.all([
      getOfficialMetadataFields(),
      getAllHostGenomes(),
    ]);

    officialFields = map(
      field =>
        set("options", field.options ? field.options.join(", ") : "", field),
      officialFields
    );

    hostGenomes = hostGenomes.filter(hg => hg.showAsOption);

    this.setState({
      officialFields,
      hostGenomes,
      currentHostGenome: get("id", first(hostGenomes)),
    });
  }

  processFields = fields => {
    const processedFields = map(
      field =>
        set(
          "examples",
          getExamplesForHostGenome(field, this.state.currentHostGenome),
          field
        ),
      fields
    );

    // Move required fields to the front, and alphabetically sort both sets.
    const [requiredFields, nonrequiredFields] = partition(
      "is_required",
      processedFields
    );

    return [
      ...map(
        field =>
          set("name", <div className={cs.required}>{field.name}*</div>, field),
        sortBy("name", requiredFields)
      ),
      ...sortBy("name", nonrequiredFields),
    ];
  };

  // Get the field groups for the current host genome. Sort by pre-defined order in constants.
  getFieldGroups = () => {
    if (!this.state.officialFields) return null;

    const fieldsForHostGenome = filter(
      field => field.host_genome_ids.includes(this.state.currentHostGenome),
      this.state.officialFields
    );

    const groups = map(
      (fields, name) => ({
        name,
        fields: this.processFields(fields),
      }),
      groupBy("group", fieldsForHostGenome)
    );

    return sortBy(group => getGroupIndex(group.name), groups);
  };

  handleHostGenomeChange = id => {
    this.setState({
      currentHostGenome: id,
    });
  };

  getHostGenomeOptions = () =>
    sortBy(
      "text",
      this.state.hostGenomes.map(hostGenome => ({
        text: hostGenome.name,
        value: hostGenome.id,
      }))
    );

  getColumnWidth = column => (column === "name" ? 250 : "");

  render() {
    const fieldGroupsToDisplay = this.getFieldGroups();
    return (
      <NarrowContainer className={cs.metadataDictionary}>
        <div className={cs.title}>Metadata Dictionary</div>
        {!fieldGroupsToDisplay && <div>Loading...</div>}
        {fieldGroupsToDisplay && (
          <div>
            <div>* = Required</div>
            <Dropdown
              className={cs.hostGenome}
              label="Host Organism"
              options={this.getHostGenomeOptions()}
              value={this.state.currentHostGenome}
              onChange={this.handleHostGenomeChange}
              rounded
            />
          </div>
        )}
        {fieldGroupsToDisplay &&
          map(
            fieldGroup => (
              <div className={cs.metadataGroup} key={fieldGroup.name}>
                <div className={cs.groupName}>{fieldGroup.name} Fields</div>
                <DataTable
                  data={fieldGroup.fields}
                  headers={dictionaryHeaders}
                  columns={["name", "description", "examples"]}
                  getColumnWidth={this.getColumnWidth}
                />
              </div>
            ),
            fieldGroupsToDisplay
          )}
      </NarrowContainer>
    );
  }
}

export default MetadataDictionary;

{'fraction_non_alphanumeric': 0.131753663003663, 'fraction_numerical': 0.0011446886446886447, 'mean_word_length': 2.3578016910069177, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 68, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5043264', 'n_tokens_mistral': 2528, 'n_tokens_neox': 2438, 'n_words': 500}