<commit_before>import {
  LocalizationProps,
  Localized,
  withLocalization,
} from 'fluent-react/compat';
import * as React from 'react';
import { useState } from 'react';
import { ACCENTS, AGES } from '../../../stores/demographics';
import {
  localeConnector,
  LocalePropsFromState,
  LocalizedGetAttribute,
} from '../../locale-helpers';
import { CloudIcon } from '../../ui/icons';
import {
  Button,
  LabeledCheckbox,
  LabeledInput,
  LabeledSelect,
  LinkButton,
  TextButton,
} from '../../ui/ui';
import CircleStats from './circle-stats';
import stats from './stats';

import './dataset-info.css';

const msToHours = (ms: number) => Math.floor(ms / 1000 / 60 / 60);

const validHours = Object.entries(stats.locales).reduce(
  (obj, [locale, localeStats]) => {
    const avgDuration = localeStats.duration / localeStats.clips;
    console.log(locale, avgDuration);
    obj[locale] = msToHours(localeStats.buckets.validated * avgDuration);
    return obj;
  },
  {} as any
);

const total = msToHours(
  Object.values(stats.locales).reduce((sum, l) => sum + l.duration, 0)
);
const valid = Object.values(validHours).reduce(
  (sum: number, n: number) => sum + n,
  0
) as number;
const languages = Object.keys(stats.locales).length;
const globalStats = { total, valid, languages };

const DEFAULT_CATEGORY_COUNT = 2;

const Splits = ({
  category,
  values,
  locale,
}: {
  category: string;
  values: { [key: string]: number };
  locale: string;
}) => {
  const [expanded, setExpanded] = useState(false);
  return (
    <div key={category} className="splits">
      <LocalizedGetAttribute id={'profile-form-' + category} attribute="label">
        {label => <h5>{label}</h5>}
      </LocalizedGetAttribute>
      <ol onClick={() => setExpanded(!expanded)} tabIndex={0} role="button">
        {Object.entries(values)
          .filter(([key, value]) => key && value > 0)
          .sort((a, b) => (a[1] < b[1] ? 1 : -1))
          .slice(0, expanded ? values.length : DEFAULT_CATEGORY_COUNT)
          .map(([key, value]) => (
            <li key={key}>
              <b>{Math.round(value * 100)}%</b>
              <span> </span>
              {category == 'gender' ? (
                <Localized id={key}>
                  <span />
                </Localized>
              ) : category == 'accent' ? (
                ACCENTS[locale] ? (
                  ACCENTS[locale][key]
                ) : (
                  key
                )
              ) : category == 'age' ? (
                (AGES as any)[key]
              ) : (
                key
              )}
            </li>
          ))}
        {!expanded && Object.keys(values).length > DEFAULT_CATEGORY_COUNT && (
          <li key="more">...</li>
        )}
      </ol>
    </div>
  );
};

type Props = LocalePropsFromState & LocalizationProps;

type State = {
  locale: string;
  showIntroTextMdDown: boolean;
  hideEmailForm: boolean;
  email: string;
  confirmSize: boolean;
  confirmNoIdentify: boolean;
};

class DatasetInfo extends React.Component<Props, State> {
  emailInputRef = React.createRef<HTMLInputElement>();

  constructor(props: Props, context: any) {
    super(props, context);
    this.state = {<commit_msg>don't show "other" accent<commit_after>import {
  LocalizationProps,
  Localized,
  withLocalization,
} from 'fluent-react/compat';
import * as React from 'react';
import { useState } from 'react';
import { ACCENTS, AGES } from '../../../stores/demographics';
import {
  localeConnector,
  LocalePropsFromState,
  LocalizedGetAttribute,
} from '../../locale-helpers';
import { CloudIcon } from '../../ui/icons';
import {
  Button,
  LabeledCheckbox,
  LabeledInput,
  LabeledSelect,
  LinkButton,
  TextButton,
} from '../../ui/ui';
import CircleStats from './circle-stats';
import stats from './stats';

import './dataset-info.css';

const msToHours = (ms: number) => Math.floor(ms / 1000 / 60 / 60);

const validHours = Object.entries(stats.locales).reduce(
  (obj, [locale, localeStats]) => {
    const avgDuration = localeStats.duration / localeStats.clips;
    console.log(locale, avgDuration);
    obj[locale] = msToHours(localeStats.buckets.validated * avgDuration);
    return obj;
  },
  {} as any
);

const total = msToHours(
  Object.values(stats.locales).reduce((sum, l) => sum + l.duration, 0)
);
const valid = Object.values(validHours).reduce(
  (sum: number, n: number) => sum + n,
  0
) as number;
const languages = Object.keys(stats.locales).length;
const globalStats = { total, valid, languages };

const DEFAULT_CATEGORY_COUNT = 2;

const Splits = ({
  category,
  values,
  locale,
}: {
  category: string;
  values: { [key: string]: number };
  locale: string;
}) => {
  const [expanded, setExpanded] = useState(false);
  const categories = Object.entries(values).filter(
    ([key, value]) => key && key != 'other' && value > 0
  );
  return (
    <div key={category} className="splits">
      <LocalizedGetAttribute id={'profile-form-' + category} attribute="label">
        {label => <h5>{label}</h5>}
      </LocalizedGetAttribute>
      <ol onClick={() => setExpanded(!expanded)} tabIndex={0} role="button">
        {categories
          .sort((a, b) => (a[1] < b[1] ? 1 : -1))
          .slice(0, expanded ? categories.length : DEFAULT_CATEGORY_COUNT)
          .map(([key, value]) => (
            <li key={key}>
              <b>{Math.round(value * 100)}%</b>
              <span> </span>
              {category == 'gender' ? (
                <Localized id={key}>
                  <span />
                </Localized>
              ) : category == 'accent' ? (
                ACCENTS[locale] ? (
                  ACCENTS[locale][key]
                ) : (
                  key
                )
              ) : category == 'age' ? (
                (AGES as any)[key]
              ) : (
                key
              )}
            </li>
          ))}
        {!expanded && categories.length > DEFAULT_CATEGORY_COUNT && (
          <li key="more">...</li>
        )}
      </ol>
    </div>
  );
};

type Props = LocalePropsFromState & LocalizationProps;

type State = {
  locale: string;
  showIntroTextMdDown: boolean;
  hideEmailForm: boolean;
  email: string;
  confirmSize: boolean;
  confirmNoIdentify: boolean;
};

class DatasetInfo extends React.Component<Props, State> {
  emailInputRef = React.createRef<HTMLInputElement>();

  constructor(props: Props, context: any) {
    super(props, context);
    this.state = {
{'fraction_non_alphanumeric': 0.15804908164840253, 'fraction_numerical': 0.007099861089674332, 'mean_word_length': 2.4394904458598727, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3827950', 'n_tokens_mistral': 2090, 'n_tokens_neox': 2029, 'n_words': 450}