<commit_before>import * as React from "react";
import { Filters } from "./Filters";
import { createPortal } from "react-dom";
import { SearchBox } from "./SearchBox";
import { SearchTags } from "./SearchTags";

export function Root({
  fields,
  url,
  initialFilterValues,
  initialQuery,
  initialSortBy,
  initialSeed,
  fieldStats,
}) {
  const [query, setQuery] = React.useState(initialQuery);
  const [sortBy, setSortBy] = React.useState(initialSortBy);
  const [seed, setSeed] = React.useState(initialSeed);
  const [filterValues, setFilterValues] = React.useState(initialFilterValues);
  const [isSubmitting, setIsSubmitting] = React.useState(false);
  const onSubmit = React.useCallback(() => setIsSubmitting(true), []);

  const doSubmit = () => {
    let newUrl = `${url}`;
    const addParam = (key, value) => {
      if (value === "" || value === undefined) {
        return;
      }
      if (newUrl.indexOf("?") === -1) {
        newUrl += "?";
      } else {
        newUrl += "&";
      }
      newUrl += `${key}=${encodeURIComponent(value)}`;
    };

    addParam("q", query);
    fields
      .filter((field) => field.availableAsFilter)
      .forEach((field) => {
        const filter = filterValues[field.name];
        switch (field.type) {
          case "integer":
            const args = [];
            if (filter.v.min !== "") {
              args.push(`≥${filter.v.min}`);
            }
            if (filter.v.max !== "") {
              args.push(`≤${filter.v.max}`);
            }
            if (args.length > 0 && filter.includeUnknown === true) {
              args.push("unknown");
            }
            addParam(field.name, args.join("~"));
            break;
          case "string": {
            let value = filter.v
              .map((value) => value.replace(/~/g, "~~"))
              .join("~");
            if (value !== "" && filter.includeUnknown) {
              value += "~unknown~";
            }
            addParam(field.name, value);
            break;
          }
          case "boolean":
            let value = filter.v;
            if (value !== "" && filter.includeUnknown) {
              value += "~unknown";
            }
            addParam(field.name, value);
            break;
          case "text":
          case "url":
          default:
            throw new Error(`Unknown field type ${field.type}`);
        }
      });
    addParam("sortBy", sortBy);
    if (sortBy === "random") {
      addParam("seed", seed);
    }

    document.location.href = newUrl;<commit_msg>Apply "unknown" filter to string fields even if no other filter is selected. (#366)<commit_after>import * as React from "react";
import { Filters } from "./Filters";
import { createPortal } from "react-dom";
import { SearchBox } from "./SearchBox";
import { SearchTags } from "./SearchTags";

export function Root({
  fields,
  url,
  initialFilterValues,
  initialQuery,
  initialSortBy,
  initialSeed,
  fieldStats,
}) {
  const [query, setQuery] = React.useState(initialQuery);
  const [sortBy, setSortBy] = React.useState(initialSortBy);
  const [seed, setSeed] = React.useState(initialSeed);
  const [filterValues, setFilterValues] = React.useState(initialFilterValues);
  const [isSubmitting, setIsSubmitting] = React.useState(false);
  const onSubmit = React.useCallback(() => setIsSubmitting(true), []);

  const doSubmit = () => {
    let newUrl = `${url}`;
    const addParam = (key, value) => {
      if (value === "" || value === undefined) {
        return;
      }
      if (newUrl.indexOf("?") === -1) {
        newUrl += "?";
      } else {
        newUrl += "&";
      }
      newUrl += `${key}=${encodeURIComponent(value)}`;
    };

    addParam("q", query);
    fields
      .filter((field) => field.availableAsFilter)
      .forEach((field) => {
        const filter = filterValues[field.name];
        switch (field.type) {
          case "integer":
            const args = [];
            if (filter.v.min !== "") {
              args.push(`≥${filter.v.min}`);
            }
            if (filter.v.max !== "") {
              args.push(`≤${filter.v.max}`);
            }
            if (args.length > 0 && filter.includeUnknown === true) {
              args.push("unknown");
            }
            addParam(field.name, args.join("~"));
            break;
          case "string": {
            const args = filter.v.map((value) => value.replace(/~/g, "~~"));
            if (filter.includeUnknown) {
              args.push("unknown~");
            }
            addParam(field.name, args.join("~"));
            break;
          }
          case "boolean":
            let value = filter.v;
            if (value !== "" && filter.includeUnknown) {
              value += "~unknown";
            }
            addParam(field.name, value);
            break;
          case "text":
          case "url":
          default:
            throw new Error(`Unknown field type ${field.type}`);
        }
      });
    addParam("sortBy", sortBy);
    if (sortBy === "random") {
      addParam("seed", seed);
    }

    document.location.href = newUrl;
{'fraction_non_alphanumeric': 0.1654620042977144, 'fraction_numerical': 0.0013674545809728462, 'mean_word_length': 1.956120092378753, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5825485', 'n_tokens_mistral': 1503, 'n_tokens_neox': 1496, 'n_words': 320}