<commit_before>defmodule Diff do
  @moduledoc """
  Utilities for comparing build artifacts.
  """

  @known_chunks ~w(
    abstract_code
    debug_info
    attributes
    compile_info
    exports
    labeled_exports
    imports
    indexed_imports
    locals
    labeled_locals
    atoms
  )a

  @doc """
  Compares the build artififacts of two build directories.
  """
  @spec compare_dirs(Path.t(), Path.t()) ::
          {
            only1_paths :: list(Path.t()),
            only2_paths :: list(Path.t()),
            diff :: list({Path.t(), diff :: String.t()})
          }
  def compare_dirs(dir1, dir2) do
    dir1 = Path.expand(dir1)
    dir2 = Path.expand(dir2)

    assert_dir!(dir1)
    assert_dir!(dir2)

    dir1_paths = relative_paths(dir1)
    dir2_paths = relative_paths(dir2)

    only1_paths = dir1_paths -- dir2_paths
    only2_paths = dir2_paths -- dir1_paths
    common_paths = dir1_paths -- only1_paths
    common_files = Enum.reject(common_paths, &File.dir?/1)

    diff =<commit_msg>Fix typo in Diff doc (#10304)<commit_after>defmodule Diff do
  @moduledoc """
  Utilities for comparing build artifacts.
  """

  @known_chunks ~w(
    abstract_code
    debug_info
    attributes
    compile_info
    exports
    labeled_exports
    imports
    indexed_imports
    locals
    labeled_locals
    atoms
  )a

  @doc """
  Compares the build artifacts of two build directories.
  """
  @spec compare_dirs(Path.t(), Path.t()) ::
          {
            only1_paths :: list(Path.t()),
            only2_paths :: list(Path.t()),
            diff :: list({Path.t(), diff :: String.t()})
          }
  def compare_dirs(dir1, dir2) do
    dir1 = Path.expand(dir1)
    dir2 = Path.expand(dir2)

    assert_dir!(dir1)
    assert_dir!(dir2)

    dir1_paths = relative_paths(dir1)
    dir2_paths = relative_paths(dir2)

    only1_paths = dir1_paths -- dir2_paths
    only2_paths = dir2_paths -- dir1_paths
    common_paths = dir1_paths -- only1_paths
    common_files = Enum.reject(common_paths, &File.dir?/1)

    diff =
{'fraction_non_alphanumeric': 0.1096361848574238, 'fraction_numerical': 0.025073746312684365, 'mean_word_length': 2.5827464788732395, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3470684', 'n_tokens_mistral': 711, 'n_tokens_neox': 692, 'n_words': 145}