<commit_before>{ lib
, fetchPypi
, buildPythonPackage
, isPy3k
, markupsafe
, click
, typecode
, gemfileparser
, pefile
, fingerprints
, spdx-tools
, fasteners
, pycryptodome
, urlpy
, dparse
, jaraco_functools
, pkginfo
, debian-inspector
, extractcode
, ftfy
, pyahocorasick
, colorama
, jsonstreams
, packageurl-python
, pymaven-patch
, nltk
, pygments
, bitarray
, jinja2
, javaproperties
, boolean-py
, license-expression
, extractcode-7z
, extractcode-libarchive
, typecode-libmagic
, pytestCheckHook
}:
buildPythonPackage rec {
  pname = "scancode-toolkit";
  version = "21.6.7";
  disabled = !isPy3k;

  src = fetchPypi {
    inherit pname version;
    sha256 = "056923ce556cd6d402e5cc98567cb0331a1b6349d03ec565f8ce1c9c37f3a783";
  };

  dontConfigure = true;

  # https://github.com/nexB/scancode-toolkit/issues/2501
  # * dparse2 is a "Temp fork for Python 2 support", but pdfminer requires
  # Python 3, so it's "fine" to leave dparse2 unpackaged and use the "normal"
  # version
  # * ftfy was pinned for similar reasons (to support Python 2), but rather than
  # packaging an older version, I figured it would be better to remove the
  # erroneous (at least for our usage) version bound
  # * bitarray's version bound appears to be unnecessary for similar reasons
  postPatch = ''
    substituteInPlace setup.cfg \
      --replace "dparse2" "dparse" \
      --replace "ftfy <  5.0.0" "ftfy" \
      --replace "bitarray >= 0.8.1, < 1.0.0" "bitarray"
  '';

  propagatedBuildInputs = [
    markupsafe<commit_msg>python3Packages.scancode-toolkit: 21.6.7 -> 21.8.4<commit_after>{ lib
, fetchPypi
, buildPythonPackage
, isPy3k
, markupsafe
, click
, typecode
, gemfileparser
, pefile
, fingerprints
, spdx-tools
, fasteners
, pycryptodome
, urlpy
, dparse
, jaraco_functools
, pkginfo
, debian-inspector
, extractcode
, ftfy
, pyahocorasick
, colorama
, jsonstreams
, packageurl-python
, pymaven-patch
, nltk
, pygments
, bitarray
, jinja2
, javaproperties
, boolean-py
, license-expression
, extractcode-7z
, extractcode-libarchive
, typecode-libmagic
, pytestCheckHook
}:
buildPythonPackage rec {
  pname = "scancode-toolkit";
  version = "21.8.4";
  disabled = !isPy3k;

  src = fetchPypi {
    inherit pname version;
    sha256 = "c18340067244274c67e166f701c60e747e1d0bccb17efc99f277a4bc0a5a13c6";
  };

  dontConfigure = true;

  # https://github.com/nexB/scancode-toolkit/issues/2501
  # * dparse2 is a "Temp fork for Python 2 support", but pdfminer requires
  # Python 3, so it's "fine" to leave dparse2 unpackaged and use the "normal"
  # version
  # * ftfy was pinned for similar reasons (to support Python 2), but rather than
  # packaging an older version, I figured it would be better to remove the
  # erroneous (at least for our usage) version bound
  # * bitarray's version bound appears to be unnecessary for similar reasons
  postPatch = ''
    substituteInPlace setup.cfg \
      --replace "dparse2" "dparse" \
      --replace "ftfy <  5.0.0" "ftfy" \
      --replace "bitarray >= 0.8.1, < 1.0.0" "bitarray"
  '';

  propagatedBuildInputs = [
    markupsafe
{'fraction_non_alphanumeric': 0.10217320791436912, 'fraction_numerical': 0.049626986701265, 'mean_word_length': 3.781395348837209, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4439694', 'n_tokens_mistral': 1210, 'n_tokens_neox': 1104, 'n_words': 285}