<commit_before>default:
  # local-only configuration
  sparklyr.cores.local: !expr parallel::detectCores()
  spark.sql.shuffle.partitions.local: !expr parallel::detectCores()
  spark.env.SPARK_LOCAL_IP.local: 127.0.0.1

  # include the embedded csv package for spark 1.x
  sparklyr.csv.embedded: "^1.*"

  # default spark packages to load
  # sparklyr.defaultPackages:

  # command line arguments to spark-shell
  # sparklyr.shell.*

<commit_msg>feat: include driver class path in base config<commit_after>default:
  # local-only configuration
  sparklyr.cores.local: !expr parallel::detectCores()
  spark.sql.shuffle.partitions.local: !expr parallel::detectCores()
  spark.env.SPARK_LOCAL_IP.local: 127.0.0.1

  # include the embedded csv package for spark 1.x
  sparklyr.csv.embedded: "^1.*"

  # default spark packages to load
  # sparklyr.defaultPackages:

  # command line arguments to spark-shell
  # sparklyr.shell.*
  sparklyr.shell.driver-class-path: !Sys.getenv("SPARK_DRIVER_CLASSPATH")

{'fraction_non_alphanumeric': 0.11422845691382766, 'fraction_numerical': 0.01603206412825651, 'mean_word_length': 4.429347826086956, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2859356', 'n_tokens_mistral': 351, 'n_tokens_neox': 330, 'n_words': 75}