<commit_before>#!/bin/sh

set -euo pipefail

PLATFORM=$(uname)

declare -a EXTRA_INCLUDES

if [[ ${PLATFORM} = 'Darwin' ]]; then
  CLANGLIB=${PREFIX}/lib/libclang.dylib
  # EXTRA_INCLUDES+=(-i ${PREFIX}/include/c++/v1/)
elif [[ ${PLATFORM} = 'Linux' ]]; then
  CLANGLIB=${PREFIX}/lib/libclang.so
  # EXTRA_INCLUDES+=(-i ${PREFIX}/x86_64-conda-linux-gnu/include/c++/11.2.0/)
fi

export PYTHONPATH=${SRC_DIR}/pywrap/
${PYTHON} -m bindgen -n ${CPU_COUNT} \
	-i ${PREFIX}/include/ \
	-i ${PREFIX}/include/vtk-9.0/ \
	-i ${PREFIX}/lib/clang/10.0.1/include/ \
	"${EXTRA_INCLUDES[@]}" \
	-l ${CLANGLIB} \
	-v parse ocp.toml out.pkl
${PYTHON} -m bindgen -n ${CPU_COUNT} -v transform ocp.toml out.pkl out_f.pkl<commit_msg>Clean up<commit_after>#!/bin/sh

set -euo pipefail

PLATFORM=$(uname)


if [[ ${PLATFORM} = 'Darwin' ]]; then
  CLANGLIB=${PREFIX}/lib/libclang.dylib
elif [[ ${PLATFORM} = 'Linux' ]]; then
  CLANGLIB=${PREFIX}/lib/libclang.so
fi

export PYTHONPATH=${SRC_DIR}/pywrap/
${PYTHON} -m bindgen -n ${CPU_COUNT} \
	-i ${PREFIX}/include/ \
	-i ${PREFIX}/include/vtk-9.0/ \
	-i ${PREFIX}/lib/clang/10.0.1/include/ \
	-l ${CLANGLIB} \
	-v parse ocp.toml out.pkl
${PYTHON} -m bindgen -n ${CPU_COUNT} -v transform ocp.toml out.pkl out_f.pkl
{'fraction_non_alphanumeric': 0.21774193548387097, 'fraction_numerical': 0.016935483870967744, 'mean_word_length': 3.7007575757575757, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6938064', 'n_tokens_mistral': 585, 'n_tokens_neox': 567, 'n_words': 98}