<commit_before>#!/usr/bin/env bash

# filter_wbench.sh
# 
#
# Created by <NAME> on 15/10/2014.
# Copyright 2014 ITQB / UNL. All rights reserved.

# call filter_wbench.sh [file] [source]

#rename variables
file=$1
source=$2



# read softwares dir
. ${source}"/config/software_dirs.cfg"
. ${source}"/config/workdirs.cfg"

if [[ $HEADLESS == "TRUE" ]]; then
  xserv=${SOURCE}"/./xvfb-run-safe"
fi

 
# configuration file path
CFG=${source}"/config/wbench_filter_in_use.cfg"

# define input directory and get input filename(s)
IN_FILE=$(basename ${file})
IN_DIR=$(dirname ${file})
IN_ROOT=${IN_FILE%.*}
<commit_msg>Changed SOURCE to source in filter_wbench<commit_after>#!/usr/bin/env bash

# filter_wbench.sh
# 
#
# Created by <NAME> on 15/10/2014.
# Copyright 2014 ITQB / UNL. All rights reserved.

# call filter_wbench.sh [file] [source]

#rename variables
file=$1
source=$2



# read softwares dir
. ${source}"/config/software_dirs.cfg"
. ${source}"/config/workdirs.cfg"

if [[ $HEADLESS == "TRUE" ]]; then
  xserv=${source}"/./xvfb-run-safe"
fi

 
# configuration file path
CFG=${source}"/config/wbench_filter_in_use.cfg"

# define input directory and get input filename(s)
IN_FILE=$(basename ${file})
IN_DIR=$(dirname ${file})
IN_ROOT=${IN_FILE%.*}

{'fraction_non_alphanumeric': 0.17092651757188498, 'fraction_numerical': 0.022364217252396165, 'mean_word_length': 4.695454545454545, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3274265', 'n_tokens_mistral': 500, 'n_tokens_neox': 467, 'n_words': 105}