<commit_before>#!/bin/bash

if [ $# -ne 2 -a $# -ne 3 ]
then
    echo "Usage: idl-analyze.sh config-file.json tree_name [file_filter]"
    exit 1
fi

set -e # Errors are fatal
set -x # Show commands

CONFIG_FILE=$(realpath $1)
TREE_NAME=$2

MOZSEARCH_PATH=$(cd $(dirname "$0") && git rev-parse --show-toplevel)
. $MOZSEARCH_PATH/scripts/load-vars.sh $CONFIG_FILE $TREE_NAME

FILTER=$3
if [ "x${FILTER}" = "x" ]
then
    FILTER=".*"
fi

rm -rf /tmp/pymodules
mkdir /tmp/pymodules
pushd /tmp/pymodules
wget "https://hg.mozilla.org/mozilla-central/raw-file/tip/xpcom/idl-parser/xpidl/xpidl.py"
mkdir ply
pushd ply
wget "https://hg.mozilla.org/mozilla-central/raw-file/tip/other-licenses/ply/ply/__init__.py"
wget "https://hg.mozilla.org/mozilla-central/raw-file/tip/other-licenses/ply/ply/lex.py"
wget "https://hg.mozilla.org/mozilla-central/raw-file/tip/other-licenses/ply/ply/yacc.py"
popd
popd

export PYTHONPATH=/tmp/pymodules

cat $INDEX_ROOT/idl-files | grep "$FILTER" | \
    parallel python $MOZSEARCH_PATH/idl-analyze.py \
    $INDEX_ROOT $FILES_ROOT/{} ">" $INDEX_ROOT/analysis/{}
echo $?
<commit_msg>Don't download XPIDL parser if not needed<commit_after>#!/bin/bash

if [ $# -ne 2 -a $# -ne 3 ]
then
    echo "Usage: idl-analyze.sh config-file.json tree_name [file_filter]"
    exit 1
fi

set -e # Errors are fatal
set -x # Show commands

CONFIG_FILE=$(realpath $1)
TREE_NAME=$2

MOZSEARCH_PATH=$(cd $(dirname "$0") && git rev-parse --show-toplevel)
. $MOZSEARCH_PATH/scripts/load-vars.sh $CONFIG_FILE $TREE_NAME

FILTER=$3
if [ "x${FILTER}" = "x" ]
then
    FILTER=".*"
fi

if [ ! -d /tmp/pymodules ]
then
    mkdir /tmp/pymodules
    pushd /tmp/pymodules
    wget "https://hg.mozilla.org/mozilla-central/raw-file/tip/xpcom/idl-parser/xpidl/xpidl.py"
    mkdir ply
    pushd ply
    wget "https://hg.mozilla.org/mozilla-central/raw-file/tip/other-licenses/ply/ply/__init__.py"
    wget "https://hg.mozilla.org/mozilla-central/raw-file/tip/other-licenses/ply/ply/lex.py"
    wget "https://hg.mozilla.org/mozilla-central/raw-file/tip/other-licenses/ply/ply/yacc.py"
    popd
    popd
fi

export PYTHONPATH=/tmp/pymodules

cat $INDEX_ROOT/idl-files | grep "$FILTER" | \
    parallel python $MOZSEARCH_PATH/idl-analyze.py \
    $INDEX_ROOT $FILES_ROOT/{} ">" $INDEX_ROOT/analysis/{}
echo $?

{'fraction_non_alphanumeric': 0.1654331737048324, 'fraction_numerical': 0.0060949063996517195, 'mean_word_length': 3.596, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1422496', 'n_tokens_mistral': 985, 'n_tokens_neox': 924, 'n_words': 166}