<commit_before>#!/bin/bash -ex
export LANG=en_US.UTF-8 LC_COLLATE=C

CWD="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

W2V=all.norm-sz500-w10-cb0-it3-min5.w2v

CLUSTERS=20170122-russian/sz500-k20-l1.0

MODEL=regularized_synonym

for ISAS in $@; do
  if [ ${ISAS:-13} == "-exp-isas.txt" ]; then
    continue
  fi

  EXPANDED=${ISAS%-isas.txt}-exp-isas.txt

  $CWD/../../projlearn/expand.py \
    --w2v=$CWD/../../projlearn/$W2V \
    --kmeans=$CWD/../../projlearn/$CLUSTERS/kmeans.pickle \
    --path=$CWD/../../projlearn/$CLUSTERS \
    --model=$MODEL |
  $CWD/expanded.awk >$EXPANDED
done<commit_msg>Passing a file is a good idea<commit_after>#!/bin/bash -ex
export LANG=en_US.UTF-8 LC_COLLATE=C

CWD="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

W2V=all.norm-sz500-w10-cb0-it3-min5.w2v

CLUSTERS=20170122-russian/sz500-k20-l1.0

MODEL=regularized_synonym

for ISAS in $@; do
  if [ ${ISAS:-13} == "-exp-isas.txt" ]; then
    continue
  fi

  EXPANDED=${ISAS%-isas.txt}-exp-isas.txt

  $CWD/../../projlearn/expand.py \
    --w2v=$CWD/../../projlearn/$W2V \
    --kmeans=$CWD/../../projlearn/$CLUSTERS/kmeans.pickle \
    --path=$CWD/../../projlearn/$CLUSTERS \
    --model=$MODEL "$ISAS" |
  $CWD/expanded.awk >$EXPANDED
done
{'fraction_non_alphanumeric': 0.2297077922077922, 'fraction_numerical': 0.05032467532467533, 'mean_word_length': 2.951923076923077, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3460771', 'n_tokens_mistral': 606, 'n_tokens_neox': 551, 'n_words': 68}