<commit_before>#!/bin/bash
# Set loaded files against the: 
# 001 - JFRC main brain
# 002 - Ito Half Brain
# 003 - Court Adult VNS 
# 004 - Wood Larval Brain 
# template.  
if [ ! -f ./linkData.tsv ]
then
  echo "linkData.tsv file is missing! See linkData.tsv for an example"
else
  export imageDir='/disk/data/VFB/IMAGE_DATA/VFB/i/'
  export templateDir='/disk/data/VFB/IMAGE_DATA/VFB/t/'${template}'/'
  export jsoTemplate='templateForImage.jso'
  cat linkData.tsv | while IFS=$'\t' read -ra VFBI
  do
    label=${VFBI[0]}
    ref=${VFBI[1]}
    if [[ $ref == "VFB_"* ]]
    then
      last=$(echo ${ref} | replace 'VFB_' '' | cut -c 5-)
      first=$(echo ${ref} | replace 'VFB_' '' |  cut -c 1-4)
      cat ${templateDir}${jsoTemplate} | replace 'FFFF' ${first} | replace 'LLLL' ${last} > ${imageDir}${first}/${last}/data.jso 
      echo Created json data file for $ref
    else
      echo WARNING: Not processing $ref, $label
    fi
  done
  echo ------------------------------------------------------
  echo Now run the following comand inside the owl directory of the repository you want to add the files new files to
  echo .
  echo "cat /partition/karenin/VFB/IMAGE_DATA/StackLoader/linkData.tsv | while IFS=$'\t' read -ra VFBI; do last=$(echo ${VFBI[1]} | replace 'VFB_' '' | cut -c 5-) ; first=$(echo ${VFBI[1]} | replace 'VFB_' '' |  cut -c 1-4) ; ref=$(echo ${VFBI[1]} | replace 'VFB_' 'VFBi_' ) ; ln -sf ../data/VFB/i/$first/$last/ ./$ref ; echo $ref ;done"
  echo .
  echo ------------------------------------------------------
fi

<commit_msg>adding short name direct link<commit_after>#!/bin/bash
# Set loaded files against the: 
# 001 - JFRC main brain
# 002 - Ito Half Brain
# 003 - Court Adult VNS 
# 004 - Wood Larval Brain 
# template.  
if [ ! -f ./linkData.tsv ]
then
  echo "linkData.tsv file is missing! See linkData.tsv for an example"
else
  export imageDir='/disk/data/VFB/IMAGE_DATA/VFB/i/'
  export templateDir='/disk/data/VFB/IMAGE_DATA/VFB/t/'${template}'/'
  export jsoTemplate='templateForImage.jso'
  cat linkData.tsv | while IFS=$'\t' read -ra VFBI
  do
    label=${VFBI[0]}
    ref=${VFBI[1]}
    if [[ $ref == "VFB_"* ]]
    then
      last=$(echo ${ref} | replace 'VFB_' '' | cut -c 5-)
      first=$(echo ${ref} | replace 'VFB_' '' |  cut -c 1-4)
      cat ${templateDir}${jsoTemplate} | replace 'FFFF' ${first} | replace 'LLLL' ${last} > ${imageDir}${first}/${last}/data.jso 
      echo Created json data file for $ref
      echo '<html><head><meta HTTP-EQUIV="REFRESH" content="0; url=/site/tools/view_stack/3rdPartyStack.htm?tpbid='${ref}'"></head></html>' > ${imageDir}${first}/${last}/index.html 
      echo Created html link file for $ref
    else
      echo WARNING: Not processing $ref, $label
    fi
  done
  chmod -R 777 ${imageDir}${first}
  echo ------------------------------------------------------
  echo Now run the following comand inside the owl directory of the repository you want to add the files new files to
  echo .
  echo "cat /partition/karenin/VFB/IMAGE_DATA/StackLoader/linkData.tsv | while IFS=$'\t' read -ra VFBI; do last=$(echo ${VFBI[1]} | replace 'VFB_' '' | cut -c 5-) ; first=$(echo ${VFBI[1]} | replace 'VFB_' '' |  cut -c 1-4) ; ref=$(echo ${VFBI[1]} | replace 'VFB_' 'VFBi_' ) ; ln -sf ../data/VFB/i/$first/$last/ ./$ref ; ln -sf ./${ref}/index.html ./${VFBI[1]} ; echo $ref ;done"
  echo .
  echo ------------------------------------------------------
fi


{'fraction_non_alphanumeric': 0.22584400465657742, 'fraction_numerical': 0.015133876600698487, 'mean_word_length': 2.4717171717171715, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3580190', 'n_tokens_mistral': 1244, 'n_tokens_neox': 1192, 'n_words': 343}