<commit_before>#!/bin/bash

# IFS=','
# wget –no-check-certificate -q -O - 'https://docs.google.com/spreadsheets/d/1gcAHUEZCFQ83t5LYpzTR6KbDhf6NYLfUZ7hyz_bhT1c/pub?gid=0&single=true&output=csv' | cut -d',' -f1-5 | read -a
# while read -a line; do
#     for i in "${!line[@]}"; do
#         echo "${headers[i]}: ${line[i]}"
#     done
# done


# INPUT_FILE='unix_file.csv'

# IFS=','

# while read OS HS
# do

# echo "Operating system - $OS"
# echo "Hosting server type - $HS"

# done < $INPUT_FILE

COUNT=0

# get URLs and produce thumbnails

wget –no-check-certificate -q -O - 'https://docs.google.com/spreadsheets/d/1gcAHUEZCFQ83t5LYpzTR6KbDhf6NYLfUZ7hyz_bhT1c/pub?gid=0&single=true&output=csv' | cut -d',' -f1-5 | tail -n +2 | while IFS=, read Subdomain Number Name URL Printed Action Notes
  do
    # read col1 col2 < <(echo $line)
    if [ -n "$Subdomain" ] && [ -n "$Number" ] && [ -n "$Name" ] && [ -n "$URL" ]
    then
      let COUNT=COUNT+1
      RENAME="${Name//[^\x00-\x7F]/_}"
      FILENAME="${Subdomain}-${Number}_${RENAME}"
      echo "($COUNT) Creating: ${FILENAME}"
      webkit2png --ignore-ssl-check -C --delay=2 --clipwidth=300 --clipheight=300 ${URL} -o "./images/${FILENAME}"
      mv "./images/${FILENAME}-clipped.png" "./images/${FILENAME}.png"
      convert ./images/${FILENAME}.png \
          -gravity South   -background White  -splice 0x40 \
          -fill Black -draw 'line 0,600 600,600' \
          -pointsize 18 -annotate +0+2 "$FILENAME" \
          -gravity Center -append ./images/${FILENAME}.png
      echo ""
    fi
  done

# assemble final PDF
SUFFIX=$(date +%s)
convert 'images/*.png' page_index_${SUFFIX}.pdf

<commit_msg>:space_invader: more fixing and truncating filenames<commit_after>#!/bin/bash

# IFS=','
# wget –no-check-certificate -q -O - 'https://docs.google.com/spreadsheets/d/1gcAHUEZCFQ83t5LYpzTR6KbDhf6NYLfUZ7hyz_bhT1c/pub?gid=0&single=true&output=csv' | cut -d',' -f1-5 | read -a
# while read -a line; do
#     for i in "${!line[@]}"; do
#         echo "${headers[i]}: ${line[i]}"
#     done
# done


# INPUT_FILE='unix_file.csv'

# IFS=','

# while read OS HS
# do

# echo "Operating system - $OS"
# echo "Hosting server type - $HS"

# done < $INPUT_FILE

COUNT=0

# get URLs and produce thumbnails

wget –no-check-certificate -q -O - 'https://docs.google.com/spreadsheets/d/1gcAHUEZCFQ83t5LYpzTR6KbDhf6NYLfUZ7hyz_bhT1c/pub?gid=0&single=true&output=csv' | cut -d',' -f1-5 | tail -n +2 | while IFS=, read Subdomain Number Name URL Printed Action Notes
  do
    # read col1 col2 < <(echo $line)
    if [ -n "$Subdomain" ] && [ -n "$Number" ] && [ -n "$Name" ] && [ -n "$URL" ]
    then
      let COUNT=COUNT+1
      RENAME="${Name//[^\x00-\x7F]/_}" # for non ascii stuff
      RENAME="${RENAME// /_}" # spaces suck
      RENAME="${RENAME//./_}" # also periods
      RENAME=`echo $RENAME | cut -c 1-30` # truncate the name
      FILENAME="${Subdomain}-${Number}_${RENAME}"
      echo "($COUNT) Creating: ${FILENAME}"
      webkit2png --ignore-ssl-check -C --delay=2 --clipwidth=300 --clipheight=300 ${URL} -o "./images/${FILENAME}"
      mv "./images/${FILENAME}-clipped.png" "./images/${FILENAME}.png"
      convert ./images/${FILENAME}.png \
          -gravity South   -background White  -splice 0x40 \
          -fill Black -draw 'line 0,600 600,600' \
          -pointsize 18 -annotate +0+2 "$FILENAME" \
          -gravity Center -append ./images/${FILENAME}.png
      echo ""
    fi
  done

# assemble final PDF
SUFFIX=$(date +%s)
convert 'images/*.png' page_index_${SUFFIX}.pdf


{'fraction_non_alphanumeric': 0.1852060982495765, 'fraction_numerical': 0.03190287972896669, 'mean_word_length': 2.9149171270718233, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4784501', 'n_tokens_mistral': 1450, 'n_tokens_neox': 1361, 'n_words': 307}