<commit_before>#!/bin/sh
# Requires csvgrep, from csvkit (0.9.0)
# Requires csvcut, from csvkit (0.9.0)
# Requires uni2ascii (4.18)
cd "$(dirname "$0")"

year='2015'
baseUrl='https://commons.wikimedia.org/wiki/File:'
extension='.svg'
relativePath='../../../wikimedia.org/'

# select first flag (P41) currently valid (empty end date)
fileName="$(
  csvgrep -c 3 -r '^P41$' ../data.csv | # select flags (P41)
  csvgrep -c 8 -r '^$' | # keep only currently valid flags (empty end date)
  head -n 2 | # keep only header and first record
  tail -n 1 | # keep only first record
  csvcut -c 5
)"
if test -z "$fileName"
then
  echo "Abort: no flag found."
  exit 1
fi

echo "Flag File: $fileName"

# URL with spaces redirects to URL with underscores instead<commit_msg>Avoid matching header record by mistake when no flag is found<commit_after>#!/bin/sh
# Requires csvgrep, from csvkit (0.9.0)
# Requires csvcut, from csvkit (0.9.0)
# Requires uni2ascii (4.18)
cd "$(dirname "$0")"

year='2015'
baseUrl='https://commons.wikimedia.org/wiki/File:'
extension='.svg'
relativePath='../../../wikimedia.org/'

# select first flag (P41) currently valid (empty end date)
fileName="$(
  csvgrep -c 3 -r '^P41$' ../data.csv | # select flags (P41)
  csvgrep -c 8 -r '^$' | # keep only currently valid flags (empty end date)
  tail -n +2 | # remove header
  head -n 1 | # keep only first record
  csvcut -c 5
)"
if test -z "$fileName"
then
  echo "Abort: no flag found."
  exit 1
fi

echo "Flag File: $fileName"

# URL with spaces redirects to URL with underscores instead
{'fraction_non_alphanumeric': 0.1528046421663443, 'fraction_numerical': 0.03481624758220503, 'mean_word_length': 3.5513196480938416, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '424536', 'n_tokens_mistral': 592, 'n_tokens_neox': 550, 'n_words': 199}