<commit_before>#!/bin/bash

KEYWORDS_GERMANY="Germany"
KEYWORDS_NAZIGERMANY="Nazi"
KEYWORDS_HITLER="Hitler"
KEYWORDS_NAZIGERMANY_SECONDARY="Hitler(| )youth"
KEYWORDS_HITLER_EXCLUDE="$KEYWORDS_NAZIGERMANY_SECONDARY"
KEYWORDS_BERLIN="Berlin"
KEYWORDS_AUSCHWITZ="Auschwitz|Oświęcim"
KEYWORDS_NAZIGERMANY_EXCLUDE="$KEYWORDS_HITLER|$KEYWORDS_AUSCHWITZ"
KEYWORDS_GERMANY_EXCLUDE="$KEYWORDS_NAZIGERMANY|$KEYWORDS_HITLER|$KEYWORDS_NAZIGERMANY_SECONDARY|$KEYWORDS_AUSCHWITZ|$KEYWORDS_BERLIN"

KEYWORDS_GERMANY_ALL="$KEYWORDS_GERMANY|$KEYWORDS_NAZIGERMANY|$KEYWORDS_HITLER|$KEYWORDS_NAZIGERMANY_SECONDARY|$KEYWORDS_BERLIN|$KEYWORDS_AUSCHWITZ"

if [ "$1" == "" ]; #Normal operation
then
  
  debug_start "Germany"

  GERMANY=$(egrep -i "$KEYWORDS_GERMANY" newpages.txt | egrep -iv "$KEYWORDS_GERMANY_EXCLUDE")
  NAZI=$(egrep -i "$KEYWORDS_NAZIGERMANY" newpages.txt | egrep -iv "$KEYWORDS_NAZIGERMANY_EXCLUDE")
  NAZISECONDARY=$(egrep -i "$KEYWORDS_NAZIGERMANY_SECONDARY" newpages.txt)
  HITLER=$(egrep -i "$KEYWORDS_HITLER" newpages.txt | egrep -iv "$KEYWORDS_HITLER_EXCLUDE")
  BERLIN=$(egrep -i "$KEYWORDS_BERLIN" newpages.txt)

  if [ "$GERMANY" != "" ];
  then
    printf "%s" "$GERMANY" > Germany.txt
    export CATFILE="Germany.txt"
    export CATNAME="Germany"
    $CATEGORIZE
    rm Germany.txt
    unset GERMANY
  fi

  if [ "$NAZI" != "" ] || [ "$NAZISECONDARY" != "" ];
  then
    printf "%s" "$NAZI" > NaziGermany.txt
    printf "%s" "$NAZISECONDARY" >> NaziGermany.txt
    export CATFILE="NaziGermany.txt"
    export CATNAME="N<NAME>"
    $CATEGORIZE
    rm NaziGermany.txt
    unset NAZI
    unset NAZISECONDARY
  fi

  if [ "$HITLER" != "" ];
  then
    printf "%s" "$HITLER" > AdolfHitler.txt
    export CATFILE="AdolfHitler.txt"
    export CATNAME="<NAME>"
    $CATEGORIZE
    rm AdolfHitler.txt
    unset HITLER
  fi

  if [ "$BERLIN" != "" ];
  then
    printf "%s" "$BERLIN" > Berlin.txt
    export CATFILE="Berlin.txt"
    export CATNAME="Berlin"
    $CATEGORIZE
    rm Berlin.txt
    unset BERLIN
  fi

  debug_end "Germany"

fi<commit_msg>Adding <NAME><commit_after>#!/bin/bash

KEYWORDS_GERMANY="Germany"
KEYWORDS_NAZIGERMANY="Nazi"
KEYWORDS_HITLER="Hitler"
KEYWORDS_NAZIGERMANY_SECONDARY="Hitler(| )youth"
KEYWORDS_HITLER_EXCLUDE="$KEYWORDS_NAZIGERMANY_SECONDARY"
KEYWORDS_BERLIN="Berlin"
KEYWORDS_AUSCHWITZ="Auschwitz|Oświęcim"
KEYWORDS_ANNEFRANK="(Anne|Otto)(| )Frank"
KEYWORDS_NAZIGERMANY_EXCLUDE="$KEYWORDS_HITLER|$KEYWORDS_AUSCHWITZ|$KEYWORDS_ANNEFRANK"
KEYWORDS_GERMANY_EXCLUDE="$KEYWORDS_NAZIGERMANY|$KEYWORDS_HITLER|$KEYWORDS_NAZIGERMANY_SECONDARY|$KEYWORDS_AUSCHWITZ|$KEYWORDS_BERLIN"

KEYWORDS_GERMANY_ALL="$KEYWORDS_GERMANY|$KEYWORDS_NAZIGERMANY|$KEYWORDS_HITLER|$KEYWORDS_NAZIGERMANY_SECONDARY|$KEYWORDS_BERLIN|$KEYWORDS_AUSCHWITZ"

if [ "$1" == "" ]; #Normal operation
then
  
  debug_start "Germany"

  GERMANY=$(egrep -i "$KEYWORDS_GERMANY" newpages.txt | egrep -iv "$KEYWORDS_GERMANY_EXCLUDE")
  NAZI=$(egrep -i "$KEYWORDS_NAZIGERMANY" newpages.txt | egrep -iv "$KEYWORDS_NAZIGERMANY_EXCLUDE" && egrep -i "$KEYWORDS_NAZIGERMANY_SECONDARY" newpages.txt)
  HITLER=$(egrep -i "$KEYWORDS_HITLER" newpages.txt | egrep -iv "$KEYWORDS_HITLER_EXCLUDE")
  ANNEFRANK=$(egrep -i "$KEYWORDS_ANNEFRANK" newpages.txt)
  BERLIN=$(egrep -i "$KEYWORDS_BERLIN" newpages.txt)

  if [ "$GERMANY" != "" ];
  then
    printf "%s" "$GERMANY" > Germany.txt
    export CATFILE="Germany.txt"
    export CATNAME="Germany"
    $CATEGORIZE
    rm Germany.txt
    unset GERMANY
  fi

  if [ "$NAZI" != "" ];
  then
    printf "%s" "$NAZI" > NaziGermany.txt
    printf "%s" "$NAZISECONDARY" >> NaziGermany.txt
    export CATFILE="NaziGermany.txt"
    export CATNAME="Nazi Germany"
    $CATEGORIZE
    rm NaziGermany.txt
    unset NAZI
    unset NAZISECONDARY
  fi

  if [ "$HITLER" != "" ];
  then
    printf "%s" "$HITLER" > AdolfHitler.txt
    export CATFILE="AdolfHitler.txt"
    export CATNAME="Adolf Hitler"
    $CATEGORIZE
    rm AdolfHitler.txt
    unset HITLER
  fi

  if [ "$ANNEFRANK" != "" ];
  then
    printf "%s" "$ANNEFRANK" > AnneFrank.txt
    export CATFILE="AnneFrank.txt"
    export CATNAME="<NAME>"
    $CATEGORIZE
    rm AnneFrank.txt
    unset ANNEFRANK
  fi

  if [ "$BERLIN" != "" ];
  then
    printf "%s" "$BERLIN" > Berlin.txt
    export CATFILE="Berlin.txt"
    export CATNAME="Berlin"
    $CATEGORIZE
    rm Berlin.txt
    unset BERLIN
  fi

  debug_end "Germany"

fi
{'fraction_non_alphanumeric': 0.13151308304891923, 'fraction_numerical': 0.0004550625711035267, 'mean_word_length': 4.900671140939597, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6339409', 'n_tokens_mistral': 2001, 'n_tokens_neox': 1892, 'n_words': 276}