<commit_before>#!/bin/bash

#Toyota cars

KEYWORDS_TOYOTA_2000GT="Toyota(| )2000(| )GT|[0-9]{1,}(| )2000(| )GT"
KEYWORDS_TOYOTA_86="Toyota(| )86"
KEYWORDS_TOYOTA_ALLEX="Toyota(| )Allex|[0-9](| )Allex"
KEYWORDS_TOYOTA_CAMRY="Camry"
KEYWORDS_TOYOTA_COROLLA="Corolla|$KEYWORDS_TOYOTA_ALLEX" #The Allex is a rebadged Corolla hatchback
KEYWORDS_TOYOTA_PUBLICA="Toyota(| )(1000|Publica)|[0-9]{1,}(| )Publica"
KEYWORDS_TOYOTA_PRIUS="Prius"
KEYWORDS_TOYOTA_CARS="$KEYWORDS_TOYOTA_2000GT|$KEYWORDS_TOYOTA_86|$KEYWORDS_TOYOTA_ALLEX|$KEYWORDS_TOYOTA_CAMRY|$KEYWORDS_TOYOTA_COROLLA|$KEYWORDS_TOYOTA_PUBLICA|$KEYWORDS_TOYOTA_PRIUS"
KEYWORDS_TOYOTA_CARS_NONPOP="$KEYWORDS_TOYOTA_2000GT|$KEYWORDS_TOYOTA_PUBLICA" #Cars which don't have their own category and should be placed in the parent

#Toyota SUVs

KEYWORDS_TOYOTA_4RUNNER="4Runner| 4(| )Runner\b"


#Toyota vans

KEYWORDS_TOYOTA_ALPHARD="Toyota(| )Alphard|[0-9]{1,}(| )Alphard" #Alphard is also the name of a star, so we can't grab just Alphard


#Search cars

egrep -i "$KEYWORDS_TOYOTA_CAMRY" newpages.txt >> ToyotaCamry.txt
egrep -i "$KEYWORDS_TOYOTA_COROLLA" newpages.txt >> ToyotaCorolla.txt
egrep -i "$KEYWORDS_TOYOTA_PRIUS" newpages.txt >> ToyotaPrius.txt

#Search SUVs

#Search vans

#Stat cars

CAMRY=`stat --print=%s ToyotaCamry.txt`
COROLLA=`stat --print=%s ToyotaCorolla.txt`
PRIUS=`stat --print=%s ToyotaPrius`

#Stat SUVs

#Stat vans

#Categorize cars

if [ $CAMRY -ne 0 ];
then
  export CATFILE="ToyotaCamry.txt"
  export CATNAME="Toyota Camry"
  $CATEGORIZE
fi

if [ $COROLLA -ne 0 ];
then
  export CATFILE="ToyotaCorolla.txt"
  export CATNAME="Toyota Corolla"
  $CATEGORIZE
fi<commit_msg>Minor fix to Toyota Prius<commit_after>#!/bin/bash

#Toyota cars

KEYWORDS_TOYOTA_2000GT="Toyota(| )2000(| )GT|[0-9]{1,}(| )2000(| )GT"
KEYWORDS_TOYOTA_86="Toyota(| )86"
KEYWORDS_TOYOTA_ALLEX="Toyota(| )Allex|[0-9](| )Allex"
KEYWORDS_TOYOTA_CAMRY="Camry"
KEYWORDS_TOYOTA_COROLLA="Corolla|$KEYWORDS_TOYOTA_ALLEX" #The Allex is a rebadged Corolla hatchback
KEYWORDS_TOYOTA_PUBLICA="Toyota(| )(1000|Publica)|[0-9]{1,}(| )Publica"
KEYWORDS_TOYOTA_PRIUS="Prius"
KEYWORDS_TOYOTA_CARS="$KEYWORDS_TOYOTA_2000GT|$KEYWORDS_TOYOTA_86|$KEYWORDS_TOYOTA_ALLEX|$KEYWORDS_TOYOTA_CAMRY|$KEYWORDS_TOYOTA_COROLLA|$KEYWORDS_TOYOTA_PUBLICA|$KEYWORDS_TOYOTA_PRIUS"
KEYWORDS_TOYOTA_CARS_NONPOP="$KEYWORDS_TOYOTA_2000GT|$KEYWORDS_TOYOTA_PUBLICA" #Cars which don't have their own category and should be placed in the parent

#Toyota SUVs

KEYWORDS_TOYOTA_4RUNNER="4Runner| 4(| )Runner\b"


#Toyota vans

KEYWORDS_TOYOTA_ALPHARD="Toyota(| )Alphard|[0-9]{1,}(| )Alphard" #Alphard is also the name of a star, so we can't grab just Alphard


#Search cars

egrep -i "$KEYWORDS_TOYOTA_CAMRY" newpages.txt >> ToyotaCamry.txt
egrep -i "$KEYWORDS_TOYOTA_COROLLA" newpages.txt >> ToyotaCorolla.txt
egrep -i "$KEYWORDS_TOYOTA_PRIUS" newpages.txt >> ToyotaPrius.txt

#Search SUVs

#Search vans

#Stat cars

CAMRY=`stat --print=%s ToyotaCamry.txt`
COROLLA=`stat --print=%s ToyotaCorolla.txt`
PRIUS=`stat --print=%s ToyotaPrius.txt`

#Stat SUVs

#Stat vans

#Categorize cars

if [ $CAMRY -ne 0 ];
then
  export CATFILE="ToyotaCamry.txt"
  export CATNAME="Toyota Camry"
  $CATEGORIZE
fi

if [ $COROLLA -ne 0 ];
then
  export CATFILE="ToyotaCorolla.txt"
  export CATNAME="Toyota Corolla"
  $CATEGORIZE
fi
{'fraction_non_alphanumeric': 0.12900302114803625, 'fraction_numerical': 0.027794561933534745, 'mean_word_length': 7.446428571428571, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '980886', 'n_tokens_mistral': 1676, 'n_tokens_neox': 1478, 'n_words': 259}