<commit_before>#!/bin/bash

# This validates that prep tables, cohort builder menu, data dictionary, cope survey versions and whole genome variant tables exist.
set -e

export BQ_PROJECT=$1        # CDR project
export BQ_DATASET=$2        # CDR dataset
export CDR_VERSION=$3       # CDR version
export BUCKET=$4            # Bucket

TEMP_FILE_DIR="csv"
CSV_HOME_DIR="cdr_csv_files"
CRITERIA_MENU="cb_criteria_menu.csv"
PREP_CRITERIA="prep_criteria.csv"
PREP_CRITERIA_ANCESTOR="prep_criteria_ancestor.csv"
PREP_CLINICAL_TERMS="prep_clinical_terms_nc.csv"
All_FILES=($CRITERIA_MENU $PREP_CRITERIA $PREP_CRITERIA_ANCESTOR $PREP_CLINICAL_TERMS)
INCOMPATIBLE_DATASETS=("R2019Q4R3" "R2019Q4R4")

if [[ ${INCOMPATIBLE_DATASETS[@]} =~ $BQ_DATASET ]];
  then
  echo "Can't run CDR build indices against "$BQ_DATASET"!"
  exit 1
fi

# Purge all backup csv files except for the last 10 versions
fileCount=$(gsutil ls gs://all-of-us-workbench-private-cloudsql/DummySR/cdr_csv_files/backup | wc -l)
allFilesCount=${#All_FILES[@]}
numberToDelete=$((fileCount - allFilesCount * 10))
if [[ $numberToDelete > 0 ]];
then
  echo "Purging backup files"
  while IFS= read -r line; do
    ts=$(echo $line | cut -d_ -f1)
    if [[ ${#ts} > 0 ]];
    then
      echo "Removing $line"
      gsutil rm gs://$BUCKET/$BQ_DATASET/$CSV_HOME_DIR/backup/$line
    fi
  # This lists all the files in the backup bucket sorted by timestamp and gets only the number to delete
  done < <(gsutil ls gs://$BUCKET/$BQ_DATASET/$CSV_HOME_DIR/backup | rev | cut -d/ -f1 | rev | sort | head -$numberToDelete)
fi

rm -rf $TEMP_FILE_DIR
mkdir $TEMP_FILE_DIR

if gsutil -m cp gs://$BUCKET/$BQ_DATASET/$CSV_HOME_DIR/*.csv $TEMP_FILE_DIR
 then
  timestamp=$(date +%s)
  for file in ${All_FILES[@]}; do
    read -r header < $TEMP_FILE_DIR/$file
    IFS=',' read -r -a columns <<< $header
    case $file in
      $CRITERIA_MENU)
        echo "Processing $file"
        if [[ $columns =~ id ]];
        then
          echo "Removing $file header"
          # Remove the first line of file
          sed 1d $TEMP_FILE_DIR/$file > $TEMP_FILE_DIR/temp_$file
          # Zip the file
          gzip -cvf $TEMP_FILE_DIR/temp_$file > $TEMP_FILE_DIR/$file.gz
        else
          # Zip the file
          gzip $TEMP_FILE_DIR/$file
        fi
        # Copy it back to bucket
        gsutil cp $TEMP_FILE_DIR/$file.gz gs://$BUCKET/$BQ_DATASET/$CDR_VERSION/
        # Backup the csv file
        echo "Backing up $file"
        gsutil cp $TEMP_FILE_DIR/$file.gz gs://$BUCKET/$BQ_DATASET/$CSV_HOME_DIR/backup/"$timestamp"_"$file".gz
      ;;
    $PREP_CRITERIA|$PREP_CRITERIA_ANCESTOR|$PREP_CLINICAL_TERMS)
      tableName=${file%.*}
      if [[ $columns =~ id || $columns =~ ancestor_id || $columns =~ parent ]];
      then
        echo "Removing $file header"
        # Remove the first line of file
        sed 1d $TEMP_FILE_DIR/$file > $TEMP_FILE_DIR/temp_$file<commit_msg>[noTicket][risk=no]Validate ds_data_dictionary file (#4753)<commit_after>#!/bin/bash

# This validates that prep tables, cohort builder menu, data dictionary, cope survey versions and whole genome variant tables exist.
set -e

export BQ_PROJECT=$1        # CDR project
export BQ_DATASET=$2        # CDR dataset
export CDR_VERSION=$3       # CDR version
export BUCKET=$4            # Bucket

TEMP_FILE_DIR="csv"
CSV_HOME_DIR="cdr_csv_files"
CRITERIA_MENU="cb_criteria_menu.csv"
PREP_CRITERIA="prep_criteria.csv"
PREP_CRITERIA_ANCESTOR="prep_criteria_ancestor.csv"
PREP_CLINICAL_TERMS="prep_clinical_terms_nc.csv"
DS_DATA_DICTIONARY="ds_data_dictionary.csv"
All_FILES=($CRITERIA_MENU $PREP_CRITERIA $PREP_CRITERIA_ANCESTOR $PREP_CLINICAL_TERMS $DS_DATA_DICTIONARY)
INCOMPATIBLE_DATASETS=("R2019Q4R3" "R2019Q4R4")

if [[ ${INCOMPATIBLE_DATASETS[@]} =~ $BQ_DATASET ]];
  then
  echo "Can't run CDR build indices against "$BQ_DATASET"!"
  exit 1
fi

# Purge all backup csv files except for the last 10 versions
fileCount=$(gsutil ls gs://$BUCKET/$BQ_DATASET/$CSV_HOME_DIR/backup | wc -l)
allFilesCount=${#All_FILES[@]}
numberToDelete=$((fileCount - allFilesCount * 10))
if [[ $numberToDelete > 0 ]];
then
  echo "Purging backup files"
  while IFS= read -r line; do
    ts=$(echo $line | cut -d_ -f1)
    if [[ ${#ts} > 0 ]];
    then
      echo "Removing $line"
      gsutil rm gs://$BUCKET/$BQ_DATASET/$CSV_HOME_DIR/backup/$line
    fi
  # This lists all the files in the backup bucket sorted by timestamp and gets only the number to delete
  done < <(gsutil ls gs://$BUCKET/$BQ_DATASET/$CSV_HOME_DIR/backup | rev | cut -d/ -f1 | rev | sort | head -$numberToDelete)
fi

rm -rf $TEMP_FILE_DIR
mkdir $TEMP_FILE_DIR

if gsutil -m cp gs://$BUCKET/$BQ_DATASET/$CSV_HOME_DIR/*.csv $TEMP_FILE_DIR
 then
  timestamp=$(date +%s)
  for file in ${All_FILES[@]}; do
    read -r header < $TEMP_FILE_DIR/$file
    IFS=',' read -r -a columns <<< $header
    case $file in
      $CRITERIA_MENU|$DS_DATA_DICTIONARY)
        echo "Processing $file"
        if [[ $columns =~ id ]];
        then
          echo "Removing $file header"
          # Remove the first line of file
          sed 1d $TEMP_FILE_DIR/$file > $TEMP_FILE_DIR/temp_$file
          # Zip the file
          gzip -cvf $TEMP_FILE_DIR/temp_$file > $TEMP_FILE_DIR/$file.gz
        else
          # Zip the file
          gzip $TEMP_FILE_DIR/$file
        fi
        # Copy it back to bucket
        gsutil cp $TEMP_FILE_DIR/$file.gz gs://$BUCKET/$BQ_DATASET/$CDR_VERSION/
        # Backup the csv file
        echo "Backing up $file"
        gsutil cp $TEMP_FILE_DIR/$file.gz gs://$BUCKET/$BQ_DATASET/$CSV_HOME_DIR/backup/"$timestamp"_"$file".gz
      ;;
    $PREP_CRITERIA|$PREP_CRITERIA_ANCESTOR|$PREP_CLINICAL_TERMS)
      tableName=${file%.*}
      if [[ $columns =~ id || $columns =~ ancestor_id || $columns =~ parent ]];
      then
        echo "Removing $file header"
        # Remove the first line of file
        sed 1d $TEMP_FILE_DIR/$file > $TEMP_FILE_DIR/temp_$file
{'fraction_non_alphanumeric': 0.12005388112476847, 'fraction_numerical': 0.009765953864286917, 'mean_word_length': 3.316860465116279, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2271512', 'n_tokens_mistral': 2260, 'n_tokens_neox': 2208, 'n_words': 522}