<commit_before>#!/bin/bash
#
# Script to fetch generated API references docs from the Android build server and stage them.

source gbash.sh || exit

readonly defaultDb=""
DEFINE_string buildId --required "" "The build ID from the Android build server"
DEFINE_string dateStr "<insert date here>" "Date string used for CL message. Enclose date in double quotes (ex: \"April 29, 2021\")"
DEFINE_string db "$defaultDb" "The database used for staging. Omitting this value will stage changes to the staging DB."
DEFINE_bool useToT false "Stage docs from tip-of-tree docs build rather than public docs build"

gbash::init_google "$@"

# Allowlist for including specific directories being generated by Dackka.
#
# There are separate lists for Java and Kotlin refdocs as some libraries (such
# as Compose) only publish refdocs for a single language.
#
# Each directory's spelling must match the library's directory in
# frameworks/support.
#
# This list should match, or be a subset of, the list of libraries defined in
# https://cs.android.com/androidx/platform/frameworks/support/+/androidx-main:buildSrc/src/main/kotlin/androidx/build/docs/AndroidXDocsPlugin.kt;l=568
readonly javaLibraryDirs=(
#  "benchmark"
#  "collection"
  "navigation"
  "paging"
  "wear"
  "window"
)
readonly kotlinLibraryDirs=(
#  "benchmark"
  "compose"
#  "collection"
  "navigation"
  "paging"
  "wear"
  "window"
)


# Change directory to this script's location and store the directory
cd "$(dirname $0)"
scriptDirectory=$(pwd)

# Working directories for the refdocs
outDir="$scriptDirectory/out"
readonly newDir="reference-docs"
readonly dackkaNewDir="reference-docs-dackka"

# Remove and recreate the existing out directory to avoid conflicts from previous runs
rm -rf $outDir
mkdir -p $outDir/$newDir
mkdir -p $outDir/$dackkaNewDir
cd $outDir

printf "=================================================================== \n"
printf "== Download the doc zip files from the build server \n"
printf "=================================================================== \n"

if (( FLAGS_useToT )); then
  printf "Downloading docs-tip-of-tree zip files \n"
  androidxJavaDocsZip="doclava-tip-of-tree-docs-${FLAGS_buildId}.zip"
  androidxKotlinDocsZip="dokka-tip-of-tree-docs-${FLAGS_buildId}.zip"
  androidxDackkaDocsZip="dackka-tip-of-tree-docs-${FLAGS_buildId}.zip"
else
  printf "Downloading docs-public zip files \n"
  androidxJavaDocsZip="doclava-public-docs-${FLAGS_buildId}.zip"
  androidxKotlinDocsZip="dokka-public-docs-${FLAGS_buildId}.zip"
  androidxDackkaDocsZip="dackka-public-docs-${FLAGS_buildId}.zip"
fi

/google/data/ro/projects/android/fetch_artifact --bid $FLAGS_buildId --target androidx $androidxJavaDocsZip
/google/data/ro/projects/android/fetch_artifact --bid $FLAGS_buildId --target androidx $androidxKotlinDocsZip
/google/data/ro/projects/android/fetch_artifact --bid $FLAGS_buildId --target androidx $androidxDackkaDocsZip

printf "\n"
printf "=================================================================== \n"
printf "== Unzip the doc zip files \n"
printf "=================================================================== \n"

unzip $androidxJavaDocsZip -d $newDir
unzip $androidxKotlinDocsZip -d $newDir
unzip $androidxDackkaDocsZip -d $dackkaNewDir

printf "\n"
printf "=================================================================== \n"
printf "== Format the doc zip files \n"
printf "=================================================================== \n"

cd $newDir<commit_msg>Allow staging refdocs from local directory<commit_after>#!/bin/bash
#
# Script to fetch generated API references docs from the Android build server and stage them.

source gbash.sh || exit

readonly defaultDb=""
DEFINE_string buildId --required "" "The build ID from the Android build server. This is ignored when specifying the 'sourceDir' flag."
DEFINE_string dateStr "<insert date here>" "Date string used for CL message. Enclose date in double quotes (ex: \"April 29, 2021\")"
DEFINE_string db "$defaultDb" "The database used for staging. Omitting this value will stage changes to the staging DB."
DEFINE_string sourceDir "" "Local directory to fetch doc artifacts from"
DEFINE_bool useToT false "Stage docs from tip-of-tree docs build rather than public docs build"

gbash::init_google "$@"

# Allowlist for including specific directories being generated by Dackka.
#
# There are separate lists for Java and Kotlin refdocs as some libraries (such
# as Compose) only publish refdocs for a single language.
#
# Each directory's spelling must match the library's directory in
# frameworks/support.
#
# This list should match, or be a subset of, the list of libraries defined in
# https://cs.android.com/androidx/platform/frameworks/support/+/androidx-main:buildSrc/src/main/kotlin/androidx/build/docs/AndroidXDocsPlugin.kt;l=568
readonly javaLibraryDirs=(
#  "benchmark"
#  "collection"
  "navigation"
  "paging"
  "wear"
  "window"
)
readonly kotlinLibraryDirs=(
#  "benchmark"
  "compose"
#  "collection"
  "navigation"
  "paging"
  "wear"
  "window"
)


# Change directory to this script's location and store the directory
cd "$(dirname $0)"
scriptDirectory=$(pwd)

# Working directories for the refdocs
outDir="$scriptDirectory/out"
readonly newDir="reference-docs"
readonly dackkaNewDir="reference-docs-dackka"

# Remove and recreate the existing out directory to avoid conflicts from previous runs
rm -rf $outDir
mkdir -p $outDir/$newDir
mkdir -p $outDir/$dackkaNewDir
cd $outDir

if [ "$FLAGS_sourceDir" == "" ]; then
  printf "=================================================================== \n"
  printf "== Download the doc zip files from the build server \n"
  printf "=================================================================== \n"

  if (( FLAGS_useToT )); then
    printf "Downloading docs-tip-of-tree zip files \n"
    androidxJavaDocsZip="doclava-tip-of-tree-docs-${FLAGS_buildId}.zip"
    androidxKotlinDocsZip="dokka-tip-of-tree-docs-${FLAGS_buildId}.zip"
    androidxDackkaDocsZip="dackka-tip-of-tree-docs-${FLAGS_buildId}.zip"
  else
    printf "Downloading docs-public zip files \n"
    androidxJavaDocsZip="doclava-public-docs-${FLAGS_buildId}.zip"
    androidxKotlinDocsZip="dokka-public-docs-${FLAGS_buildId}.zip"
    androidxDackkaDocsZip="dackka-public-docs-${FLAGS_buildId}.zip"
  fi

  /google/data/ro/projects/android/fetch_artifact --bid $FLAGS_buildId --target androidx $androidxJavaDocsZip
  /google/data/ro/projects/android/fetch_artifact --bid $FLAGS_buildId --target androidx $androidxKotlinDocsZip
  /google/data/ro/projects/android/fetch_artifact --bid $FLAGS_buildId --target androidx $androidxDackkaDocsZip

  printf "\n"
  printf "=================================================================== \n"
  printf "== Unzip the doc zip files \n"
  printf "=================================================================== \n"

  unzip $androidxJavaDocsZip -d $newDir
  unzip $androidxKotlinDocsZip -d $newDir
  unzip $androidxDackkaDocsZip -d $dackkaNewDir
else
  printf "=================================================================== \n"
  printf "== Copying doc sources from local directory $FLAGS_sourceDir \n"
  printf "=================================================================== \n"

  cp -r "$FLAGS_sourceDir/javadoc/." $newDir
  mkdir -p $newDir/reference/kotlin
  cp -r "$FLAGS_sourceDir/dokkaKotlinDocs/." $newDir/reference/kotlin
  cp -r "$FLAGS_sourceDir/dackkaDocs/." $dackkaNewDir
fi

printf "\n"
printf "=================================================================== \n"
printf "== Format the doc files \n"
printf "=================================================================== \n"

cd $newDir
{'fraction_non_alphanumeric': 0.218399791503779, 'fraction_numerical': 0.0026062027625749283, 'mean_word_length': 5.51528013582343, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5214245', 'n_tokens_mistral': 2378, 'n_tokens_neox': 2202, 'n_words': 659}