<commit_before>#!/bin/bash -xe

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

PROJECT=$1
BRANCH=$2
JOBNAME=$3

# Replace /'s in the branch name with -'s because Zanata does not
# allow /'s in version names.
ZANATA_VERSION=${BRANCH//\//-}

SCRIPTSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source $SCRIPTSDIR/common_translation_update.sh

init_branch $BRANCH

function cleanup_module {
    local modulename=$1

    # Remove obsolete files.
    cleanup_po_files "$modulename"
    cleanup_pot_files "$modulename"

    # Compress downloaded po files, this needs to be done after
    # cleanup_po_files since that function needs to have information the
    # number of untranslated strings.
    compress_po_files "$modulename"
}

# Add all po files to the git repo in a target directory
function git_add_po_files {
    local target_dir=$1

    local po_file_count=`find $1 -name *.po | wc -l`

    if [ $po_file_count -ne 0 ]; then
        git add $target_dir/*/*
    fi
}

# Add all JSON files to the git repo (used in javascript translations)
function git_add_json_files {
    local target_dir=$1

    local json_file_count=`find $1 -name '*.json' | wc -l`

    if [ $json_file_count -ne 0 ]; then
        git add $target_dir/*
    fi
}

# Propose updates for manuals
function propose_manuals {

    # Pull updated translations from Zanata.
    pull_from_zanata "$PROJECT"

    # Compress downloaded po files
    case "$PROJECT" in
        openstack-manuals)
            # Cleanup po and pot files
            cleanup_module "doc"
            ;;
        api-site)
            # Cleanup po and pot files
            cleanup_module "api-quick-start"
            cleanup_module "firstapp"
            ;;
        security-doc)
            cleanup_module "security-guide"
            ;;
    esac

    # Add imported upstream translations to git
    for FILE in ${DocFolder}/*; do
        DOCNAME=${FILE#${DocFolder}/}
        if [ -d ${DocFolder}/${DOCNAME}/locale ] ; then
            git_add_po_files ${DocFolder}/${DOCNAME}/locale
        fi
        if [ -d ${DocFolder}/${DOCNAME}/source/locale ] ; then
            git_add_po_files ${DocFolder}/${DOCNAME}/source/locale
        fi
    done
}

# Propose updates for training-guides
function propose_training_guides {

    # Pull updated translations from Zanata.
    pull_from_zanata "$PROJECT"

    # Cleanup po and pot files
    cleanup_module "doc/upstream-training"

    # Add all changed files to git
    git_add_po_files doc/upstream-training/source/locale
}

# Propose updates for i18n
function propose_i18n {

    # Pull updated translations from Zanata.
    pull_from_zanata "$PROJECT"

    # Cleanup po and pot files
    cleanup_module "doc"

    # Add all changed files to git
    git_add_po_files doc/source/locale
}


# Propose updates for python and django projects
function propose_python_django {
    local modulename=$1
    local version=$2

    # Check for empty directory and exit early
    local content=$(ls -A $modulename/locale/)

    if [[ "$content" == "" ]] ; then
        return
    fi

    # Now add all changed files to git.
    # Note we add them here to not have to differentiate in the functions
    # between new files and files already under git control.
    git_add_po_files $modulename/locale

    # Cleanup po and pot files
    cleanup_module "$modulename"
    if [ "$version" == "master" ] ; then
        # Remove not anymore translated log files on master, but not
        # on released stable branches.
        cleanup_log_files "$modulename"
    fi

    # Check first whether directory exists, it might be missing if
    # there are no translations.
    if [[ -d "$modulename/locale/" ]] ; then

        # Some files were changed, add changed files again to git, so
        # that we can run git diff properly.
        git_add_po_files $modulename/locale
    fi
}


# Handle either python or django proposals
function handle_python_django {
    local project=$1
    # kind can be "python" or "django"
    local kind=$2
    local module_names

    module_names=$(get_modulename $project $kind)
    if [ -n "$module_names" ]; then
        setup_project "$project" "$ZANATA_VERSION" $module_names
        if [[ "$kind" == "django" ]] ; then
            install_horizon
        fi
        # Pull updated translations from Zanata
        pull_from_zanata "$project"
        propose_releasenotes "$ZANATA_VERSION"
        for modulename in $module_names; do<commit_msg>Prevent duplicate pull_from_zanata on the same repo<commit_after>#!/bin/bash -xe

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

PROJECT=$1
BRANCH=$2
JOBNAME=$3

# Replace /'s in the branch name with -'s because Zanata does not
# allow /'s in version names.
ZANATA_VERSION=${BRANCH//\//-}

SCRIPTSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source $SCRIPTSDIR/common_translation_update.sh

init_branch $BRANCH

function cleanup_module {
    local modulename=$1

    # Remove obsolete files.
    cleanup_po_files "$modulename"
    cleanup_pot_files "$modulename"

    # Compress downloaded po files, this needs to be done after
    # cleanup_po_files since that function needs to have information the
    # number of untranslated strings.
    compress_po_files "$modulename"
}

# Add all po files to the git repo in a target directory
function git_add_po_files {
    local target_dir=$1

    local po_file_count=`find $1 -name *.po | wc -l`

    if [ $po_file_count -ne 0 ]; then
        git add $target_dir/*/*
    fi
}

# Add all JSON files to the git repo (used in javascript translations)
function git_add_json_files {
    local target_dir=$1

    local json_file_count=`find $1 -name '*.json' | wc -l`

    if [ $json_file_count -ne 0 ]; then
        git add $target_dir/*
    fi
}

# Propose updates for manuals
function propose_manuals {

    # Pull updated translations from Zanata.
    pull_from_zanata "$PROJECT"

    # Compress downloaded po files
    case "$PROJECT" in
        openstack-manuals)
            # Cleanup po and pot files
            cleanup_module "doc"
            ;;
        api-site)
            # Cleanup po and pot files
            cleanup_module "api-quick-start"
            cleanup_module "firstapp"
            ;;
        security-doc)
            cleanup_module "security-guide"
            ;;
    esac

    # Add imported upstream translations to git
    for FILE in ${DocFolder}/*; do
        DOCNAME=${FILE#${DocFolder}/}
        if [ -d ${DocFolder}/${DOCNAME}/locale ] ; then
            git_add_po_files ${DocFolder}/${DOCNAME}/locale
        fi
        if [ -d ${DocFolder}/${DOCNAME}/source/locale ] ; then
            git_add_po_files ${DocFolder}/${DOCNAME}/source/locale
        fi
    done
}

# Propose updates for training-guides
function propose_training_guides {

    # Pull updated translations from Zanata.
    pull_from_zanata "$PROJECT"

    # Cleanup po and pot files
    cleanup_module "doc/upstream-training"

    # Add all changed files to git
    git_add_po_files doc/upstream-training/source/locale
}

# Propose updates for i18n
function propose_i18n {

    # Pull updated translations from Zanata.
    pull_from_zanata "$PROJECT"

    # Cleanup po and pot files
    cleanup_module "doc"

    # Add all changed files to git
    git_add_po_files doc/source/locale
}


# Propose updates for python and django projects
function propose_python_django {
    local modulename=$1
    local version=$2

    # Check for empty directory and exit early
    local content=$(ls -A $modulename/locale/)

    if [[ "$content" == "" ]] ; then
        return
    fi

    # Now add all changed files to git.
    # Note we add them here to not have to differentiate in the functions
    # between new files and files already under git control.
    git_add_po_files $modulename/locale

    # Cleanup po and pot files
    cleanup_module "$modulename"
    if [ "$version" == "master" ] ; then
        # Remove not anymore translated log files on master, but not
        # on released stable branches.
        cleanup_log_files "$modulename"
    fi

    # Check first whether directory exists, it might be missing if
    # there are no translations.
    if [[ -d "$modulename/locale/" ]] ; then

        # Some files were changed, add changed files again to git, so
        # that we can run git diff properly.
        git_add_po_files $modulename/locale
    fi
}


# Handle either python or django proposals
function handle_python_django {
    local project=$1
    # kind can be "python" or "django"
    local kind=$2
    local module_names

    module_names=$(get_modulename $project $kind)
    if [ -n "$module_names" ]; then
        setup_project "$project" "$ZANATA_VERSION" $module_names
        if [[ "$kind" == "django" ]] ; then
            install_horizon
        fi
        # Pull updated translations from Zanata only the first time
        if [ ! -f .zanata-pull ] ; then
            touch .zanata-pull
            pull_from_zanata "$project"
        fi
        propose_releasenotes "$ZANATA_VERSION"
        for modulename in $module_names; do
{'fraction_non_alphanumeric': 0.0843098311817279, 'fraction_numerical': 0.004568023833167826, 'mean_word_length': 2.74247491638796, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '968709', 'n_tokens_mistral': 3046, 'n_tokens_neox': 2900, 'n_words': 1042}