<commit_before>#!/usr/bin/perl -w

#**
# @file parse.pl
# @brief Scan all specified Commotion git repositories and generate updated
# PO files for upload to the Transifex translation service.
#
# @author <NAME>, <EMAIL>
#
# @internal
# Created November 10, 2013
# Company The Open Technology Institute
# Copyright (c) 2014, <NAME>
#
# This file is part of Commotion, Copyright (c) 2014, <NAME>
#
# Commotion is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Commotion is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Commotion. If not, see <http://www.gnu.org/licenses/>.
#
# =====================================================================================
#*

use strict;
use File::Path qw(make_path remove_tree);
use Git::Repository;
use File::Copy;
use File::Next;
use Text::Balanced qw(extract_bracketed extract_delimited extract_tagged);
use DateTime;

#**
# @var verbosity Print debug messages
# @param 0|1 0 == off, 1 == on
#*
my $verbosity = 1;
my @todo = ("\n\nTo Do:");
push(@todo, "Fix use vs. require");

## TODO: for next major revision:
## Look for ways to minimize string overlap
## http://www.perlmonks.org/?node_id=816086
push(@todo, "minimize duplicate strings in po files - http://www.perlmonks.org/?node_id=816086");

#**
# @var repos Repos to be scanned for translatable strings
# @param name Repo common name. Used as primary key.
# @param source Named k:v pair specifying location of remote git repo. Value of name parameter.
# @param branch Named k:v pair listing appropriate repo branch as defined in commotion-feed. Value of name parameter.
#*
my %repos = (
	'luci-commotion-apps' => {
		'source' => 'https://github.com/opentechinstitute/luci-commotion-apps.git',
		'branch' => 'master',
	},
	
	'luci-theme-commotion' => {
		'source' => 'https://github.com/opentechinstitute/luci-theme-commotion.git',
		'branch' => 'master',
	},
	'luci-commotion' => {
		'source' => 'https://github.com/opentechinstitute/luci-commotion.git',
		'branch' => 'master',
	},
	'commotion-dashboard-helper' => {
		'source' => 'https://github.com/opentechinstitute/commotion-dashboard-helper.git',
		'branch' => 'master',
	}, 
	'commotiond commotion-service-manager' => {
		'source' => 'https://github.com/opentechinstitute/commotion-service-manager.git',
		'branch' => 'master',
	},
	'commotion-router' => {
		'source' => 'https://github.com/opentechinstitute/commotion-router.git',
		'branch' => 'master',
	},
	'commotion-feed' => {
		'source' => 'https://github.com/opentechinstitute/commotion-feed.git',
		'branch' => 'master',
	}, 
	'commotion-lua-helpers' => {
		'source' => 'https://github.com/opentechinstitute/commotion-lua-helpers.git',
		'branch' => 'master',
	},
	'serval-dna' => {
		'source' => 'https://github.com/opentechinstitute/commotion-dashboard-helper.git',
		'branch' => 'commotion-wireless',
	},
	'luci-commotion-splash' => {
		'source' => 'https://github.com/opentechinstitute/luci-commotion-splash.git',
		'branch' => 'master',
	},
	'commotion-debug-helper' => {
		'source' => 'https://github.com/opentechinstitute/commotion-debug-helper.git',
		'branch' => 'master',
	},
	'luci-i18n-commotion' => {
		'source' => 'https://github.com/opentechinstitute/luci-i18n-commotion.git',
		'branch' => 'master',
	}, 
	'olsrd' => {
		'source' => 'https://github.com/opentechinstitute/olsrd.git',
		'branch' => 'master',
	},
);

#**
# @section Directories 
# @brief Define internal directory structure, including translation source and working area.
# Include trailing slashes!
#*
my $working_dir = 'working/';
my $working_source_dir = $working_dir . 'source/';
my $working_translations_dir = $working_dir . 'translations/';
my $stable_translations_dir = $working_source_dir . 'luci-i18n-commotion/translations/';

#**
# @brief Prepare working directories
#*
if (not -e $working_dir) {
	print("Creating working directories\n");
	make_path($working_dir, $working_source_dir, $working_translations_dir, {verbose=>1})
		|| die "ERROR: Couldn't create " . $working_dir . "\n";	
}

#**
# @brief Iterate over commotion-router packages defined in %repos.
# If working copy does not exist, clone it and check out proper branch.
# If working copy does exist, check out proper branch and pull updates.
#*<commit_msg>Updated sources list to reflect repo merge<commit_after>#!/usr/bin/perl -w

#**
# @file parse.pl
# @brief Scan all specified Commotion git repositories and generate updated
# PO files for upload to the Transifex translation service.
#
# @author <NAME>, <EMAIL>
#
# @internal
# Created November 10, 2013
# Company The Open Technology Institute
# Copyright (c) 2014, <NAME>
#
# This file is part of Commotion, Copyright (c) 2014, <NAME>
#
# Commotion is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Commotion is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Commotion. If not, see <http://www.gnu.org/licenses/>.
#
# =====================================================================================
#*

use strict;
use File::Path qw(make_path remove_tree);
use Git::Repository;
use File::Copy;
use File::Next;
use Text::Balanced qw(extract_bracketed extract_delimited extract_tagged);
use DateTime;

#**
# @var verbosity Print debug messages
# @param 0|1 0 == off, 1 == on
#*
my $verbosity = 1;
my @todo = ("\n\nTo Do:");
push(@todo, "Fix use vs. require");

## TODO: for next major revision:
## Look for ways to minimize string overlap
## http://www.perlmonks.org/?node_id=816086
push(@todo, "minimize duplicate strings in po files - http://www.perlmonks.org/?node_id=816086");

#**
# @var repos Repos to be scanned for translatable strings
# @param name Repo common name. Used as primary key.
# @param source Named k:v pair specifying location of remote git repo. Value of name parameter.
# @param branch Named k:v pair listing appropriate repo branch as defined in commotion-feed. Value of name parameter.
#*
my %repos = (
        'commotion-router' => {
                'source' => 'https://github.com/opentechinstitute/commotion-router.git',
                'branch' => 'master',
        },
        'commotion-feed' => {
                'source' => 'https://github.com/opentechinstitute/commotion-feed.git',
                'branch' => 'master',
        }, 
        'commotiond' => {
            'source' => 'https://github.com/opentechinstitute/commotiond.git',
            'branch' => 'master',
        },
        'commotion-service-manager' => {
            'source' => 'https://github.com/opentechinstitute/commotion-service-manager.git',
            'branch' => 'master',
        },
        'luci-commotion' => {
            'source' => 'https://github.com/opentechinstitute/luci-commotion.git',
            'branch' => 'master',
        },
        'luci-i18n-commotion' => {
            'source' => 'https://github.com/opentechinstitute/luci-i18n-commotion.git',
            'branch' => 'master',
        }, 
        'olsrd' => {
            'source' => 'https://github.com/opentechinstitute/olsrd.git',
            'branch' => 'master',
        },
        'serval-dna' => {
            'source' => 'https://github.com/opentechinstitute/serval-dna.git',
            'branch' => 'commotion-wireless',
        },
);

#**
# @section Directories 
# @brief Define internal directory structure, including translation source and working area.
# Include trailing slashes!
#*
my $working_dir = 'working/';
my $working_source_dir = $working_dir . 'source/';
my $working_translations_dir = $working_dir . 'translations/';
my $stable_translations_dir = $working_source_dir . 'luci-i18n-commotion/translations/';

#**
# @brief Prepare working directories
#*
if (not -e $working_dir) {
	print("Creating working directories\n");
	make_path($working_dir, $working_source_dir, $working_translations_dir, {verbose=>1})
		|| die "ERROR: Couldn't create " . $working_dir . "\n";	
}

#**
# @brief Iterate over commotion-router packages defined in %repos.
# If working copy does not exist, clone it and check out proper branch.
# If working copy does exist, check out proper branch and pull updates.
#*
{'fraction_non_alphanumeric': 0.16020989170481187, 'fraction_numerical': 0.008708272859216255, 'mean_word_length': 3.921978021978022, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 78, 'https://': 21, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3464800', 'n_tokens_mistral': 2930, 'n_tokens_neox': 2781, 'n_words': 810}