<commit_before>/**
 * Axelor Business Solutions
 *
 * Copyright (C) 2012-2014 Axelor (<http://axelor.com>).
 *
 * This program is free software: you can redistribute it and/or  modify
 * it under the terms of the GNU Affero General Public License, version 3,
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package com.axelor.meta.loader;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URL;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.inject.Singleton;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import au.com.bytecode.opencsv.CSVReader;

import com.axelor.common.FileUtils;
import com.axelor.common.StringUtils;
import com.axelor.meta.MetaScanner;
import com.axelor.meta.db.MetaTranslation;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.inject.persist.Transactional;

@Singleton
public class I18nLoader extends AbstractLoader {

	private Logger log = LoggerFactory.getLogger(I18nLoader.class);

	@Override
	protected void doLoad(Module module, boolean update) {

		final List<URL> files = MetaScanner.findAll(module.getName(), "i18n", "(.*?)\\.csv");
		final List<URL> sorted = Lists.newArrayList(files);
		
		for(final URL resource : sorted) {
			try(InputStream is = resource.openStream()) {
				log.debug("Load translation: {}", resource);
				process(is, resource.getFile(), module.getName());
			} catch (IOException e) {
				log.error("Unable to import file: {}", resource.getFile());
			}
		}
	}

	@Transactional
	public void load(String importPath) {
		//Import by module resolver order
		for(final Module module : ModuleManager.getAll()) {
			if(Strings.isNullOrEmpty(importPath)) {
				this.doLoad(module, false);
			} else {
				this.loadModule(module, importPath);
			}
		}
	}

	private void loadModule(Module module, String importPath) {

		File moduleDir = FileUtils.getFile(importPath, module.getName());
		if(!moduleDir.exists() || !moduleDir.isDirectory() || moduleDir.listFiles() == null) {
			return;
		}
		
		log.debug("Load {} translations", module.getName());
		for(File file : moduleDir.listFiles()) {
			try {
				log.debug("Load {} translations", file.getPath());
				process(new FileInputStream(file), file.getPath(), module.getName());
			} catch (IOException e) {
				log.error("Unable to import file: {}", file.getName());
			}
		}
	}

	private void process(final InputStream stream, String fileName, String moduleName) throws IOException {
		
		// Get language name from the file name
		String language = "";
		Pattern pattern = Pattern.compile(".*(?:/|\\\\)(.+)\\.(\\w+)$");
		Matcher matcher = pattern.matcher(fileName);
		if (matcher.matches()) {
			language = matcher.group(1);
		}

		Reader reader = new InputStreamReader(stream);
		CSVReader csvReader = new CSVReader(reader);

		try {
			String[] fields = csvReader.readNext();
			String[] values = null;
			
			if (fields.length > 2) {
				throw new IOException("Old translation file format.");
			}<commit_msg>Fixed i18n loader (import launguage files only).<commit_after>/**
 * Axelor Business Solutions
 *
 * Copyright (C) 2012-2014 Axelor (<http://axelor.com>).
 *
 * This program is free software: you can redistribute it and/or  modify
 * it under the terms of the GNU Affero General Public License, version 3,
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package com.axelor.meta.loader;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.net.URL;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.inject.Singleton;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import au.com.bytecode.opencsv.CSVReader;

import com.axelor.common.FileUtils;
import com.axelor.common.StringUtils;
import com.axelor.meta.MetaScanner;
import com.axelor.meta.db.MetaTranslation;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.inject.persist.Transactional;

@Singleton
public class I18nLoader extends AbstractLoader {

	private Logger log = LoggerFactory.getLogger(I18nLoader.class);

	@Override
	protected void doLoad(Module module, boolean update) {

		final List<URL> files = MetaScanner.findAll(module.getName(), "i18n", "(.*?)\\.csv");
		final List<URL> sorted = Lists.newArrayList(files);
		
		for(final URL resource : sorted) {
			try(InputStream is = resource.openStream()) {
				log.debug("Load translation: {}", resource);
				process(is, resource.getFile(), module.getName());
			} catch (IOException e) {
				log.error("Unable to import file: {}", resource.getFile());
			}
		}
	}

	@Transactional
	public void load(String importPath) {
		//Import by module resolver order
		for(final Module module : ModuleManager.getAll()) {
			if(Strings.isNullOrEmpty(importPath)) {
				this.doLoad(module, false);
			} else {
				this.loadModule(module, importPath);
			}
		}
	}

	private void loadModule(Module module, String importPath) {

		File moduleDir = FileUtils.getFile(importPath, module.getName());
		if(!moduleDir.exists() || !moduleDir.isDirectory() || moduleDir.listFiles() == null) {
			return;
		}
		
		log.debug("Load {} translations", module.getName());
		for(File file : moduleDir.listFiles()) {
			try {
				log.debug("Load {} translations", file.getPath());
				process(new FileInputStream(file), file.getPath(), module.getName());
			} catch (IOException e) {
				log.error("Unable to import file: {}", file.getName());
			}
		}
	}

	private void process(final InputStream stream, String fileName, String moduleName) throws IOException {
		
		// Get language name from the file name
		String language = "";
		Pattern pattern = Pattern.compile(".*(?:messages_)([a-zA-Z_]+)\\.csv$");
		Matcher matcher = pattern.matcher(fileName);
		if (!matcher.matches()) return;
		
		language = matcher.group(1);

		Reader reader = new InputStreamReader(stream);
		CSVReader csvReader = new CSVReader(reader);

		try {
			String[] fields = csvReader.readNext();
			String[] values = null;
{'fraction_non_alphanumeric': 0.1272853185595568, 'fraction_numerical': 0.005401662049861496, 'mean_word_length': 4.221258134490238, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3108670', 'n_tokens_mistral': 2371, 'n_tokens_neox': 2133, 'n_words': 612}