<commit_before>#include "PackLanguageTable.h"
#include "check_params.h"

#include <easysprpacker.h>
#include <easyrespacker.h>

#include <cu/cu_stl.h>
#include <gum/StringHelper.h>

#include <fstream>

#include <assert.h>

namespace edb
{

std::string PackLanguageTable::Command() const
{
	return "pack-lang";
}

std::string PackLanguageTable::Description() const
{
	return "pack language";
}

std::string PackLanguageTable::Usage() const
{
	std::string usage = Command() + " [src file] [dst file]";
	return usage;
}

int PackLanguageTable::Run(int argc, char *argv[])
{
	if (!check_number(this, argc, 4)) return -1;
	if (!check_file(argv[2])) return -1;

	Trigger(argv[2], argv[3]);

	return 0;
}

void PackLanguageTable::Trigger(const std::string& src_file, const std::string& dst_file)
{
	LoadFromCSV(src_file);
	PackToBin(dst_file);
}

void PackLanguageTable::LoadFromCSV(const std::string& filepath)
{
	m_header.clear();
	m_body.clear();

	std::ifstream fin(filepath.c_str());

	// header
	CU_STR str_line;
	getline(fin, str_line);
	gum::StringHelper::Split(str_line, ",", m_header);
	int language_num = m_header.size() - 1;

	// body
	while (getline(fin, str_line))
	{
		CU_VEC<CU_STR> items;
		gum::StringHelper::Split(str_line, ",", items);
		while (items.size() < language_num + 1) {
			items.push_back("");
		}
		assert(items.size() == language_num + 1);
		for (int i = 1, n = items.size(); i < n; ++i) {
			gum::StringHelper::Replace(items[i], "\\n", "\n");
		}
		m_body.push_back(items);<commit_msg>[FIXED] pack language should trans to utf8<commit_after>#include "PackLanguageTable.h"
#include "check_params.h"

#include <easysprpacker.h>
#include <easyrespacker.h>

#include <cu/cu_stl.h>
#include <gum/StringHelper.h>

#include <fstream>

#include <assert.h>

namespace edb
{

std::string PackLanguageTable::Command() const
{
	return "pack-lang";
}

std::string PackLanguageTable::Description() const
{
	return "pack language";
}

std::string PackLanguageTable::Usage() const
{
	std::string usage = Command() + " [src file] [dst file]";
	return usage;
}

int PackLanguageTable::Run(int argc, char *argv[])
{
	if (!check_number(this, argc, 4)) return -1;
	if (!check_file(argv[2])) return -1;

	Trigger(argv[2], argv[3]);

	return 0;
}

void PackLanguageTable::Trigger(const std::string& src_file, const std::string& dst_file)
{
	LoadFromCSV(src_file);
	PackToBin(dst_file);
}

void PackLanguageTable::LoadFromCSV(const std::string& filepath)
{
	m_header.clear();
	m_body.clear();

	std::ifstream fin(filepath.c_str());

	// header
	CU_STR str_line;
	getline(fin, str_line);
	gum::StringHelper::Split(str_line, ",", m_header);
	int language_num = m_header.size() - 1;
	for (auto& header : m_header) {
		header = gum::StringHelper::GBKToUTF8(header.c_str());
	}

	// body
	while (getline(fin, str_line))
	{
		CU_VEC<CU_STR> items;
		gum::StringHelper::Split(str_line, ",", items);
		while (items.size() < language_num + 1) {
			items.push_back("");
		}
		assert(items.size() == language_num + 1);
		for (int i = 1, n = items.size(); i < n; ++i) {
			gum::StringHelper::Replace(items[i], "\\n", "\n");
		}
		for (auto& item : items) {
			item = gum::StringHelper::GBKToUTF8(item.c_str());
		}
		m_body.push_back(items);
{'fraction_non_alphanumeric': 0.1788919839059115, 'fraction_numerical': 0.00773754255648406, 'mean_word_length': 5.287937743190661, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1673635', 'n_tokens_mistral': 1294, 'n_tokens_neox': 1233, 'n_words': 251}