<commit_before>#include "acmacs-base/range-v3.hh"
#include "acmacs-base/log.hh"
#include "acmacs-base/string-compare.hh"
#include "acmacs-whocc/sheet-extractor.hh"

// ----------------------------------------------------------------------

#include "acmacs-base/global-constructors-push.hh"

static const std::regex re_table_title_crick{R"(^Table\s+[XY0-9-]+\.\s*Antigenic analys[ie]s of influenza ([AB](?:\(H3N2\)|\(H1N1\)pdm09)?) viruses\s*-?\s*\(?(Plaque Reduction Neutralisation \(MDCK-SIAT\)|(?:Victoria|Yamagata)\s+lineage)?\)?\s*\(?(20[0-2][0-9]-[01][0-9]-[0-3][0-9])\)?)", acmacs::regex::icase};

static const std::regex re_antigen_name{"^[AB]/[A-Z '_-]+/[^/]+/[0-9]+", acmacs::regex::icase};
static const std::regex re_antigen_passage{"^(MDCK|SIAT|E|HCK)[0-9X]", acmacs::regex::icase};

#include "acmacs-base/diagnostics-pop.hh"

// ----------------------------------------------------------------------

std::unique_ptr<acmacs::sheet::Extractor> acmacs::sheet::v1::extractor_factory(const Sheet& sheet)
{
    std::unique_ptr<Extractor> extractor;
    std::smatch match;
    if (sheet.matches(re_table_title_crick, match, 0, 0)) {
        if (acmacs::string::startswith_ignore_case(match.str(2), "Plaque")) {
            extractor = std::make_unique<ExtractorCrickPRN>(sheet);
        }
        else {
            extractor = std::make_unique<ExtractorCrick>(sheet);
            auto subtype = match.str(1);
            extractor->subtype(subtype);
            if (subtype == "B" && match.length(2) > 4) {
                switch (match.str(2)[0]) {
                    case 'V':
                    case 'v':
                        extractor->lineage("VICTORIA");
                        break;
                    case 'Y':
                    case 'y':
                        extractor->lineage("YAMAGATA");
                        break;
                }
            }
        }
        extractor->date(date::from_string(match.str(3), date::allow_incomplete::no, date::throw_on_error::no));
    }
    else {
        AD_WARNING("no specific extractor found");
        extractor = std::make_unique<Extractor>(sheet);
    }
    extractor->preprocess();
    return extractor;

} // acmacs::sheet::v1::extractor_factory

// ----------------------------------------------------------------------

std::string acmacs::sheet::v1::Extractor::antigen_name(size_t ag_no) const
{
    if (antigen_name_column().has_value())
        return fmt::format("{}", sheet().cell(antigen_rows().at(ag_no), *antigen_name_column()));
    else
        return {};

} // acmacs::sheet::v1::Extractor::antigen_name

// ----------------------------------------------------------------------

std::string acmacs::sheet::v1::Extractor::antigen_date(size_t ag_no) const
{
    if (antigen_date_column().has_value())
        return fmt::format("{}", sheet().cell(antigen_rows().at(ag_no), *antigen_date_column()));
    else
        return {};

} // acmacs::sheet::v1::Extractor::antigen_date

// ----------------------------------------------------------------------

std::string acmacs::sheet::v1::Extractor::antigen_passage(size_t ag_no) const
{
    if (antigen_passage_column().has_value())
        return fmt::format("{}", sheet().cell(antigen_rows().at(ag_no), *antigen_passage_column()));
    else
        return {};

} // acmacs::sheet::v1::Extractor::antigen_passage

// ----------------------------------------------------------------------

void acmacs::sheet::v1::Extractor::preprocess()
{
    find_titers();
    find_antigen_name_column();
    find_antigen_date_column();
    find_antigen_passage_column();
    find_serum_rows();

} // acmacs::sheet::v1::Extractor::preprocess

// ----------------------------------------------------------------------

void acmacs::sheet::v1::Extractor::find_titers()
{
    std::vector<std::pair<size_t, range>> rows;
    range common;
    for (const auto row : range_from_0_to(sheet().number_of_rows())) {
        if (auto titers = sheet().titer_range(row); !titers.empty() && titers.first > 0) {
            // AD_DEBUG("row:{:2d} {:c}:{:c}", row + 1, titers.first + 'A', titers.second - 1 + 'A');
            if (!common.valid())
                common = titers;
            else if (common != titers)
                AD_WARNING("variable titer row ranges: {:c}:{:c} vs. {:c}:{:c}", titers.first + 'A', titers.second - 1 + 'A', common.first + 'A', common.second - 1 + 'A');
            rows.emplace_back(row, std::move(titers));
        }
    }

    titer_columns_ = common;
    antigen_rows_.resize(rows.size());
    std::transform(std::begin(rows), std::end(rows), std::begin(antigen_rows_), [](const auto& row) { return row.first; });

} // acmacs::sheet::v1::Extractor::find_titers

// ----------------------------------------------------------------------

void acmacs::sheet::v1::Extractor::find_antigen_name_column()
{
    const auto is_name = [this](size_t row, size_t col) {
        if (sheet().matches(re_antigen_name, row, col)) {
            longest_antigen_name_ = std::max(longest_antigen_name_, sheet().size(row, col));
            return true;
        }
        else
            return false;
    };

    for (const auto col : range_from_0_to(titer_columns_.first)) { // to the left from titers
        if (static_cast<size_t>(ranges::count_if(antigen_rows_, [col, is_name](size_t row) { return is_name(row, col); })) > (antigen_rows_.size() / 2)) {
            antigen_name_column_ = col;
            break;
        }
    }

    if (antigen_name_column_.has_value()) {
        AD_INFO("Antigen name column: {:c}", *antigen_name_column_ + 'A');
        // remote antigen rows that have no name
        ranges::actions::remove_if(antigen_rows_, [this, is_name](size_t row) { return !is_name(row, *antigen_name_column_); });
    }
    else
        AD_WARNING("Antigen name column not found");

} // acmacs::sheet::v1::Extractor::find_antigen_name_column

// ----------------------------------------------------------------------

void acmacs::sheet::v1::Extractor::find_antigen_date_column()
{
    for (const auto col : range_from_0_to(sheet().number_of_columns())) {
        if (static_cast<size_t>(ranges::count_if(antigen_rows_, [col, this](size_t row) { return sheet().is_date(row, col); })) >= (antigen_rows_.size() / 2)) {
            antigen_date_column_ = col;
            break;
        }
    }

    if (antigen_date_column_.has_value())
        AD_INFO("Antigen date column: {:c}", *antigen_date_column_ + 'A');
    else
        AD_WARNING("Antigen date column not found");

} // acmacs::sheet::v1::Extractor::find_antigen_date_column

// ----------------------------------------------------------------------

void acmacs::sheet::v1::Extractor::find_antigen_passage_column()
{
    const auto is_passage = [this](size_t row, size_t col) {
        if (sheet().matches(re_antigen_passage, row, col)) {
            longest_antigen_passage_ = std::max(longest_antigen_passage_, sheet().size(row, col));
            return true;
        }
        else
            return false;
    };

    for (const auto col : range_from_0_to(sheet().number_of_columns())) {
        if (static_cast<size_t>(ranges::count_if(antigen_rows_, [col, is_passage](size_t row) { return is_passage(row, col); })) >= (antigen_rows_.size() / 2)) {
            antigen_passage_column_ = col;
            break;
        }
    }

    if (antigen_passage_column_.has_value())
        AD_INFO("Antigen passage column: {:c}", *antigen_passage_column_ + 'A');
    else
        AD_WARNING("Antigen passage column not found");

} // acmacs::sheet::v1::Extractor::find_antigen_passage_column

// ----------------------------------------------------------------------

void acmacs::sheet::v1::Extractor::find_serum_rows()
{

} // acmacs::sheet::v1::Extractor::find_serum_rows

// ======================================================================

acmacs::sheet::v1::ExtractorCrick::ExtractorCrick(const Sheet& a_sheet)
    : Extractor(a_sheet)
{
    lab("CRICK");

} // acmacs::sheet::v1::ExtractorCrick::ExtractorCrick

// ----------------------------------------------------------------------

void acmacs::sheet::v1::ExtractorCrick::find_serum_rows()
{

} // acmacs::sheet::v1::ExtractorCrick::find_serum_rows


// ======================================================================
<commit_msg>whocc-xlsx-to-torg<commit_after>#include "acmacs-base/range-v3.hh"
#include "acmacs-base/log.hh"
#include "acmacs-base/string-compare.hh"
#include "acmacs-whocc/sheet-extractor.hh"

// ----------------------------------------------------------------------

#include "acmacs-base/global-constructors-push.hh"

static const std::regex re_table_title_crick{R"(^Table\s+[XY0-9-]+\.\s*Antigenic analys[ie]s of influenza ([AB](?:\(H3N2\)|\(H1N1\)pdm09)?) viruses\s*-?\s*\(?(Plaque Reduction Neutralisation \(MDCK-SIAT\)|(?:Victoria|Yamagata)\s+lineage)?\)?\s*\(?(20[0-2][0-9]-[01][0-9]-[0-3][0-9])\)?)", acmacs::regex::icase};

static const std::regex re_antigen_name{"^[AB]/[A-Z '_-]+/[^/]+/[0-9]+", acmacs::regex::icase};
static const std::regex re_antigen_passage{"^(MDCK|SIAT|E|HCK)[0-9X]", acmacs::regex::icase};

static const std::regex re_crick_serum_name_1{"^[AB]/[A-Z '_-]+$", acmacs::regex::icase};
static const std::regex re_crick_serum_name_2{"^[A-Z0-9-]+(/[0-9]+)?$", acmacs::regex::icase};

#include "acmacs-base/diagnostics-pop.hh"

// ----------------------------------------------------------------------

std::unique_ptr<acmacs::sheet::Extractor> acmacs::sheet::v1::extractor_factory(const Sheet& sheet)
{
    std::unique_ptr<Extractor> extractor;
    std::smatch match;
    if (sheet.matches(re_table_title_crick, match, 0, 0)) {
        if (acmacs::string::startswith_ignore_case(match.str(2), "Plaque")) {
            extractor = std::make_unique<ExtractorCrickPRN>(sheet);
        }
        else {
            extractor = std::make_unique<ExtractorCrick>(sheet);
            auto subtype = match.str(1);
            extractor->subtype(subtype);
            if (subtype == "B" && match.length(2) > 4) {
                switch (match.str(2)[0]) {
                    case 'V':
                    case 'v':
                        extractor->lineage("VICTORIA");
                        break;
                    case 'Y':
                    case 'y':
                        extractor->lineage("YAMAGATA");
                        break;
                }
            }
        }
        extractor->date(date::from_string(match.str(3), date::allow_incomplete::no, date::throw_on_error::no));
    }
    else {
        AD_WARNING("no specific extractor found");
        extractor = std::make_unique<Extractor>(sheet);
    }
    extractor->preprocess();
    return extractor;

} // acmacs::sheet::v1::extractor_factory

// ----------------------------------------------------------------------

std::string acmacs::sheet::v1::Extractor::antigen_name(size_t ag_no) const
{
    if (antigen_name_column().has_value())
        return fmt::format("{}", sheet().cell(antigen_rows().at(ag_no), *antigen_name_column()));
    else
        return {};

} // acmacs::sheet::v1::Extractor::antigen_name

// ----------------------------------------------------------------------

std::string acmacs::sheet::v1::Extractor::antigen_date(size_t ag_no) const
{
    if (antigen_date_column().has_value())
        return fmt::format("{}", sheet().cell(antigen_rows().at(ag_no), *antigen_date_column()));
    else
        return {};

} // acmacs::sheet::v1::Extractor::antigen_date

// ----------------------------------------------------------------------

std::string acmacs::sheet::v1::Extractor::antigen_passage(size_t ag_no) const
{
    if (antigen_passage_column().has_value())
        return fmt::format("{}", sheet().cell(antigen_rows().at(ag_no), *antigen_passage_column()));
    else
        return {};

} // acmacs::sheet::v1::Extractor::antigen_passage

// ----------------------------------------------------------------------

void acmacs::sheet::v1::Extractor::preprocess()
{
    find_titers();
    find_antigen_name_column();
    find_antigen_date_column();
    find_antigen_passage_column();
    find_serum_rows();

} // acmacs::sheet::v1::Extractor::preprocess

// ----------------------------------------------------------------------

void acmacs::sheet::v1::Extractor::find_titers()
{
    std::vector<std::pair<size_t, range>> rows;
    range common;
    for (const auto row : range_from_0_to(sheet().number_of_rows())) {
        if (auto titers = sheet().titer_range(row); !titers.empty() && titers.first > 0) {
            // AD_DEBUG("row:{:2d} {:c}:{:c}", row + 1, titers.first + 'A', titers.second - 1 + 'A');
            if (!common.valid())
                common = titers;
            else if (common != titers)
                AD_WARNING("variable titer row ranges: {:c}:{:c} vs. {:c}:{:c}", titers.first + 'A', titers.second - 1 + 'A', common.first + 'A', common.second - 1 + 'A');
            rows.emplace_back(row, std::move(titers));
        }
    }

    titer_columns_ = common;
    antigen_rows_.resize(rows.size());
    std::transform(std::begin(rows), std::end(rows), std::begin(antigen_rows_), [](const auto& row) { return row.first; });

} // acmacs::sheet::v1::Extractor::find_titers

// ----------------------------------------------------------------------

void acmacs::sheet::v1::Extractor::find_antigen_name_column()
{
    const auto is_name = [this](size_t row, size_t col) {
        if (sheet().matches(re_antigen_name, row, col)) {
            longest_antigen_name_ = std::max(longest_antigen_name_, sheet().size(row, col));
            return true;
        }
        else
            return false;
    };

    for (const auto col : range_from_0_to(titer_columns_.first)) { // to the left from titers
        if (static_cast<size_t>(ranges::count_if(antigen_rows_, [col, is_name](size_t row) { return is_name(row, col); })) > (antigen_rows_.size() / 2)) {
            antigen_name_column_ = col;
            break;
        }
    }

    if (antigen_name_column_.has_value()) {
        AD_INFO("Antigen name column: {:c}", *antigen_name_column_ + 'A');
        // remote antigen rows that have no name
        ranges::actions::remove_if(antigen_rows_, [this, is_name](size_t row) { return !is_name(row, *antigen_name_column_); });
    }
    else
        AD_WARNING("Antigen name column not found");

} // acmacs::sheet::v1::Extractor::find_antigen_name_column

// ----------------------------------------------------------------------

void acmacs::sheet::v1::Extractor::find_antigen_date_column()
{
    for (const auto col : range_from_0_to(sheet().number_of_columns())) {
        if (static_cast<size_t>(ranges::count_if(antigen_rows_, [col, this](size_t row) { return sheet().is_date(row, col); })) >= (antigen_rows_.size() / 2)) {
            antigen_date_column_ = col;
            break;
        }
    }

    if (antigen_date_column_.has_value())
        AD_INFO("Antigen date column: {:c}", *antigen_date_column_ + 'A');
    else
        AD_WARNING("Antigen date column not found");

} // acmacs::sheet::v1::Extractor::find_antigen_date_column

// ----------------------------------------------------------------------

void acmacs::sheet::v1::Extractor::find_antigen_passage_column()
{
    const auto is_passage = [this](size_t row, size_t col) {
        if (sheet().matches(re_antigen_passage, row, col)) {
            longest_antigen_passage_ = std::max(longest_antigen_passage_, sheet().size(row, col));
            return true;
        }
        else
            return false;
    };

    for (const auto col : range_from_0_to(sheet().number_of_columns())) {
        if (static_cast<size_t>(ranges::count_if(antigen_rows_, [col, is_passage](size_t row) { return is_passage(row, col); })) >= (antigen_rows_.size() / 2)) {
            antigen_passage_column_ = col;
            break;
        }
    }

    if (antigen_passage_column_.has_value())
        AD_INFO("Antigen passage column: {:c}", *antigen_passage_column_ + 'A');
    else
        AD_WARNING("Antigen passage column not found");

} // acmacs::sheet::v1::Extractor::find_antigen_passage_column

// ----------------------------------------------------------------------

void acmacs::sheet::v1::Extractor::find_serum_rows()
{

} // acmacs::sheet::v1::Extractor::find_serum_rows

// ======================================================================

acmacs::sheet::v1::ExtractorCrick::ExtractorCrick(const Sheet& a_sheet)
    : Extractor(a_sheet)
{
    lab("CRICK");

} // acmacs::sheet::v1::ExtractorCrick::ExtractorCrick

// ----------------------------------------------------------------------

void acmacs::sheet::v1::ExtractorCrick::find_serum_rows()
{
    fmt::memory_buffer report;
    for (const auto row : range_from_to(1ul, antigen_rows()[0])) {
        if (const size_t matches = static_cast<size_t>(ranges::count_if(range_from_to(titer_columns()), [row, this](size_t col) { return sheet().matches(re_crick_serum_name_1, row, col); })); matches == titer_columns().size()) {
            serum_name_1_row_ = row;
            break;
        }
        else if (matches)
            fmt::format_to(report, "    re_crick_serum_name_1 row:{} matches:{}\n", row, matches);
    }

    if (serum_name_1_row_.has_value())
        AD_INFO("[Crick]: Serum name row 1: {}", *serum_name_1_row_);
    else
        AD_WARNING("[Crick]: No serum name row 1 found (titer columns: {:c}:{:c} size:{})\n{}", titer_columns().first + 'A', titer_columns().second + 'A', titer_columns().size(), fmt::to_string(report));

    if (serum_name_1_row_.has_value() && static_cast<size_t>(ranges::count_if(range_from_to(titer_columns()), [this](size_t col) { return sheet().matches(re_crick_serum_name_2, *serum_name_1_row_ + 1, col); })) == titer_columns().size())
            serum_name_2_row_ = *serum_name_1_row_ + 1;
    else
        AD_DEBUG("re_crick_serum_name_2 {}: {}", *serum_name_1_row_ + 1, static_cast<size_t>(ranges::count_if(range_from_to(titer_columns()), [this](size_t col) { return sheet().matches(re_crick_serum_name_2, *serum_name_1_row_ + 1, col); })));

    if (serum_name_2_row_.has_value())
        AD_INFO("[Crick]: Serum name row 2: {}", *serum_name_2_row_);
    else
        AD_WARNING("[Crick]: No serum name row 2 found");

} // acmacs::sheet::v1::ExtractorCrick::find_serum_rows


// ======================================================================

{'fraction_non_alphanumeric': 0.26171322969888033, 'fraction_numerical': 0.009674964670072833, 'mean_word_length': 2.3642347778387274, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1266094', 'n_tokens_mistral': 6048, 'n_tokens_neox': 5886, 'n_words': 880}