<commit_before>#include "acmacs-base/fmt.hh"
#include "acmacs-base/csv.hh"
#include "acmacs-base/read-file.hh"
#include "acmacs-base/enumerate.hh"
#include "acmacs-virus/virus-name.hh"
#include "seqdb-3/seqdb.hh"
#include "locationdb/locdb.hh"
#include "acmacs-map-draw/export.hh"

// ----------------------------------------------------------------------

std::string export_layout_sequences_into_csv(std::string_view filename, const acmacs::chart::Chart& chart, size_t projection_no)
{
    auto antigens = chart.antigens();
    auto sera = chart.sera();
    auto layout = chart.projection(projection_no)->layout();
    const auto number_of_dimensions = layout->number_of_dimensions();
    const auto& seqdb = acmacs::seqdb::get();
    const auto& locdb = get_locdb();
    const auto entry_seqs = seqdb.match(*antigens, chart.info()->virus_type(acmacs::chart::Info::Compute::Yes)); // entry for each antigen

    acmacs::CsvWriter writer;

    writer.add_field("AG/SR");
    writer.add_field("name");
    writer.add_field("x");
    if (number_of_dimensions > acmacs::number_of_dimensions_t{1}) {
        writer.add_field("y");
        if (number_of_dimensions > acmacs::number_of_dimensions_t{2})
            writer.add_field("z");
    }
    writer.add_field("date");
    writer.add_field("reassortant");
    writer.add_field("annotations");
    writer.add_field("passage/serum_id");
    writer.add_field("country");
    writer.add_field("region");
    writer.add_field("latitude");
    writer.add_field("longitude");
    writer.add_field("sequence");
    writer.new_row();

    auto add_location_data = [&writer, &locdb](std::string name) {
        try {
            const auto find_for_virus_name = [&locdb](std::string aVirusName) {
                try {
                    return locdb.find(::virus_name::location(aVirusName));
                }
                catch (std::exception&) {
                    return locdb.find(::virus_name::location_for_cdc_name(aVirusName));
                }
            };

            const auto location = find_for_virus_name(name);
            writer.add_field(std::string{location.country()});
            writer.add_field(std::string{locdb.continent_of_country(location.country())});
            writer.add_field(std::to_string(location.latitude()));
            writer.add_field(std::to_string(location.longitude()));
        }
        catch (LocationNotFound& /*err*/) {
            fmt::print(stderr, "WARNING: LocationNotFound: \"{}\" of \"{}\"\n", virus_name::location(name), name);
        }
        catch (virus_name::Unrecognized& /*err*/) {
            fmt::print(stderr, "WARNING: cannot parse name to find location: \"{}\"\n", name);
        }
    };

    for (auto [ag_no, antigen] : acmacs::enumerate(*antigens)) {
        // fmt::print(stderr, "DEBUG: {} {}\n", ag_no, antigen->name());
        writer.add_field("AG");
        writer.add_field(antigen->name());
        for (auto dim : acmacs::range(number_of_dimensions)) {
            if (const auto coord = layout->coordinate(ag_no, dim); !std::isnan(coord))
                writer.add_field(acmacs::to_string(coord));
            else
                writer.add_field("");
        }
        writer.add_field(antigen->date());
        writer.add_field(antigen->reassortant());
        writer.add_field(antigen->annotations().join());
        writer.add_field(antigen->passage());
        add_location_data(antigen->name());
        if (const auto& entry_seq = entry_seqs[ag_no]; entry_seq) {
            try {
                writer.add_field(std::string{entry_seq.seq().aa_aligned()});
            }
            catch (std::exception& err) {
                fmt::print(stderr, "WARNING: {} {}: {}\n", ag_no, antigen->full_name(), err);
                writer.add_field("");
            }
        }
        else
            writer.add_field("");
        writer.new_row();
    }

    const auto number_of_antigens = antigens->size();
    for (auto [sr_no, serum] : acmacs::enumerate(*sera)) {
        writer.add_field("SR");
        writer.add_field(serum->name());
        for (auto dim : acmacs::range(number_of_dimensions)) {
            if (const auto coord = layout->coordinate(sr_no + number_of_antigens, dim); !std::isnan(coord))
                writer.add_field(acmacs::to_string(coord));
            else
                writer.add_field("");
        }
        writer.add_field(""); // date
        writer.add_field(serum->reassortant());
        writer.add_field(serum->annotations().join());
        writer.add_field(serum->serum_id());
        add_location_data(serum->name());
        writer.add_field(""); // sequence
        writer.new_row();
    }

    if (!filename.empty())<commit_msg>export lab_id into layout-sequences csv<commit_after>#include "acmacs-base/fmt.hh"
#include "acmacs-base/csv.hh"
#include "acmacs-base/read-file.hh"
#include "acmacs-base/enumerate.hh"
#include "acmacs-virus/virus-name.hh"
#include "seqdb-3/seqdb.hh"
#include "locationdb/locdb.hh"
#include "acmacs-map-draw/export.hh"

// ----------------------------------------------------------------------

std::string export_layout_sequences_into_csv(std::string_view filename, const acmacs::chart::Chart& chart, size_t projection_no)
{
    auto antigens = chart.antigens();
    auto sera = chart.sera();
    auto layout = chart.projection(projection_no)->layout();
    const auto number_of_dimensions = layout->number_of_dimensions();
    const auto& seqdb = acmacs::seqdb::get();
    const auto& locdb = get_locdb();
    const auto entry_seqs = seqdb.match(*antigens, chart.info()->virus_type(acmacs::chart::Info::Compute::Yes)); // entry for each antigen

    acmacs::CsvWriter writer;

    writer.add_field("AG/SR");
    writer.add_field("name");
    writer.add_field("x");
    if (number_of_dimensions > acmacs::number_of_dimensions_t{1}) {
        writer.add_field("y");
        if (number_of_dimensions > acmacs::number_of_dimensions_t{2})
            writer.add_field("z");
    }
    writer.add_field("date");
    writer.add_field("reassortant");
    writer.add_field("annotations");
    writer.add_field("passage/serum_id");
    writer.add_field("lab_id");
    writer.add_field("country");
    writer.add_field("region");
    writer.add_field("latitude");
    writer.add_field("longitude");
    writer.add_field("sequence");
    writer.new_row();

    auto add_location_data = [&writer, &locdb](std::string name) {
        try {
            const auto find_for_virus_name = [&locdb](std::string aVirusName) {
                try {
                    return locdb.find(::virus_name::location(aVirusName));
                }
                catch (std::exception&) {
                    return locdb.find(::virus_name::location_for_cdc_name(aVirusName));
                }
            };

            const auto location = find_for_virus_name(name);
            writer.add_field(std::string{location.country()});
            writer.add_field(std::string{locdb.continent_of_country(location.country())});
            writer.add_field(std::to_string(location.latitude()));
            writer.add_field(std::to_string(location.longitude()));
        }
        catch (LocationNotFound& /*err*/) {
            fmt::print(stderr, "WARNING: LocationNotFound: \"{}\" of \"{}\"\n", virus_name::location(name), name);
        }
        catch (virus_name::Unrecognized& /*err*/) {
            fmt::print(stderr, "WARNING: cannot parse name to find location: \"{}\"\n", name);
        }
    };

    for (auto [ag_no, antigen] : acmacs::enumerate(*antigens)) {
        // fmt::print(stderr, "DEBUG: {} {}\n", ag_no, antigen->name());
        writer.add_field("AG");
        writer.add_field(antigen->name());
        for (auto dim : acmacs::range(number_of_dimensions)) {
            if (const auto coord = layout->coordinate(ag_no, dim); !std::isnan(coord))
                writer.add_field(acmacs::to_string(coord));
            else
                writer.add_field("");
        }
        writer.add_field(antigen->date());
        writer.add_field(antigen->reassortant());
        writer.add_field(antigen->annotations().join());
        writer.add_field(antigen->passage());
        writer.add_field(antigen->lab_ids().join());
        add_location_data(antigen->name());
        if (const auto& entry_seq = entry_seqs[ag_no]; entry_seq) {
            try {
                writer.add_field(std::string{entry_seq.seq().aa_aligned()});
            }
            catch (std::exception& err) {
                fmt::print(stderr, "WARNING: {} {}: {}\n", ag_no, antigen->full_name(), err);
                writer.add_field("");
            }
        }
        else
            writer.add_field("");
        writer.new_row();
    }

    const auto number_of_antigens = antigens->size();
    for (auto [sr_no, serum] : acmacs::enumerate(*sera)) {
        writer.add_field("SR");
        writer.add_field(serum->name());
        for (auto dim : acmacs::range(number_of_dimensions)) {
            if (const auto coord = layout->coordinate(sr_no + number_of_antigens, dim); !std::isnan(coord))
                writer.add_field(acmacs::to_string(coord));
            else
                writer.add_field("");
        }
        writer.add_field(""); // date
        writer.add_field(serum->reassortant());
        writer.add_field(serum->annotations().join());
        writer.add_field(serum->serum_id());
        writer.add_field(""); // lab_id
        add_location_data(serum->name());
        writer.add_field(""); // sequence
        writer.new_row();
    }

    if (!filename.empty())
{'fraction_non_alphanumeric': 0.17280601064384848, 'fraction_numerical': 0.0006261087342168423, 'mean_word_length': 2.589513108614232, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5058489', 'n_tokens_mistral': 3038, 'n_tokens_neox': 2960, 'n_words': 439}