<commit_before>/*
 *  qi_glmsetup.cpp
 *
 *  Copyright (c) 2016 <NAME>.
 *
 *  This Source Code Form is subject to the terms of the Mozilla Public
 *  License, v. 2.0. If a copy of the MPL was not distributed with this
 *  file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 */

#include <iostream>
#include <fstream>
#include <string>
#include <algorithm>

#include "itkTileImageFilter.h"
#include "itkSubtractImageFilter.h"
#include "itkDivideImageFilter.h"
#include "Filters/MeanImageFilter.h"

#include "QI/Types.h"
#include "QI/Util.h"
#include "QI/Option.h"

const std::string usage{
"Usage is: qi_glmsetup [options] files\n\
\n\
A utility for setting up merged 4D files for use with FSL randomise etc.\n\
Input individual files as arguments.\n\
The group for each input file must be specified with --groups (one per line)\n\
A value of 0 means ignore this file.\n\
An output file must be specified with --out.\n\
\n\
Additionally, simple GLM including co-variates can be generated.\n"};

int main(int argc, char **argv) {
    Eigen::initParallel();
    QI::OptionList opts(usage);
    QI::Switch sort('s',"sort","Sort merged file and GLM in ascending group order", opts);
    QI::Option<std::string> covars_path("",'C',"covars","Path to covariates file (added to design matrix)", opts);
    QI::Option<std::string> ftests_path("",'f',"ftests","Generate and save F-tests", opts);
    QI::Option<std::string> contrasts_path("",'c',"contrasts","Generate and save contrasts", opts);
    QI::Option<std::string> design_path("",'d',"design","Path to save design matrix", opts);
    QI::Option<std::string> output_path("",'o',"out","Path for output merged file", opts);
    QI::Option<std::string> group_path("",'g',"groups","File to read group numbers from", opts);
    QI::Switch verbose('v',"verbose","Print more information", opts);
    QI::Help help(opts);
    std::deque<std::string> file_paths = opts.parse(argc, argv);
    if (!group_path.set()) {
        std::cerr << opts << std::endl;
        std::cerr << "Group file must be set with --groups option" << std::endl;
        return EXIT_FAILURE;
    }
    if (!output_path.set()) {
        std::cerr << opts << std::endl;
        std::cerr << "Output file must be set with --out option" << std::endl;
        return EXIT_FAILURE;
    }

    std::ifstream group_file(*group_path);
    if (!group_file) {
        std::cerr << "Group file: " << *group_path << " does not exist" << std::endl;
        return EXIT_FAILURE;
    }
    if (*verbose) std::cout << "Reading group file" << std::endl;
    std::vector<int> group_list;
    int temp;
    while (group_file >> temp) {
        group_list.push_back(temp);
    }
    int n_groups = *std::max_element(group_list.begin(), group_list.end());
    int n_images = std::count_if(group_list.begin(), group_list.end(), [](int i){return i > 0;}); // Count non-zero elements

    if (file_paths.size() != group_list.size()) {
        std::cerr << "Group list size and number of files do not match." << std::endl;
        return EXIT_FAILURE;
    }

    std::vector<std::vector<QI::VolumeF::Pointer>> groups(n_groups);
    if (*verbose) std::cout << "Number of groups: " << n_groups << std::endl;
    if (*verbose) std::cout << "Number of images: " << n_images << std::endl;
    itk::FixedArray<unsigned int, 4> layout;
    layout[0] = layout[1] = layout[2] = 1;
    layout[3] = n_images;
    auto tiler = itk::TileImageFilter<QI::VolumeF, QI::SeriesF>::New();
    tiler->SetLayout(layout);

    std::ofstream design_file;
    if (design_path.set()) {
        if (*verbose) std::cout << "Design matrix will be saved to: " << *design_path << std::endl;
        design_file = std::ofstream(*design_path);
    }
    std::vector<std::vector<std::vector<double>>> covars(n_groups);
    std::ifstream covars_file;
    if (covars_path.set()) {
        covars_file.open(*covars_path);
        if (!covars_file) {
            std::cerr << "Failed to open covariate file: " << *covars_path << std::endl;
            return EXIT_FAILURE;
        }
    }
    int out_index = 0;
    for (int i = 0; i < group_list.size(); i++) {
        const int group = group_list.at(i);
        if (group > 0) { // Ignore entries with a 0
            if (*verbose) std::cout << "File: " << file_paths.at(i) << " Group: " << group << std::flush;
            QI::VolumeF::Pointer ptr = QI::ReadImage(file_paths.at(i));
            groups.at(group - 1).push_back(ptr);
            std::vector<double> covar;
            if (covars_path.set()) {
                std::string covar_line;
                std::getline(covars_file, covar_line);
                std::stringstream css(covar_line);
                double c;
                while (css >> c) {
                    covar.push_back(c);
                } 
                covars.at(group - 1).push_back(covar);
                if (*verbose) std::cout << ", read covariates.";
            }
            if (*verbose) std::cout << std::endl;
            if (!*sort) {
                tiler->SetInput(out_index, ptr);
                out_index++;
                if (design_path.set()) {
                    for (int g = 1; g <= n_groups; g++) {
                        if (g == group) {
                            design_file << "1\t";
                        } else {
                            design_file << "0\t";
                        }
                    }
                    if (covars_path.set()) {
                        for (auto& c : covar) {
                            design_file << c << "\t";
                        }
                    }
                    design_file << std::endl;
                }
            }
        } else {
            if (*verbose) std::cout << "Ignoring file: " << file_paths.at(i) << std::endl;
            // Eat a line in the covariates file as well
            if (covars_path.set()) {
                std::string covar_line;
                std::getline(covars_file, covar_line);
            }
        }
    }
    if (*sort) {
        if (*verbose) std::cout << "Sorting." << std::endl;
        for (int g = 0; g < n_groups; g++) {
            for (int i = 0; i < groups.at(g).size(); i++) {
                tiler->SetInput(out_index, groups.at(g).at(i));
                out_index++;
                if (design_path.set()) {
                    for (int g2 = 0; g2 < n_groups; g2++) {
                        if (g2 == g) {
                            design_file << "1\t";
                        } else {
                            design_file << "0\t";
                        }
                    }<commit_msg>ENH: Allowed multiple covariates files.<commit_after>/*
 *  qi_glmsetup.cpp
 *
 *  Copyright (c) 2016 <NAME>.
 *
 *  This Source Code Form is subject to the terms of the Mozilla Public
 *  License, v. 2.0. If a copy of the MPL was not distributed with this
 *  file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 */

#include <iostream>
#include <fstream>
#include <string>
#include <algorithm>

#include "itkTileImageFilter.h"
#include "itkSubtractImageFilter.h"
#include "itkDivideImageFilter.h"
#include "Filters/MeanImageFilter.h"

#include "QI/Types.h"
#include "QI/Util.h"
#include "QI/Option.h"

const std::string usage{
"Usage is: qi_glmsetup [options] files\n\
\n\
A utility for setting up merged 4D files for use with FSL randomise etc.\n\
Input individual files as arguments.\n\
The group for each input file must be specified with --groups (one per line)\n\
A value of 0 means ignore this file.\n\
An output file must be specified with --out.\n\
\n\
Additionally, simple GLM including co-variates can be generated.\n"};

int main(int argc, char **argv) {
    Eigen::initParallel();
    QI::OptionList opts(usage);
    QI::Switch sort('s',"sort","Sort merged file and GLM in ascending group order", opts);
    QI::Option<std::string> covars_path("",'C',"covars","Path to covariates file (added to design matrix)", opts);
    QI::Option<std::string> ftests_path("",'f',"ftests","Generate and save F-tests", opts);
    QI::Option<std::string> contrasts_path("",'c',"contrasts","Generate and save contrasts", opts);
    QI::Option<std::string> design_path("",'d',"design","Path to save design matrix", opts);
    QI::Option<std::string> output_path("",'o',"out","Path for output merged file", opts);
    QI::Option<std::string> group_path("",'g',"groups","File to read group numbers from", opts);
    QI::Switch verbose('v',"verbose","Print more information", opts);
    QI::Help help(opts);
    std::deque<std::string> file_paths = opts.parse(argc, argv);
    if (!group_path.set()) {
        std::cerr << opts << std::endl;
        std::cerr << "Group file must be set with --groups option" << std::endl;
        return EXIT_FAILURE;
    }
    if (!output_path.set()) {
        std::cerr << opts << std::endl;
        std::cerr << "Output file must be set with --out option" << std::endl;
        return EXIT_FAILURE;
    }

    std::ifstream group_file(*group_path);
    if (!group_file) {
        std::cerr << "Group file: " << *group_path << " does not exist" << std::endl;
        return EXIT_FAILURE;
    }
    if (*verbose) std::cout << "Reading group file" << std::endl;
    std::vector<int> group_list;
    int temp;
    while (group_file >> temp) {
        group_list.push_back(temp);
    }
    int n_groups = *std::max_element(group_list.begin(), group_list.end());
    int n_images = std::count_if(group_list.begin(), group_list.end(), [](int i){return i > 0;}); // Count non-zero elements

    if (file_paths.size() != group_list.size()) {
        std::cerr << "Group list size and number of files do not match." << std::endl;
        return EXIT_FAILURE;
    }

    std::vector<std::vector<QI::VolumeF::Pointer>> groups(n_groups);
    if (*verbose) std::cout << "Number of groups: " << n_groups << std::endl;
    if (*verbose) std::cout << "Number of images: " << n_images << std::endl;
    itk::FixedArray<unsigned int, 4> layout;
    layout[0] = layout[1] = layout[2] = 1;
    layout[3] = n_images;
    auto tiler = itk::TileImageFilter<QI::VolumeF, QI::SeriesF>::New();
    tiler->SetLayout(layout);

    std::ofstream design_file;
    if (design_path.set()) {
        if (*verbose) std::cout << "Design matrix will be saved to: " << *design_path << std::endl;
        design_file = std::ofstream(*design_path);
    }
    std::vector<std::vector<std::vector<double>>> covars(n_groups);
    std::vector<std::ifstream> covars_files;
    if (covars_path.set()) {
        std::istringstream stream_covars(*covars_path);
        while (!stream_covars.eof()) {
            std::string path;
            getline(stream_covars, path, ',');
            std::ifstream covars_file(path);
            if (!covars_file) {
                std::cerr << "Failed to open covariate file: " << path << std::endl;
                return EXIT_FAILURE;
            }
            covars_files.push_back(std::move(covars_file));
        }
    }
    int out_index = 0;
    for (int i = 0; i < group_list.size(); i++) {
        const int group = group_list.at(i);
        if (group > 0) { // Ignore entries with a 0
            if (*verbose) std::cout << "File: " << file_paths.at(i) << " Group: " << group << std::flush;
            QI::VolumeF::Pointer ptr = QI::ReadImage(file_paths.at(i));
            groups.at(group - 1).push_back(ptr);
            std::vector<double> covar;
            if (covars_path.set()) {
                for (auto &f : covars_files) {
                    double c;
                    f >> c;
                    covar.push_back(c);
                }
                covars.at(group - 1).push_back(covar);
                if (*verbose) std::cout << ", read covariates.";
            }
            if (*verbose) std::cout << std::endl;
            if (!*sort) {
                tiler->SetInput(out_index, ptr);
                out_index++;
                if (design_path.set()) {
                    for (int g = 1; g <= n_groups; g++) {
                        if (g == group) {
                            design_file << "1\t";
                        } else {
                            design_file << "0\t";
                        }
                    }
                    if (covars_path.set()) {
                        for (auto& c : covar) {
                            design_file << c << "\t";
                        }
                    }
                    design_file << std::endl;
                }
            }
        } else {
            if (*verbose) std::cout << "Ignoring file: " << file_paths.at(i) << std::endl;
            // Eat a value in the covariates files as well
            for (auto &f : covars_files) {
                double c;
                f >> c;
            }
        }
    }
    if (*sort) {
        if (*verbose) std::cout << "Sorting." << std::endl;
        for (int g = 0; g < n_groups; g++) {
            for (int i = 0; i < groups.at(g).size(); i++) {
                tiler->SetInput(out_index, groups.at(g).at(i));
                out_index++;
                if (design_path.set()) {
                    for (int g2 = 0; g2 < n_groups; g2++) {
                        if (g2 == g) {
                            design_file << "1\t";
                        } else {
                            design_file << "0\t";
                        }
                    }
{'fraction_non_alphanumeric': 0.15512983611464493, 'fraction_numerical': 0.005238344683080147, 'mean_word_length': 2.076427255985267, 'pattern_counts': {'":': 0, '<': 244, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3704630', 'n_tokens_mistral': 4264, 'n_tokens_neox': 4092, 'n_words': 1065}