<commit_before>#include "ghost/config.h"
#include "ghost/types.h"
#include "ghost/sell.h"
#include "ghost/log.h"
#include "ghost/cu_util.h"
#include "ghost/cu_complex.h"
#include "ghost/util.h"

#include <complex.h>
#include <cuda_runtime.h>
#include <cusparse_v2.h>

typedef cusparseStatus_t (*cusparse_sell1_spmv_kernel_t) (cusparseHandle_t handle, cusparseOperation_t transA, 
        int m, int n, int nnz, const void           *alpha, 
        const cusparseMatDescr_t descrA, 
        const void           *csrValA, 
        const void *csrRowPtrA, const void *csrColIndA,
        const void           *x, const void           *beta, 
        void           *y);

typedef cusparseStatus_t (*cusparse_sell1_spmmv_cm_kernel_t) (cusparseHandle_t handle, cusparseOperation_t transA, 
        int m, int n, int k, int nnz, const void           *alpha, 
        const cusparseMatDescr_t descrA, 
        const void           *csrValA, 
        const void *csrRowPtrA, const void *csrColIndA,
        const void           *x, int ldx, const void           *beta, 
        void           *y, int ldy);

typedef cusparseStatus_t (*cusparse_sell1_spmmv_rm_kernel_t) (cusparseHandle_t handle, cusparseOperation_t transA,
        cusparseOperation_t transB,
        int m, int n, int k, int nnz, const void           *alpha, 
        const cusparseMatDescr_t descrA, 
        const void           *csrValA, 
        const void *csrRowPtrA, const void *csrColIndA,
        const void           *x, int ldx, const void           *beta, 
        void           *y, int ldy);

    template<typename dt1, typename dt2>
static ghost_error_t ghost_cu_sell1spmv_tmpl(ghost_sparsemat_t *mat, ghost_densemat_t * lhs, ghost_densemat_t * rhs, ghost_spmv_flags_t options, va_list argp, cusparse_sell1_spmv_kernel_t sell1kernel)
{
    GHOST_FUNC_ENTER(GHOST_FUNCTYPE_MATH);
    cusparseHandle_t cusparse_handle;
    cusparseMatDescr_t descr;

    cusparseCreateMatDescr(&descr);
    GHOST_CALL_RETURN(ghost_cu_cusparse_handle(&cusparse_handle));

    dt2 * __attribute__((unused)) localdot = NULL;
    dt1 * __attribute__((unused)) shift = NULL, scale, beta, __attribute__((unused)) sdelta, __attribute__((unused)) seta;
    ghost_densemat_t * __attribute__((unused)) z = NULL;

    one<dt1>(scale);

    GHOST_SPMV_PARSE_ARGS(options,argp,scale,beta,shift,localdot,z,sdelta,seta,dt2,dt1);

    if (options & GHOST_SPMV_AXPY) {
        one<dt1>(beta);
    } else if (!(options & GHOST_SPMV_AXPBY)) {
        zero<dt1>(beta);
    }
     
    CUSPARSE_CALL_RETURN(sell1kernel(cusparse_handle,CUSPARSE_OPERATION_NON_TRANSPOSE,mat->nrows,rhs->traits.nrowshalo,mat->nnz,&scale,descr,(dt1 *)SELL(mat)->cumat->val, SELL(mat)->cumat->chunkStart, SELL(mat)->cumat->col, (dt1 *)rhs->cu_val, &beta, (dt1 *)lhs->cu_val));
   
    GHOST_FUNC_EXIT(GHOST_FUNCTYPE_MATH);
    return GHOST_SUCCESS;
}
    
    template<typename dt1, typename dt2>
static ghost_error_t ghost_cu_sell1spmmv_cm_tmpl(ghost_sparsemat_t *mat, ghost_densemat_t * lhs, ghost_densemat_t * rhs, ghost_spmv_flags_t options, va_list argp, cusparse_sell1_spmmv_cm_kernel_t sell1kernel)
{
    GHOST_FUNC_ENTER(GHOST_FUNCTYPE_MATH);
    cusparseHandle_t cusparse_handle;
    cusparseMatDescr_t descr;

    cusparseCreateMatDescr(&descr);
    GHOST_CALL_RETURN(ghost_cu_cusparse_handle(&cusparse_handle));

    dt2 * __attribute__((unused)) localdot = NULL;
    dt1 * __attribute__((unused)) shift = NULL, scale, beta, __attribute__((unused)) sdelta, __attribute__((unused)) seta;
    ghost_densemat_t * __attribute__((unused)) z = NULL;

    one<dt1>(scale);

    GHOST_SPMV_PARSE_ARGS(options,argp,scale,beta,shift,localdot,z,sdelta,seta,dt2,dt1);
    
    if (options & GHOST_SPMV_AXPY) {
        one<dt1>(beta);
    } else if (!(options & GHOST_SPMV_AXPBY)) {
        zero<dt1>(beta);
    }
    CUSPARSE_CALL_RETURN(sell1kernel(cusparse_handle,CUSPARSE_OPERATION_NON_TRANSPOSE,mat->nrows,rhs->traits.ncols,rhs->traits.nrowshalo,mat->nnz,&scale,descr,(dt1 *)SELL(mat)->cumat->val, SELL(mat)->cumat->chunkStart, SELL(mat)->cumat->col, (dt1 *)rhs->cu_val, rhs->stride, &beta, (dt1 *)lhs->cu_val, lhs->stride));

    GHOST_FUNC_EXIT(GHOST_FUNCTYPE_MATH);
    return GHOST_SUCCESS;
}

    template<typename dt1, typename dt2>
static ghost_error_t ghost_cu_sell1spmmv_rm_tmpl(ghost_sparsemat_t *mat, ghost_densemat_t * lhs, ghost_densemat_t * rhs, ghost_spmv_flags_t options, va_list argp, cusparse_sell1_spmmv_rm_kernel_t sell1kernel)
{
    GHOST_FUNC_ENTER(GHOST_FUNCTYPE_MATH);
    cusparseHandle_t cusparse_handle;
    cusparseMatDescr_t descr;

    cusparseCreateMatDescr(&descr);
    GHOST_CALL_RETURN(ghost_cu_cusparse_handle(&cusparse_handle));

    dt2 * __attribute__((unused)) localdot = NULL;
    dt1 * __attribute__((unused)) shift = NULL, scale, beta, __attribute__((unused)) sdelta, __attribute__((unused)) seta;
    ghost_densemat_t * __attribute__((unused)) z = NULL;

    one<dt1>(scale);

    GHOST_SPMV_PARSE_ARGS(options,argp,scale,beta,shift,localdot,z,sdelta,seta,dt2,dt1);

    if (options & GHOST_SPMV_AXPY) {
        one<dt1>(beta);
    } else if (!(options & GHOST_SPMV_AXPBY)) {
        zero<dt1>(beta);
    }
    
    CUSPARSE_CALL_RETURN(sell1kernel(cusparse_handle,CUSPARSE_OPERATION_NON_TRANSPOSE,CUSPARSE_OPERATION_TRANSPOSE,mat->nrows,rhs->traits.ncols,rhs->traits.nrowshalo,mat->nnz,&scale,descr,(dt1 *)SELL(mat)->cumat->val, SELL(mat)->cumat->chunkStart, SELL(mat)->cumat->col, (dt1 *)rhs->cu_val, rhs->stride, &beta, (dt1 *)lhs->cu_val,lhs->stride));
    
    
    GHOST_FUNC_EXIT(GHOST_FUNCTYPE_MATH);
    return GHOST_SUCCESS;
}
    
    template<typename dt1, typename dt2>
static ghost_error_t ghost_cu_sell1spmv_augfunc_tmpl(ghost_densemat_t * lhs, ghost_densemat_t * rhs, ghost_spmv_flags_t options, va_list argp)
{
    dt2 *localdot = NULL;
    dt1 *shift = NULL, __attribute__((unused)) scale, __attribute__((unused)) beta, sdelta, seta;
    ghost_densemat_t *z = NULL;

    GHOST_SPMV_PARSE_ARGS(options,argp,scale,beta,shift,localdot,z,sdelta,seta,dt2,dt1);
    if (options & (GHOST_SPMV_SHIFT|GHOST_SPMV_VSHIFT)) {
        PERFWARNING_LOG("Shift will not be applied on-the-fly!");
        dt2 minusshift[rhs->traits.ncols];
        ghost_lidx_t col;
        if (options & GHOST_SPMV_SHIFT) {
            for (col=0; col<rhs->traits.ncols; col++) {
                minusshift[col] = -1.*(*(dt2 *)&scale)*(*(dt2 *)shift);
            }
        } else {
            for (col=0; col<rhs->traits.ncols; col++) {
                minusshift[col] = -1.*(*(dt2 *)&scale)*(((dt2 *)shift)[col]);<commit_msg>bugfix with augmented SELL-1 CUDA SpMV<commit_after>#include "ghost/config.h"
#include "ghost/types.h"
#include "ghost/sell.h"
#include "ghost/log.h"
#include "ghost/cu_util.h"
#include "ghost/cu_complex.h"
#include "ghost/util.h"

#include <complex.h>
#include <cuda_runtime.h>
#include <cusparse_v2.h>

typedef cusparseStatus_t (*cusparse_sell1_spmv_kernel_t) (cusparseHandle_t handle, cusparseOperation_t transA, 
        int m, int n, int nnz, const void           *alpha, 
        const cusparseMatDescr_t descrA, 
        const void           *csrValA, 
        const void *csrRowPtrA, const void *csrColIndA,
        const void           *x, const void           *beta, 
        void           *y);

typedef cusparseStatus_t (*cusparse_sell1_spmmv_cm_kernel_t) (cusparseHandle_t handle, cusparseOperation_t transA, 
        int m, int n, int k, int nnz, const void           *alpha, 
        const cusparseMatDescr_t descrA, 
        const void           *csrValA, 
        const void *csrRowPtrA, const void *csrColIndA,
        const void           *x, int ldx, const void           *beta, 
        void           *y, int ldy);

typedef cusparseStatus_t (*cusparse_sell1_spmmv_rm_kernel_t) (cusparseHandle_t handle, cusparseOperation_t transA,
        cusparseOperation_t transB,
        int m, int n, int k, int nnz, const void           *alpha, 
        const cusparseMatDescr_t descrA, 
        const void           *csrValA, 
        const void *csrRowPtrA, const void *csrColIndA,
        const void           *x, int ldx, const void           *beta, 
        void           *y, int ldy);

    template<typename dt1, typename dt2>
static ghost_error_t ghost_cu_sell1spmv_tmpl(ghost_sparsemat_t *mat, ghost_densemat_t * lhs, ghost_densemat_t * rhs, ghost_spmv_flags_t options, va_list argp, cusparse_sell1_spmv_kernel_t sell1kernel)
{
    GHOST_FUNC_ENTER(GHOST_FUNCTYPE_MATH);
    cusparseHandle_t cusparse_handle;
    cusparseMatDescr_t descr;

    cusparseCreateMatDescr(&descr);
    GHOST_CALL_RETURN(ghost_cu_cusparse_handle(&cusparse_handle));

    dt2 * __attribute__((unused)) localdot = NULL;
    dt1 * __attribute__((unused)) shift = NULL, scale, beta, __attribute__((unused)) sdelta, __attribute__((unused)) seta;
    ghost_densemat_t * __attribute__((unused)) z = NULL;

    one<dt1>(scale);

    GHOST_SPMV_PARSE_ARGS(options,argp,scale,beta,shift,localdot,z,sdelta,seta,dt2,dt1);

    if (options & GHOST_SPMV_AXPY) {
        one<dt1>(beta);
    } else if (!(options & GHOST_SPMV_AXPBY)) {
        zero<dt1>(beta);
    }
     
    CUSPARSE_CALL_RETURN(sell1kernel(cusparse_handle,CUSPARSE_OPERATION_NON_TRANSPOSE,mat->nrows,rhs->traits.nrowshalo,mat->nnz,&scale,descr,(dt1 *)SELL(mat)->cumat->val, SELL(mat)->cumat->chunkStart, SELL(mat)->cumat->col, (dt1 *)rhs->cu_val, &beta, (dt1 *)lhs->cu_val));
   
    GHOST_FUNC_EXIT(GHOST_FUNCTYPE_MATH);
    return GHOST_SUCCESS;
}
    
    template<typename dt1, typename dt2>
static ghost_error_t ghost_cu_sell1spmmv_cm_tmpl(ghost_sparsemat_t *mat, ghost_densemat_t * lhs, ghost_densemat_t * rhs, ghost_spmv_flags_t options, va_list argp, cusparse_sell1_spmmv_cm_kernel_t sell1kernel)
{
    GHOST_FUNC_ENTER(GHOST_FUNCTYPE_MATH);
    cusparseHandle_t cusparse_handle;
    cusparseMatDescr_t descr;

    cusparseCreateMatDescr(&descr);
    GHOST_CALL_RETURN(ghost_cu_cusparse_handle(&cusparse_handle));

    dt2 * __attribute__((unused)) localdot = NULL;
    dt1 * __attribute__((unused)) shift = NULL, scale, beta, __attribute__((unused)) sdelta, __attribute__((unused)) seta;
    ghost_densemat_t * __attribute__((unused)) z = NULL;

    one<dt1>(scale);

    GHOST_SPMV_PARSE_ARGS(options,argp,scale,beta,shift,localdot,z,sdelta,seta,dt2,dt1);
    
    if (options & GHOST_SPMV_AXPY) {
        one<dt1>(beta);
    } else if (!(options & GHOST_SPMV_AXPBY)) {
        zero<dt1>(beta);
    }
    CUSPARSE_CALL_RETURN(sell1kernel(cusparse_handle,CUSPARSE_OPERATION_NON_TRANSPOSE,mat->nrows,rhs->traits.ncols,rhs->traits.nrowshalo,mat->nnz,&scale,descr,(dt1 *)SELL(mat)->cumat->val, SELL(mat)->cumat->chunkStart, SELL(mat)->cumat->col, (dt1 *)rhs->cu_val, rhs->stride, &beta, (dt1 *)lhs->cu_val, lhs->stride));

    GHOST_FUNC_EXIT(GHOST_FUNCTYPE_MATH);
    return GHOST_SUCCESS;
}

    template<typename dt1, typename dt2>
static ghost_error_t ghost_cu_sell1spmmv_rm_tmpl(ghost_sparsemat_t *mat, ghost_densemat_t * lhs, ghost_densemat_t * rhs, ghost_spmv_flags_t options, va_list argp, cusparse_sell1_spmmv_rm_kernel_t sell1kernel)
{
    GHOST_FUNC_ENTER(GHOST_FUNCTYPE_MATH);
    cusparseHandle_t cusparse_handle;
    cusparseMatDescr_t descr;

    cusparseCreateMatDescr(&descr);
    GHOST_CALL_RETURN(ghost_cu_cusparse_handle(&cusparse_handle));

    dt2 * __attribute__((unused)) localdot = NULL;
    dt1 * __attribute__((unused)) shift = NULL, scale, beta, __attribute__((unused)) sdelta, __attribute__((unused)) seta;
    ghost_densemat_t * __attribute__((unused)) z = NULL;

    one<dt1>(scale);

    GHOST_SPMV_PARSE_ARGS(options,argp,scale,beta,shift,localdot,z,sdelta,seta,dt2,dt1);

    if (options & GHOST_SPMV_AXPY) {
        one<dt1>(beta);
    } else if (!(options & GHOST_SPMV_AXPBY)) {
        zero<dt1>(beta);
    }
    
    CUSPARSE_CALL_RETURN(sell1kernel(cusparse_handle,CUSPARSE_OPERATION_NON_TRANSPOSE,CUSPARSE_OPERATION_TRANSPOSE,mat->nrows,rhs->traits.ncols,rhs->traits.nrowshalo,mat->nnz,&scale,descr,(dt1 *)SELL(mat)->cumat->val, SELL(mat)->cumat->chunkStart, SELL(mat)->cumat->col, (dt1 *)rhs->cu_val, rhs->stride, &beta, (dt1 *)lhs->cu_val,lhs->stride));
    
    
    GHOST_FUNC_EXIT(GHOST_FUNCTYPE_MATH);
    return GHOST_SUCCESS;
}
    
    template<typename dt1, typename dt2>
static ghost_error_t ghost_cu_sell1spmv_augfunc_tmpl(ghost_densemat_t * lhs, ghost_densemat_t * rhs, ghost_spmv_flags_t options, va_list argp)
{
    dt2 *localdot = NULL;
    dt1 *shift = NULL, scale, __attribute__((unused)) beta, sdelta, seta;
    ghost_densemat_t *z = NULL;

    one<dt1>(scale); //required because we need scale for SHIFT (i.e., even if we don't SCALE)
    GHOST_SPMV_PARSE_ARGS(options,argp,scale,beta,shift,localdot,z,sdelta,seta,dt2,dt1);

    if (options & (GHOST_SPMV_SHIFT|GHOST_SPMV_VSHIFT)) {
        PERFWARNING_LOG("Shift will not be applied on-the-fly!");
        dt2 minusshift[rhs->traits.ncols];
        ghost_lidx_t col;
        if (options & GHOST_SPMV_SHIFT) {
            for (col=0; col<rhs->traits.ncols; col++) {
                minusshift[col] = -1.*(*(dt2 *)&scale)*(*(dt2 *)shift);
            }
        } else {
            for (col=0; col<rhs->traits.ncols; col++) {
                minusshift[col] = -1.*(*(dt2 *)&scale)*(((dt2 *)shift)[col]);
{'fraction_non_alphanumeric': 0.12023261082999774, 'fraction_numerical': 0.010422173551846537, 'mean_word_length': 4.017809776430466, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 120, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6050194', 'n_tokens_mistral': 5204, 'n_tokens_neox': 5001, 'n_words': 829}