<commit_before>#' Create an aggregate MSS TC model
#'
#' Create an aggregate MSS TC model with diagnostic figures
#' @param dirname character. the directory to the calibration folder
#' @import ggplot2
#' @export


cal_mss_tc_aggregate_model = function(dir){
  plot_multi_cal = function(df,index, model, outfile){
    
    if(index == "tcb"){limits = c(-500, 10000)}
    if(index == "tcg"){limits = c(-500, 5000)}
    if(index == "tcw"){limits = c(-6000, 1000)}
    if(index == "tca"){limits = c(-500, 5000)}
    if(model == "single"){
      
      
      tmp = with(df,by(df, mss_img, function(x) rlm(refsamp ~ singlepred, data = x)))
      tmp = sapply(tmp, coef)
      
      title = paste("single model scatterplot and robust linear regression lines for",index)
      png(outfile, width = 1100, height=800)
      cols = seq(from=2,to=ncol(tmp)+1)
      plot(x=df$singlepred,y=df$refsamp,
           main=title,
           xlab=paste("mss-predicted",index),
           ylab=paste("tm",index),
           xlim=limits,
           ylim=limits)
      for(i in 1:ncol(tmp)){abline(a = tmp[1,i], b=tmp[2,i], col=cols[i])}
      legend(x=limits[1]+25,y=limits[2]-25,
             colnames(tmp),
             lty=rep(1,ncol(tmp)),
             col=cols)
      
      
      dev.off()
      
      
      #       g = ggplot(df, aes(x=singlepred, y=refsamp, colour=mss_img)) +
      #         geom_density2d(bins=3)+
      #         geom_smooth(method="rlm", se = FALSE)+
      #         xlim(limits)+
      #         ylim(limits)+
      #         xlab(paste("mss predicted",index))+
      #         ylab(paste("tm",index))+
      #         ggtitle(paste("single model 2d data density contours and robust linear regression lines for",index))+
      #         coord_fixed(ratio = 1)+
      #         theme_bw()
    }
    if(model == "aggregate"){
      
      tmp = with(df,by(df, mss_img, function(x) rlm(refsamp ~ comppred, data = x)))
      tmp = sapply(tmp, coef)
      
      title = paste("aggregated model scatterplot and robust linear regression lines for",index)
      cols = seq(from=2,to=ncol(tmp)+1)
      
      png(outfile, width = 1100, height=800)
      plot(x=df$comppred,y=df$refsamp,
           main=title,
           xlab=paste("mss-predicted",index),
           ylab=paste("tm",index),
           xlim=limits,
           ylim=limits)
      
      m = rlm(refsamp ~ comppred, data=df)
      abline(coef = m$coefficients, lty=2, col="gray48", ,lwd=2.5)
      for(i in 1:ncol(tmp)){abline(a = tmp[1,i], b=tmp[2,i], col=cols[i])}
      legend(x=limits[1]+25,y=limits[2]-25,
             c("mean",colnames(tmp)),
             lty=c(2,rep(1,ncol(tmp))),
             col=c("gray48",cols))
      
      
      #print(g)
      dev.off()
      
      #       g=ggplot()+
      #         geom_density2d(bins=3,data=df, aes(x=comppred, y=refsamp, colour=mss_img))+
      #         geom_smooth(method="rlm", data=df, aes(x=comppred, y=refsamp, colour=mss_img), se = FALSE)+
      #         geom_smooth(method="rlm", data=df, aes(x=comppred, y=refsamp), colour="black", size=1, se = FALSE, linetype="dashed")+
      #         xlim(limits)+
      #         ylim(limits)+
      #         xlab(paste("mss predicted",index))+
      #         ylab(paste("tm",index))+
      #         ggtitle(paste("aggregate model 2d data density contours and robust linear regression lines for",index))+
      #         coord_fixed(ratio = 1)+
      #         theme_bw()
    }
    
    #     png(outfile, width = 1100, height=800)
    #     print(g)
    #     dev.off()
    #return(g)
  }
  
  plot_multi_cal_dif = function(df, index, model, outfile){
    if(index == "tcb"){limits = c(-500, 10000)}
    if(index == "tcg"){limits = c(-500, 5000)}
    if(index == "tcw"){limits = c(-6000, 1000)}
    if(index == "tca"){limits = c(-500, 5000)}
    meanrefsamp = mean(df$refsamp)
    if(model == "single"){
      newdf = aggregate(singlepreddif ~ mss_img , data = df, mean)
      newdf$diffm = meanrefsamp + newdf$singlepreddif
      title = paste("single model mean prediction differences from actual values for",index)
    }
    if(model == "aggregate"){
      newdf = aggregate(comppreddif ~ mss_img , data = df, mean)
      newdf$diffm = meanrefsamp + newdf$comppreddif
      title = paste("aggregate model mean prediction differences from actual values for",index)
    }
    
    #     g = ggplot(df, aes(x=refsamp))+
    #       geom_density(colour="black",linetype="dashed")+
    #       geom_vline(data = newdf,aes(xintercept = diffm, colour = mss_img), show_guide = TRUE, size=0.5) +
    #       geom_vline(xintercept = meanrefsamp , colour = "black", size=1, linetype="dashed") +
    #       xlim(limits)+
    #       xlab(paste(index))+
    #       ggtitle(title)+
    #       theme_bw()
    
    png(outfile, width = 1100, height=800)
    d = density(df$refsamp)
    plot(d,
         xlim=limits,
         main=title,
         xlab=paste(index))
    abline(v=meanrefsamp,lty=2, col=1, ,lwd=2.5)
    abline(v=newdf$diffm, col=c(2:(length(newdf$diffm)+1)))
    legend(x=limits[2]-3000,y=max(d$y),
           c("mean",as.character(newdf$mss_img)),
           lty=c(2,rep(1,length(newdf$diffm))),
           col=c(1,2:(length(newdf$diffm)+1)))
    
    #print(g)
    dev.off()
    #return(g)
  }<commit_msg>fixed extra comma syntax error<commit_after>#' Create an aggregate MSS TC model
#'
#' Create an aggregate MSS TC model with diagnostic figures
#' @param dirname character. the directory to the calibration folder
#' @import ggplot2
#' @export


cal_mss_tc_aggregate_model = function(dir){
  plot_multi_cal = function(df,index, model, outfile){
    
    if(index == "tcb"){limits = c(-500, 10000)}
    if(index == "tcg"){limits = c(-500, 5000)}
    if(index == "tcw"){limits = c(-6000, 1000)}
    if(index == "tca"){limits = c(-500, 5000)}
    if(model == "single"){
      
      
      tmp = with(df,by(df, mss_img, function(x) rlm(refsamp ~ singlepred, data = x)))
      tmp = sapply(tmp, coef)
      
      title = paste("single model scatterplot and robust linear regression lines for",index)
      png(outfile, width = 1100, height=800)
      cols = seq(from=2,to=ncol(tmp)+1)
      plot(x=df$singlepred,y=df$refsamp,
           main=title,
           xlab=paste("mss-predicted",index),
           ylab=paste("tm",index),
           xlim=limits,
           ylim=limits)
      for(i in 1:ncol(tmp)){abline(a = tmp[1,i], b=tmp[2,i], col=cols[i])}
      legend(x=limits[1]+25,y=limits[2]-25,
             colnames(tmp),
             lty=rep(1,ncol(tmp)),
             col=cols)
      
      
      dev.off()
      
      
      #       g = ggplot(df, aes(x=singlepred, y=refsamp, colour=mss_img)) +
      #         geom_density2d(bins=3)+
      #         geom_smooth(method="rlm", se = FALSE)+
      #         xlim(limits)+
      #         ylim(limits)+
      #         xlab(paste("mss predicted",index))+
      #         ylab(paste("tm",index))+
      #         ggtitle(paste("single model 2d data density contours and robust linear regression lines for",index))+
      #         coord_fixed(ratio = 1)+
      #         theme_bw()
    }
    if(model == "aggregate"){
      
      tmp = with(df,by(df, mss_img, function(x) rlm(refsamp ~ comppred, data = x)))
      tmp = sapply(tmp, coef)
      
      title = paste("aggregated model scatterplot and robust linear regression lines for",index)
      cols = seq(from=2,to=ncol(tmp)+1)
      
      png(outfile, width = 1100, height=800)
      plot(x=df$comppred,y=df$refsamp,
           main=title,
           xlab=paste("mss-predicted",index),
           ylab=paste("tm",index),
           xlim=limits,
           ylim=limits)
      
      m = rlm(refsamp ~ comppred, data=df)
      abline(coef = m$coefficients, lty=2, col="gray48", lwd=2.5)
      for(i in 1:ncol(tmp)){abline(a = tmp[1,i], b=tmp[2,i], col=cols[i])}
      legend(x=limits[1]+25,y=limits[2]-25,
             c("mean",colnames(tmp)),
             lty=c(2,rep(1,ncol(tmp))),
             col=c("gray48",cols))
      
      
      #print(g)
      dev.off()
      
      #       g=ggplot()+
      #         geom_density2d(bins=3,data=df, aes(x=comppred, y=refsamp, colour=mss_img))+
      #         geom_smooth(method="rlm", data=df, aes(x=comppred, y=refsamp, colour=mss_img), se = FALSE)+
      #         geom_smooth(method="rlm", data=df, aes(x=comppred, y=refsamp), colour="black", size=1, se = FALSE, linetype="dashed")+
      #         xlim(limits)+
      #         ylim(limits)+
      #         xlab(paste("mss predicted",index))+
      #         ylab(paste("tm",index))+
      #         ggtitle(paste("aggregate model 2d data density contours and robust linear regression lines for",index))+
      #         coord_fixed(ratio = 1)+
      #         theme_bw()
    }
    
    #     png(outfile, width = 1100, height=800)
    #     print(g)
    #     dev.off()
    #return(g)
  }
  
  plot_multi_cal_dif = function(df, index, model, outfile){
    if(index == "tcb"){limits = c(-500, 10000)}
    if(index == "tcg"){limits = c(-500, 5000)}
    if(index == "tcw"){limits = c(-6000, 1000)}
    if(index == "tca"){limits = c(-500, 5000)}
    meanrefsamp = mean(df$refsamp)
    if(model == "single"){
      newdf = aggregate(singlepreddif ~ mss_img , data = df, mean)
      newdf$diffm = meanrefsamp + newdf$singlepreddif
      title = paste("single model mean prediction differences from actual values for",index)
    }
    if(model == "aggregate"){
      newdf = aggregate(comppreddif ~ mss_img , data = df, mean)
      newdf$diffm = meanrefsamp + newdf$comppreddif
      title = paste("aggregate model mean prediction differences from actual values for",index)
    }
    
    #     g = ggplot(df, aes(x=refsamp))+
    #       geom_density(colour="black",linetype="dashed")+
    #       geom_vline(data = newdf,aes(xintercept = diffm, colour = mss_img), show_guide = TRUE, size=0.5) +
    #       geom_vline(xintercept = meanrefsamp , colour = "black", size=1, linetype="dashed") +
    #       xlim(limits)+
    #       xlab(paste(index))+
    #       ggtitle(title)+
    #       theme_bw()
    
    png(outfile, width = 1100, height=800)
    d = density(df$refsamp)
    plot(d,
         xlim=limits,
         main=title,
         xlab=paste(index))
    abline(v=meanrefsamp,lty=2, col=1,lwd=2.5)
    abline(v=newdf$diffm, col=c(2:(length(newdf$diffm)+1)))
    legend(x=limits[2]-3000,y=max(d$y),
           c("mean",as.character(newdf$mss_img)),
           lty=c(2,rep(1,length(newdf$diffm))),
           col=c(1,2:(length(newdf$diffm)+1)))
    
    #print(g)
    dev.off()
    #return(g)
  }
{'fraction_non_alphanumeric': 0.1590866201170032, 'fraction_numerical': 0.028118512926967352, 'mean_word_length': 2.470530451866405, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6249436', 'n_tokens_mistral': 4128, 'n_tokens_neox': 3763, 'n_words': 679}