<commit_before>#clean
rm(list=ls())

# load lib
library(doParallel)
library(raster)

# load params
pars <- read.table("./pars/GenSA_rf_all_2_5y_rep1.txt",row.names=1)

# list landscape
clim_files <- list.files("data/futClimGrid/stmClim",full.names=TRUE)

# source res analytics function
source('./lib/resAnalytics.r')
source('./lib/convertStates.r')

# create output dir
system('mkdir -p ./data/futStatesGrid/probs')
system('mkdir -p ./data/futStatesGrid/stm')

# open cluster
cl <- makeCluster(35)
registerDoParallel(cl)

###### PROBS

# foreach(i=1:length(clim_files),.packages=c('raster','rgdal'))%dopar%{
#
#   ### READ
#   climGrid <- read.csv(clim_files[i],header=TRUE,stringsAsFactors=FALSE)
#   names(climGrid)[4:5] <- c("tp","pp")
#
#   # set NA in order to solve
#   climGrid$tp[which(climGrid$tp == -9999)] <- NA
#   climGrid$pp[which(climGrid$pp == -9999)] <- NA
#
#   #### SOLVE
#   probsGrid <- solve_stm(climGrid_scale,pars)
#   probsGrid  <- data.frame(x=climGrid$x,y=climGrid$y,probsGrid,stringsAsFactors=FALSE)
#
#   # get metadata
#   filename <- strsplit(clim_files[i],"[/.]")[[1]][4]
#
#   # write
#   write.csv(probsGrid,file=paste0("./data/futStatesGrid/probs/",filename,".csv"),row.names=FALSE)
#
# }

###### STATES

probs_files <- list.files("./data/futStatesGrid/probs",full.names=TRUE,pattern="1985-2000")

#load raster ref
ref_rs100 <- readRDS("data/futClimGrid/rasterClim/res100/rcp85-ACCESS1_0-1985-2000.rda")
ref_rs1000 <- readRDS("data/futClimGrid/rasterClim/res1000/rcp85-ACCESS1_0-1985-2000.rda")

foreach(i=1:length(probs_files),.packages=c('raster','rgdal'))%dopar%{

  probsGrid <- read.csv(probs_files[i])

  states <- character(0)

  for(r in 1:nrow(probsGrid)){
      if(any(is.na(probsGrid[r,3:5]))){
        states <- append(states,NA)
      } else {
        states <- append(states,names(which.max(probsGrid[r,3:5])))
      }
  }

  stmGrid <- data.frame(coordinates(ref_rs1000),state=stateToId(states),stringsAsFactors=FALSE)

  # turn into raster using clim rs
  rs_stmGrid <- stmGrid
  coordinates(rs_stmGrid) <- ~ x + y
  gridded(rs_stmGrid) <- TRUE
  rs_stmGrid <-  raster(rs_stmGrid)
  projection(rs_stmGrid) <- projection(ref_rs1000)

  # reample using ref_rs100
  rs_stmGrid_1000 <- resample(rs_stmGrid,ref_rs100,method="ngb")
  df_stmGrid_1000 <- as.data.frame(rs_stmGrid_1000,xy=TRUE)

  # get metadata
  filename <- strsplit(probs_files[i],"[/.]")[[1]][6]
  filename <- paste(strsplit(filename,"[-]")[[1]][2:3],"-")<commit_msg>fix issue with filename in sript3<commit_after>#clean
rm(list=ls())

# load lib
library(doParallel)
library(raster)

# load params
pars <- read.table("./pars/GenSA_rf_all_2_5y_rep1.txt",row.names=1)

# list landscape
clim_files <- list.files("data/futClimGrid/stmClim",full.names=TRUE)

# source res analytics function
source('./lib/resAnalytics.r')
source('./lib/convertStates.r')

# create output dir
system('mkdir -p ./data/futStatesGrid/probs')
system('mkdir -p ./data/futStatesGrid/stm')

# open cluster
cl <- makeCluster(35)
registerDoParallel(cl)

###### PROBS

# foreach(i=1:length(clim_files),.packages=c('raster','rgdal'))%dopar%{
#
#   ### READ
#   climGrid <- read.csv(clim_files[i],header=TRUE,stringsAsFactors=FALSE)
#   names(climGrid)[4:5] <- c("tp","pp")
#
#   # set NA in order to solve
#   climGrid$tp[which(climGrid$tp == -9999)] <- NA
#   climGrid$pp[which(climGrid$pp == -9999)] <- NA
#
#   #### SOLVE
#   probsGrid <- solve_stm(climGrid_scale,pars)
#   probsGrid  <- data.frame(x=climGrid$x,y=climGrid$y,probsGrid,stringsAsFactors=FALSE)
#
#   # get metadata
#   filename <- strsplit(clim_files[i],"[/.]")[[1]][4]
#
#   # write
#   write.csv(probsGrid,file=paste0("./data/futStatesGrid/probs/",filename,".csv"),row.names=FALSE)
#
# }

###### STATES

probs_files <- list.files("./data/futStatesGrid/probs",full.names=TRUE,pattern="1985-2000")

#load raster ref
ref_rs100 <- readRDS("data/futClimGrid/rasterClim/res100/rcp85-ACCESS1_0-1985-2000.rda")
ref_rs1000 <- readRDS("data/futClimGrid/rasterClim/res1000/rcp85-ACCESS1_0-1985-2000.rda")

foreach(i=1:length(probs_files),.packages=c('raster','rgdal'))%dopar%{

  probsGrid <- read.csv(probs_files[i])

  states <- character(0)

  for(r in 1:nrow(probsGrid)){
      if(any(is.na(probsGrid[r,3:5]))){
        states <- append(states,NA)
      } else {
        states <- append(states,names(which.max(probsGrid[r,3:5])))
      }
  }

  stmGrid <- data.frame(coordinates(ref_rs1000),state=stateToId(states),stringsAsFactors=FALSE)

  # turn into raster using clim rs
  rs_stmGrid <- stmGrid
  coordinates(rs_stmGrid) <- ~ x + y
  gridded(rs_stmGrid) <- TRUE
  rs_stmGrid <-  raster(rs_stmGrid)
  projection(rs_stmGrid) <- projection(ref_rs1000)

  # reample using ref_rs100
  rs_stmGrid_1000 <- resample(rs_stmGrid,ref_rs100,method="ngb")
  df_stmGrid_1000 <- as.data.frame(rs_stmGrid_1000,xy=TRUE)

  # get metadata
  filename <- strsplit(probs_files[i],"[/.]")[[1]][6]
  filename <- paste(strsplit(filename,"[-]")[[1]][c(1,2,4)],collapse="-")
{'fraction_non_alphanumeric': 0.188951160928743, 'fraction_numerical': 0.042033626901521216, 'mean_word_length': 5.035024154589372, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2924962', 'n_tokens_mistral': 2213, 'n_tokens_neox': 2025, 'n_words': 244}