<commit_before>#UserSurvival - a project to accurately triage data on blocked Wikipedia users, identify
#the underlying rationales and test various hypotheses as to any outcome
#
#ipblock.r outputs the basic analysis documented at http://blog.ironholds.org/?p=31
#
# @Year = 2013
# @Copyright: <NAME>
# @License = MIT (http://opensource.org/licenses/MIT)

#Enclosing function
ipblock.fun <- function(){
  
  #Grab dataset
  query.df <- sql.fun(query_statement = "
          SELECT ipb_reason AS reason,
            substring(ipb_timestamp,1,6) AS block_timestamp,
            ipb_user
          FROM ipblocks
          WHERE ipb_timestamp BETWEEN 20060101010101 AND 20121231235959
          AND ipb_expiry = 'infinity';"
  )
  
  #graphing function
  graphing.fun <- function(x, usergroup){
    
    #Split up the list
    monthly_data.df <- as.data.frame(x[1])
    yearly_data.df <- as.data.frame(x[2])
    
    #Line-and-point graphs for yearly and monthly data.
    line_graph_yearly <- ggplot(yearly_data.df, aes(block_timestamp, value)) + 
      geom_freqpoly(aes(group = variable, colour = variable), stat = "identity") +
      labs(x = "Year", y = "Number of users") +
      ggtitle(paste("Block rationales on the English-language Wikipedia\nby year (2006-2012) - ipblocks table,",usergroup,"users",sep = " ")) +
      scale_x_discrete(breaks = 2006:2012, expand = c(0,0)) +
      scale_y_continuous(expand = c(0, 0)) +
      theme(axis.text.x = element_text(angle = 90, hjust = 1))

    line_graph_monthly <- ggplot(monthly_data.df, aes(block_timestamp, value)) +
      geom_freqpoly(aes(group = variable, colour = variable), stat = "identity") +
      labs(x = "Year", y = "Number of users") +
      ggtitle(paste("Block rationales on the English-language Wikipedia\nby month (2006-2012) - ipblocks table,",usergroup,"users",sep = " ")) +
      scale_x_discrete(expand = c(0,0)) +
      scale_y_continuous(expand = c(0,0)) +
      theme(axis.text.x = element_blank(), axis.ticks.x = element_blank())
    
    #print
    ggsave(filename = file.path(getwd(),"Output", paste(usergroup,"_ipblocks_regex_matches_by_year.png", sep = "")),
      plot = line_graph_yearly,
      width = 8,
      height = 8,
      units = "in")
    ggsave(filename = file.path(getwd(),"Output", paste(usergroup,"_ipblocks_regex_matches_by_month.png", sep = "")),
      plot = line_graph_monthly,
      width = 8,
      height = 8,
      units = "in")
    
    #Regression
    regression_graph_monthly <- ggplot(monthly_data.df,aes(x = block_timestamp,y = value, colour = variable))+
      geom_point(shape=3) +
      geom_smooth(method = lm, se = TRUE, aes(group= variable)) +
      labs(x = "Year", y = "Number of users") +
      ggtitle(paste("Block rationales on the English-language Wikipedia\nby month (2006-2012) - ipblocks table,",usergroup,"users",sep = " ")) +
      scale_x_discrete(expand = c(0,0)) +
      scale_y_continuous(expand = c(0,0)) +
      theme(axis.text.x = element_blank(), axis.ticks.x = element_blank())
    
    #Print
    ggsave(filename = file.path(getwd(),"Output", paste(usergroup,"_ipblocks_regex_matches_linear_regression.png", sep = "")),
      plot = regression_graph_monthly,
      width = 8,
      height = 8,
      units = "in")
    
    #The dataset makes it difficult to properly show everything. Let's graph some subsets
    regression_graph_monthly_badfaith <- ggplot(monthly_data.df[monthly_data.df$variable %in% c("bad.faith","spam"),],
      aes(x = block_timestamp,y = value, colour = variable))+
      geom_point(shape=3) +
      geom_smooth(method = lm, se = TRUE, aes(group= variable)) +
      labs(x = "Year", y = "Number of users") +
      ggtitle(paste("Spam and bad-faith blocks on the English-language Wikipedia\nby month (2006-2012) - ipblocks table,",usergroup,"users",sep = " ")) +
      scale_x_discrete(expand = c(0,0)) +
      scale_y_continuous(expand = c(0,0)) +
      theme(axis.text.x = element_blank(), axis.ticks.x = element_blank())
      
    regression_graph_monthly_other <- ggplot(monthly_data.df[!monthly_data.df$variable %in% c("bad.faith","spam"),],
      aes(x = block_timestamp,y = value, colour = variable))+
      geom_point(shape=3) +
      geom_smooth(method = lm, se = TRUE, aes(group= variable)) +
      labs(x = "Year", y = "Number of users") +
      ggtitle(paste("Other blocks on the English-language Wikipedia\nby month (2006-2012) - ipblocks table,"usergroup,"users",sep = " ")) +
      scale_x_discrete(expand = c(0,0)) +
      scale_y_continuous(expand = c(0,0)) +
      theme(axis.text.x = element_blank(), axis.ticks.x = element_blank())
    
    ggsave(filename = file.path(getwd(),"Output", paste(usergroup,"_ipblocks_linear_regression_badfaith.png",sep = "")),
      plot = regression_graph_monthly_badfaith,
      width = 8,
      height = 8,
      units = "in")
    ggsave(filename = file.path(getwd(),"Output", paste(usergroup,"_ipblocks_linear_regression_other.png",sep = "")),
      plot = regression_graph_monthly_other,
      width = 8,
      height = 8,
      units = "in")
  }
  
  #Split
  anons.df <- query.df[query.df$ipb_user == 0,]
  registered.df <- query.df[query.df$ipb_user > 0,]
  
  #Run parse_data.fun and graphing.fun for each
  regex_matches.list <- parse_data.fun(x = registered.df, tablename = "ipblocks", usergroup = "registered")
  graphing.fun(x = regex_matches.list, usergroup = "registered")
  regex_matches.list <- parse_data.fun(x = anons.df, tablename = "ipblocks", usergroup = "anonymous")<commit_msg>bugfix<commit_after>#UserSurvival - a project to accurately triage data on blocked Wikipedia users, identify
#the underlying rationales and test various hypotheses as to any outcome
#
#ipblock.r outputs the basic analysis documented at http://blog.ironholds.org/?p=31
#
# @Year = 2013
# @Copyright: <NAME>
# @License = MIT (http://opensource.org/licenses/MIT)

#Enclosing function
ipblock.fun <- function(){
  
  #Grab dataset
  query.df <- sql.fun(query_statement = "
          SELECT ipb_reason AS reason,
            substring(ipb_timestamp,1,6) AS block_timestamp,
            ipb_user
          FROM ipblocks
          WHERE ipb_timestamp BETWEEN 20060101010101 AND 20121231235959
          AND ipb_expiry = 'infinity';"
  )
  
  #graphing function
  graphing.fun <- function(x, usergroup){
    
    #Split up the list
    monthly_data.df <- as.data.frame(x[1])
    yearly_data.df <- as.data.frame(x[2])
    
    #Line-and-point graphs for yearly and monthly data.
    line_graph_yearly <- ggplot(yearly_data.df, aes(block_timestamp, value)) + 
      geom_freqpoly(aes(group = variable, colour = variable), stat = "identity") +
      labs(x = "Year", y = "Number of users") +
      ggtitle(paste("Block rationales on the English-language Wikipedia\nby year (2006-2012) - ipblocks table,",usergroup,"users",sep = " ")) +
      scale_x_discrete(breaks = 2006:2012, expand = c(0,0)) +
      scale_y_continuous(expand = c(0, 0)) +
      theme(axis.text.x = element_text(angle = 90, hjust = 1))

    line_graph_monthly <- ggplot(monthly_data.df, aes(block_timestamp, value)) +
      geom_freqpoly(aes(group = variable, colour = variable), stat = "identity") +
      labs(x = "Year", y = "Number of users") +
      ggtitle(paste("Block rationales on the English-language Wikipedia\nby month (2006-2012) - ipblocks table,",usergroup,"users",sep = " ")) +
      scale_x_discrete(expand = c(0,0)) +
      scale_y_continuous(expand = c(0,0)) +
      theme(axis.text.x = element_blank(), axis.ticks.x = element_blank())
    
    #print
    ggsave(filename = file.path(getwd(),"Output", paste(usergroup,"_ipblocks_regex_matches_by_year.png", sep = "")),
      plot = line_graph_yearly,
      width = 8,
      height = 8,
      units = "in")
    ggsave(filename = file.path(getwd(),"Output", paste(usergroup,"_ipblocks_regex_matches_by_month.png", sep = "")),
      plot = line_graph_monthly,
      width = 8,
      height = 8,
      units = "in")
    
    #Regression
    regression_graph_monthly <- ggplot(monthly_data.df,aes(x = block_timestamp,y = value, colour = variable))+
      geom_point(shape=3) +
      geom_smooth(method = lm, se = TRUE, aes(group= variable)) +
      labs(x = "Year", y = "Number of users") +
      ggtitle(paste("Block rationales on the English-language Wikipedia\nby month (2006-2012) - ipblocks table,",usergroup,"users",sep = " ")) +
      scale_x_discrete(expand = c(0,0)) +
      scale_y_continuous(expand = c(0,0)) +
      theme(axis.text.x = element_blank(), axis.ticks.x = element_blank())
    
    #Print
    ggsave(filename = file.path(getwd(),"Output", paste(usergroup,"_ipblocks_regex_matches_linear_regression.png", sep = "")),
      plot = regression_graph_monthly,
      width = 8,
      height = 8,
      units = "in")
    
    #The dataset makes it difficult to properly show everything. Let's graph some subsets
    regression_graph_monthly_badfaith <- ggplot(monthly_data.df[monthly_data.df$variable %in% c("bad.faith","spam"),],
      aes(x = block_timestamp,y = value, colour = variable))+
      geom_point(shape=3) +
      geom_smooth(method = lm, se = TRUE, aes(group= variable)) +
      labs(x = "Year", y = "Number of users") +
      ggtitle(paste("Spam and bad-faith blocks on the English-language Wikipedia\nby month (2006-2012) - ipblocks table,",usergroup,"users",sep = " ")) +
      scale_x_discrete(expand = c(0,0)) +
      scale_y_continuous(expand = c(0,0)) +
      theme(axis.text.x = element_blank(), axis.ticks.x = element_blank())
      
    regression_graph_monthly_other <- ggplot(monthly_data.df[!monthly_data.df$variable %in% c("bad.faith","spam"),],
      aes(x = block_timestamp,y = value, colour = variable))+
      geom_point(shape=3) +
      geom_smooth(method = lm, se = TRUE, aes(group= variable)) +
      labs(x = "Year", y = "Number of users") +
      ggtitle(paste("Other blocks on the English-language Wikipedia\nby month (2006-2012) - ipblocks table,",usergroup,"users",sep = " ")) +
      scale_x_discrete(expand = c(0,0)) +
      scale_y_continuous(expand = c(0,0)) +
      theme(axis.text.x = element_blank(), axis.ticks.x = element_blank())
    
    ggsave(filename = file.path(getwd(),"Output", paste(usergroup,"_ipblocks_linear_regression_badfaith.png",sep = "")),
      plot = regression_graph_monthly_badfaith,
      width = 8,
      height = 8,
      units = "in")
    ggsave(filename = file.path(getwd(),"Output", paste(usergroup,"_ipblocks_linear_regression_other.png",sep = "")),
      plot = regression_graph_monthly_other,
      width = 8,
      height = 8,
      units = "in")
  }
  
  #Split
  anons.df <- query.df[query.df$ipb_user == 0,]
  registered.df <- query.df[query.df$ipb_user > 0,]
  
  #Run parse_data.fun and graphing.fun for each
  regex_matches.list <- parse_data.fun(x = registered.df, tablename = "ipblocks", usergroup = "registered")
  graphing.fun(x = regex_matches.list, usergroup = "registered")
  regex_matches.list <- parse_data.fun(x = anons.df, tablename = "ipblocks", usergroup = "anonymous")
{'fraction_non_alphanumeric': 0.14217020504445654, 'fraction_numerical': 0.022500453638178187, 'mean_word_length': 3.523184242921625, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2724854', 'n_tokens_mistral': 3991, 'n_tokens_neox': 3698, 'n_words': 761}