<commit_before>library("ggplot2")
setwd("~/dev/BitFunnel/src/Scripts")

png(filename="qwords.png",width=1600,height=1200)
queries <- read.csv(header=TRUE, file="/tmp/QueryPipelineStatistics.csv")
pos = seq(1, length(queries$quadwords))
df <- data.frame(pos, queries$quadwords)

ggplot(df, aes(x=pos,y=queries.quadwords)) +
theme_bw() +
geom_point(alpha=1/100) +
theme(axis.text = element_text(size=40),
      axis.title = element_text(size=40),
      legend.title=element_text(size=40),
      legend.text=element_text(size=40))
dev.off()


png(filename="qwords-solid.png",width=1600,height=1200)
ggplot(df, aes(x=pos,y=queries.quadwords)) +
theme_bw() +
geom_point(alpha=1) +
theme(axis.text = element_text(size=40),
      axis.title = element_text(size=40),
      legend.title=element_text(size=40),
      legend.text=element_text(size=40))
dev.off()<commit_msg>Add cachlines accessed to qwords plot.<commit_after>library("ggplot2")
library("reshape")
setwd("~/dev/BitFunnel/src/Scripts")


queries <- read.csv(header=TRUE, file="/tmp/QueryPipelineStatistics.csv")
pos = seq(1, length(queries$quadwords))
df_temp <- data.frame(pos, queries$quadwords, queries$cachelines)
df <- melt(df_temp, id=c("pos"))

png(filename="qwords.png",width=1600,height=1200)
ggplot(df, aes(x=pos,y=value,colour=factor(variable))) +
theme_bw() +
geom_point(alpha=1/150) +
theme(axis.text = element_text(size=40),
      axis.title = element_text(size=40),
      legend.title=element_text(size=40),
      legend.text=element_text(size=40))
dev.off()

df_temp$queries.quadwords <- df_temp$queries.quadwords / 4
df <- melt(df_temp, id=c("pos"))

png(filename="qwords-divided.png",width=1600,height=1200)
ggplot(df, aes(x=pos,y=value,colour=factor(variable))) +
theme_bw() +
geom_point(alpha=1/150) +
theme(axis.text = element_text(size=40),
      axis.title = element_text(size=40),
      legend.title=element_text(size=40),
      legend.text=element_text(size=40))
dev.off()


# png(filename="qwords-solid.png",width=1600,height=1200)
# ggplot(df, aes(x=pos,y=value)) +
# theme_bw() +
# geom_point(alpha=1) +
# theme(axis.text = element_text(size=40),
#       axis.title = element_text(size=40),
#       legend.title=element_text(size=40),
#       legend.text=element_text(size=40))
# dev.off()
{'fraction_non_alphanumeric': 0.18273888154997797, 'fraction_numerical': 0.043593130779392336, 'mean_word_length': 6.352750809061488, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3918253', 'n_tokens_mistral': 934, 'n_tokens_neox': 860, 'n_words': 97}