<jupyter_start><jupyter_text>Anomaly Detection in Google Trends dataA statistical analysis of data, time dependent, using 'Anomaly Detection' in R.<jupyter_code>#   __      _____             _              __        _       
# <(o )___ |  _  |___ ___ ___| |_ ___ ___   |  |   ___| |_ ___ 
# ( ._> /  |   __|  _| -_|_ -|  _| -_|_ -|  |  |__| .'| . |_ -|
#  `---'   |__|  |_| |___|___|_| |___|___|  |_____|__,|___|___|
#==============================================================================
# Anomaly Detection in R
#==============================================================================
# Title          : ilegra.r
# Description    : Analyzing Google Trends Data in R.
# Author         : <NAME> <<EMAIL>>
# Date           : 20180419
# Version        : 0.0.1
# Usage          : Run in R 3.4
# Notes          : based on 
#                  <NAME> (DisplayR)
# https://datascienceplus.com/analyzing-google-trends-data-in-r/
#                  gtrendsR documentation
# https://github.com/PMassicotte/gtrendsR
#                  Peer Christensen https://rpubs.com/PChristensen/307008
#                  Tibble package
# https://www.rdocumentation.org/packages/tibbletime/versions/0.0.2/topics/tbl_time
# https://github.com/business-science/anomalize/blob/master/vignettes/anomalize_quick_start_guide.Rmd
# R version      : 3.4
#==============================================================================

#==============================================================================
# PACKAGE INSTALATION
#==============================================================================
if(!require("anomalize")) install.packages("anomalize")
if(!require("tidyverse")) install.packages("tidyverse")
if(!require("tibbletime")) install.packages("tibbletime")
if(!require("gtrendsR")) install.packages("gtrendsR") # google data
if(!require("ggplot2")) install.packages("ggplot2") # google data
if(!require("dplyr")) install.packages("dplyr") # google data
#==============================================================================
# LIBRARY DEPENDENCE
#==============================================================================
library(anomalize) #tidy anomaly detectiom
library(tidyverse) #tidyverse packages like dplyr, ggplot, tidyr
library(tibbletime) #tidyverse packages like dplyr, ggplot, tidyr
library(gtrendsR)
library(dplyr)
library(ggplot2)


#==============================================================================
# Fire in the hole!
#==============================================================================

# Search Parameters -----------------------------------------------------------
search_terms <- c("ilegra")
# Countries codes https://en.wikipedia.org/wiki/ISO_3166-2
# detaild version http://www.unece.org/cefact/codesfortrade/codes_index.html
search_country <- c("BR") # default is worldwide
# Time interval - Time span between two dates (ex.: "2010-01-01 2010-04-03")
search_time <- "2011-01-01 2018-04-18" 
# Options available are "news","images", "froogle", "youtube"
search_type <- c("web") 
# A character denoting the category, defaults to “0”.
search_category <- 0
# ISO language code
search_language <- c("pt-BR")

data_scientist <- 
gtrends(search_terms, geo = search_country, time = search_time,
	gprop = search_type, category = search_category, hl = search_language)

windows(width = 720, height = 320)

data_scientist$interest_over_time%>%
	as_tibble()%>%
	mutate(hits=as.numeric(hits))%>%
ggplot(aes(x=date,y=hits,colour=keyword))+
	geom_line()+theme_bw()+
	labs(y="Popularidade",
			x="Ano",colour="Termo de pesquisa:", 
			title="Série temporal da popularidade dos termos de pesquisas")

# Data reshape for anomalies analysis ---------------------------------------- 
count_data <- data_scientist$interest_over_time[,"hits"]
time_data <- data_scientist$interest_over_time[,"date"]
data_anolmalyDec <- data.frame(time_data, count_data)
data_anolmalyDec_tbl <- as.tibble(data_anolmalyDec)

data_anolmalyDec_tbl <- data_anolmalyDec %>% as.tibble() %>% 
  mutate(date = as.Date(time_data)) %>% select(-one_of('time_data'))
head(data_anolmalyDec_tbl)

windows(width = 720, height = 320)

data_anolmalyDec_tbl %>% 
  time_decompose(count_data, method = "stl", frequency = "auto", trend = "auto") %>%
  anomalize(remainder, method = "gesd", alpha = 0.05, max_anoms = 0.2) %>%
  plot_anomaly_decomposition() 

windows(width = 720, height = 320)
 
data_anolmalyDec_tbl %>% 
  time_decompose(count_data) %>%
  anomalize(remainder) %>%
  time_recompose() %>%
  plot_anomalies(time_recomposed = TRUE, ncol = 3, alpha_dots = 0.5)  

data_anolmalyDec_tbl %>% 
  time_decompose(count_data) %>%
  anomalize(remainder) %>%
  time_recompose() %>%
  filter(anomaly == 'Yes') 
  plot_anomalies(time_recomposed = TRUE, ncol = 3, alpha_dots = 0.5)<jupyter_output><empty_output><jupyter_text>Period 2017 towards present<jupyter_code># 2017 |- present period ---------------------------------------------------
search_time <- "2017-01-01 2018-04-18" 
data_scientist <- 
gtrends(search_terms, geo = search_country, time = search_time,
	gprop = search_type, category = search_category, hl = search_language)

windows(width = 720, height = 320)

data_scientist$interest_over_time%>%
	as_tibble()%>%
	mutate(hits=as.numeric(hits))%>%
ggplot(aes(x=date,y=hits,colour=keyword))+
	geom_line()+theme_bw()+
	labs(y="Popularidade",
			x="Ano",colour="Termo de pesquisa:", 
			title="Série temporal da popularidade dos termos de pesquisas")


count_data <- data_scientist$interest_over_time[,"hits"]
time_data <- data_scientist$interest_over_time[,"date"]
data_anolmalyDec <- data.frame(time_data, count_data)
data_anolmalyDec_tbl <- as.tibble(data_anolmalyDec)

data_anolmalyDec_tbl <- data_anolmalyDec %>% as.tibble() %>% 
  mutate(date = as.Date(time_data)) %>% select(-one_of('time_data'))
head(data_anolmalyDec_tbl)

windows(width = 720, height = 320)

data_anolmalyDec_tbl %>% 
  time_decompose(count_data, method = "stl", frequency = "auto", trend = "auto") %>%
  anomalize(remainder, method = "gesd", alpha = 0.05, max_anoms = 0.2) %>%
  plot_anomaly_decomposition() 

windows(width = 720, height = 320)
 
data_anolmalyDec_tbl %>% 
  time_decompose(count_data) %>%
  anomalize(remainder) %>%
  time_recompose() %>%
  plot_anomalies(time_recomposed = TRUE, ncol = 3, alpha_dots = 0.5)  

data_anolmalyDec_tbl %>% 
  time_decompose(count_data) %>%
  anomalize(remainder) %>%
  time_recompose() %>%
  filter(anomaly == 'Yes') 
# EOF<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.25980618366405167, 'fraction_numerical': 0.022304260882941088, 'mean_word_length': 4.614853195164076, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 48, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '117569', 'license': 'MIT', 'n_tokens_mistral': 2386, 'n_tokens_neox': 2087, 'n_words': 359, 'path': 'Other/ilegra term.ipynb', 'repo_name': 'isix/R', 'size': '6533'}