<jupyter_start><jupyter_text>Clustering at the composition scale<jupyter_code>#install.packages('rpostgis', 'rgdal', 'magrittr', 'dbscan', 'corrplot')
library('rpostgis')
library('rgdal')
library('magrittr')
library('tidyverse')
library('dbscan')
#library('corrplot')
conn <- dbConnect("PostgreSQL", dbname = 'base_data', host = 'localhost', user = 'postgres', password = '<PASSWORD>')
cp_scale_query <- "SELECT a.geom, a.id, cp_fac_lgt, cp_fac_lgt_rel,
cp_d_nxtbuil, cp_d_builnxtblk, cp_infzn_area, cp_r_ftp_infzon,
cp_d_blklim, cp_blk_area, cp_r_ftp_blk, cp_closeness
FROM public.bd a,
     public.AGGLO b
WHERE ST_Intersects(a.geom, b.geom)
 AND cp_blk_area IS NOT NULL
ORDER BY a.id;"
sub_in_query <- function(gid, agglo, query) {
    q <- gsub('GID', gid, query) %>%
         gsub('AGGLO', agglo, .)
    return(q)
}
cp_stata <- pgGetGeom(conn, query = sub_in_query(0, "stata", cp_scale_query))
row.names(cp_stata@data) <- cp_stata@data$id
cp_stata@data$id <- NULL 
cp_stata@data <- data.frame(scale(cp_stata@data))
cp_statb <- pgGetGeom(conn, query = sub_in_query(0, "statb", cp_scale_query))
row.names(cp_statb@data) <- cp_statb@data$id
cp_statb@data$id <- NULL 
cp_statb@data <- data.frame(scale(cp_statb@data))
cp_morf <- pgGetGeom(conn, query = sub_in_query(0, "morfrac", cp_scale_query))
row.names(cp_morf@data) <- cp_morf@data$id
cp_morf@data$id <- NULL 
cp_morf@data <- data.frame(scale(cp_morf@data))
cp_ccak <- pgGetGeom(conn, query = sub_in_query(0, "cca_king", cp_scale_query))
row.names(cp_ccak@data) <- cp_ccak@data$id
cp_ccak@data$id <- NULL 
cp_ccak@data <- data.frame(scale(cp_ccak@data))
cp_ccal <- pgGetGeom(conn, query = sub_in_query(0, "cca_lenient", cp_scale_query))
row.names(cp_ccal@data) <- cp_ccal@data$id
cp_ccal@data$id <- NULL 
cp_ccal@data <- data.frame(scale(cp_ccal@data))<jupyter_output>Returning MultiPolygon types in SpatialPolygons*-class.<jupyter_text>StatA<jupyter_code>kNNdistplot(cp_stata@data, k = 10)
abline(h = 1.2, col = 'red', lty = 2)
res_stata <- dbscan(cp_stata@data, eps = 1.2, minPts = 11)
res_stata
clusDBSCAN <- res_stata$cluster
cp_stata@data <- cbind(cp_stata@data, clusDBSCAN)
head(cp_stata@data)
sel <- cp_stata@data$clusDBSCAN == 1
cp_stata_to_class <- cp_stata[sel,]
head(cp_stata_to_class@data)
wss_clusters = seq(1, 30)
for (i in 1:30) {
    c = kmeans(cp_stata_to_class@data[, c(1:10)], i, nstart=5, iter.max = 50)
    wss_clusters[i] = c$betweenss / c$totss
}
plot(wss_clusters, type="o")
stata_kmeans <- kmeans(cp_stata_to_class@data[, -11], 14, nstart = 5, iter.max = 50)
round(stata_kmeans$betweenss / stata_kmeans$totss, 2)
clusKME14 <- stata_kmeans$cluster
cp_stata_to_class@data <- cbind(cp_stata_to_class@data, clusKME14)
writeOGR(cp_stata, dsn = "cp_stata.gpkg", layer = 'dbscan', driver = "GPKG")
writeOGR(cp_stata_to_class, dsn = "cp_stata_kmeans.gpkg", layer = 'km14', driver = "GPKG")<jupyter_output><empty_output><jupyter_text>StatB<jupyter_code>kNNdistplot(cp_statb@data, k = 10)
abline(h = 1.2, col = 'red', lty = 2)
res_statb <- dbscan(cp_statb@data[, c(1:10)], eps = 1.3, minPts = 11)
res_statb
clusDBSCAN <- res_statb$cluster
cp_statb@data <- cbind(cp_statb@data, clusDBSCAN)
head(cp_statb@data)
sel <- cp_statb@data$clusDBSCAN == 1
cp_statb_to_class <- cp_statb[sel,]
head(cp_statb_to_class@data)
wss_clusters = seq(1, 30)
for (i in 1:30) {
    c = kmeans(cp_statb_to_class@data[, c(1:10)], i, nstart=5, iter.max = 50)
    wss_clusters[i] = c$betweenss / c$totss
}
plot(wss_clusters, type="o")
statb_kmeans <- kmeans(cp_statb_to_class@data[, c(1:10)], 15, nstart = 5, iter.max = 50)
round(statb_kmeans$betweenss / statb_kmeans$totss, 2)
clusKME22 <- statb_kmeans$cluster
cp_statb_to_class@data <- cbind(cp_statb_to_class@data, clusKME22)
writeOGR(cp_statb, dsn = "cp_statb.gpkg", layer = 'dbscan', driver = "GPKG")
writeOGR(cp_statb_to_class, dsn = "cp_statb_kmeans.gpkg", layer = 'km15', driver = "GPKG")<jupyter_output><empty_output><jupyter_text>Fractal morphology<jupyter_code>kNNdistplot(cp_morf@data, k = 10)
abline(h = 1.3, col = 'red', lty = 2)
res_morf <- dbscan(cp_morf@data[, c(1:10)], eps = 1.3, minPts = 11)
res_morf
clusDBSCAN <- res_morf$cluster
cp_morf@data <- cbind(cp_morf@data, clusDBSCAN)
head(cp_morf@data)
sel <- cp_morf@data$clusDBSCAN == 1
cp_morf_to_class <- cp_morf[sel,]
head(cp_statb_to_class@data)
wss_clusters = seq(1, 30)
for (i in 1:30) {
    c = kmeans(cp_morf_to_class@data[, c(1:10)], i, nstart=5, iter.max = 50)
    wss_clusters[i] = c$betweenss / c$totss
}
plot(wss_clusters, type="o")
morf_kmeans <- kmeans(cp_morf_to_class@data[, c(1:10)], 16, nstart = 5, iter.max = 50)
round(morf_kmeans$betweenss / morf_kmeans$totss, 2)
clusKME16 <- morf_kmeans$cluster
cp_morf_to_class@data <- cbind(cp_morf_to_class@data, clusKME16)
writeOGR(cp_morf, dsn = "cp_morf.gpkg", layer = 'dbscan', driver = "GPKG")
writeOGR(cp_morf_to_class, dsn = "cp_morf_kmeans.gpkg", layer = 'km16', driver = "GPKG")<jupyter_output><empty_output><jupyter_text>CCA King<jupyter_code>kNNdistplot(cp_ccak@data, k = 10)
abline(h = 1.3, col = 'red', lty = 2)
res_ccak <- dbscan(cp_ccak@data[, c(1:10)], eps = 1.3, minPts = 11)
res_ccak
clusDBSCAN <- res_ccak$cluster
cp_ccak@data <- cbind(cp_ccak@data, clusDBSCAN)
head(cp_ccak@data)
sel <- cp_ccak@data$clusDBSCAN == 1
cp_ccak_to_class <- cp_ccak[sel,]
wss_clusters = seq(1, 30)
for (i in 1:30) {
    c = kmeans(cp_ccak_to_class@data[, c(1:10)], i, nstart=5, iter.max = 50)
    wss_clusters[i] = c$betweenss / c$totss
}
plot(wss_clusters, type="o")
ccak_kmeans <- kmeans(cp_ccak_to_class@data[, c(1:10)], 18, nstart = 5, iter.max = 50)
round(ccak_kmeans$betweenss / ccak_kmeans$totss, 2)
clusKME18 <- ccak_kmeans$cluster
cp_ccak_to_class@data <- cbind(cp_ccak_to_class@data, clusKME18)
writeOGR(cp_ccak, dsn = "cp_ccak.gpkg", layer = 'dbscan', driver = "GPKG")
writeOGR(cp_ccak_to_class, dsn = "cp_ccak_kmeans.gpkg", layer = 'km18', driver = "GPKG")<jupyter_output><empty_output><jupyter_text>CCA Lenient<jupyter_code>kNNdistplot(cp_ccal@data, k = 10)
abline(h = 1.3, col = 'red', lty = 2)
res_ccal <- dbscan(cp_ccal@data[, c(1:10)], eps = 1.3, minPts = 11)
res_ccal
clusDBSCAN <- res_ccal$cluster
cp_ccal@data <- cbind(cp_ccal@data, clusDBSCAN)
head(cp_ccal@data)
sel <- cp_ccal@data$clusDBSCAN == 1
cp_ccal_to_class <- cp_ccal[sel,]
wss_clusters = seq(1, 30)
for (i in 1:30) {
    c = kmeans(cp_ccal_to_class@data[, c(1:10)], i, nstart=5, iter.max = 50)
    wss_clusters[i] = c$betweenss / c$totss
}
plot(wss_clusters, type="o")
ccal_kmeans <- kmeans(cp_ccal_to_class@data[, c(1:10)], 19, nstart = 5, iter.max = 50)
round(ccal_kmeans$betweenss / ccal_kmeans$totss, 2)
clusKME19 <- ccal_kmeans$cluster
cp_ccal_to_class@data <- cbind(cp_ccal_to_class@data, clusKME19)
writeOGR(cp_ccal, dsn = "cp_ccal.gpkg", layer = 'dbscan', driver = "GPKG")
writeOGR(cp_ccal_to_class, dsn = "cp_ccal_kmeans.gpkg", layer = 'km19', driver = "GPKG")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16147267719959416, 'fraction_numerical': 0.029134657196695172, 'mean_word_length': 6.658157602663707, 'pattern_counts': {'":': 0, '<': 89, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '450358', 'license': 'MIT', 'n_tokens_mistral': 3445, 'n_tokens_neox': 3170, 'n_words': 515, 'path': 'analyse/5_clustering_cp_scale.ipynb', 'repo_name': 'Raphbub/master-thesis', 'size': '6892'}