<commit_before>library(geosphere)
library(mice)
library(dplyr)

set.seed(7)

extend <- function(flats, stations, regions) {
  flats$distance_to_subway <- apply(distm(flats[, c("longitude", "latitude")], stations[, c("longitude", "latitude")], fun = distHaversine), 1, min)
  flats$first_or_last_floor <- flats$floor == 1 | flats$floor == flats$total_floors
  flats$other_area = flats$total_area - flats$living_area - flats$kitchen_area
  distances_to_regions <- distm(flats[, c("longitude", "latitude")], regions[, c("longitude", "latitude")], fun = distHaversine) %*% diag(regions$radius_coef)
  belonging_to_regions <- distances_to_regions == apply(distances_to_regions, 1, min)
  belonging_to_regions_indexes <- which(belonging_to_regions, arr.in = TRUE)
  region_numbers <- belonging_to_regions_indexes[order(belonging_to_regions_indexes[, 1]), 2]
  flats$region <- factor(regions$name[region_numbers], levels = levels(regions$name))
  return(flats)
}

impute <- function(flats, my_flat) {
  imp_flats <- as.data.frame(bind_rows(flats, my_flat))
  imp_flats <- imp_flats[, c("rooms", "total_floors", "living_area", "kitchen_area", "other_area", "house_type", "year", "balcony", "ceiling_height", "distance_to_subway", "region")]
  imp_flats_mids <- mice(imp_flats, m = 1, maxit = 100)
  imp_flats <- complete(imp_flats_mids)
  imp_flats <- imp_flats[-nrow(imp_flats),]
  flats$year <- imp_flats$year
  flats$ceiling_height <- imp_flats$ceiling_height
  return(flats)
}

flats <- read.csv(file = "onliner-2016-06-25T18:05:07.csv", head = TRUE, sep = ",", na.strings = c(""))

flats$resale = flats$resale == 1
flats$house_type = factor(flats$house_type)
flats$balcony = flats$balcony == 1
flats$parking = factor(flats$parking)
flats$cottage = flats$cottage == 1
flats$actual = flats$actual == 1

flats <- flats[, c("latitude", "longitude", "price", "resale", "rooms", "total_floors", "floor", "total_area", "living_area", "kitchen_area", "house_type", "year", "balcony", "parking", "ceiling_height", "cottage", "actual")]
flats <- flats[flats$actual,]
flats <- flats[!flats$cottage,]<commit_msg>Add id to fields list.<commit_after>library(geosphere)
library(mice)
library(dplyr)

set.seed(7)

extend <- function(flats, stations, regions) {
  flats$distance_to_subway <- apply(distm(flats[, c("longitude", "latitude")], stations[, c("longitude", "latitude")], fun = distHaversine), 1, min)
  flats$first_or_last_floor <- flats$floor == 1 | flats$floor == flats$total_floors
  flats$other_area = flats$total_area - flats$living_area - flats$kitchen_area
  distances_to_regions <- distm(flats[, c("longitude", "latitude")], regions[, c("longitude", "latitude")], fun = distHaversine) %*% diag(regions$radius_coef)
  belonging_to_regions <- distances_to_regions == apply(distances_to_regions, 1, min)
  belonging_to_regions_indexes <- which(belonging_to_regions, arr.in = TRUE)
  region_numbers <- belonging_to_regions_indexes[order(belonging_to_regions_indexes[, 1]), 2]
  flats$region <- factor(regions$name[region_numbers], levels = levels(regions$name))
  return(flats)
}

impute <- function(flats, my_flat) {
  imp_flats <- as.data.frame(bind_rows(flats, my_flat))
  imp_flats <- imp_flats[, c("rooms", "total_floors", "living_area", "kitchen_area", "other_area", "house_type", "year", "balcony", "ceiling_height", "distance_to_subway", "region")]
  imp_flats_mids <- mice(imp_flats, m = 1, maxit = 100)
  imp_flats <- complete(imp_flats_mids)
  imp_flats <- imp_flats[-nrow(imp_flats),]
  flats$year <- imp_flats$year
  flats$ceiling_height <- imp_flats$ceiling_height
  return(flats)
}

flats <- read.csv(file = "onliner-2016-06-29T20:12:04.csv", head = TRUE, sep = ",", na.strings = c(""))

flats$resale = flats$resale == 1
flats$house_type = factor(flats$house_type)
flats$balcony = flats$balcony == 1
flats$parking = factor(flats$parking)
flats$cottage = flats$cottage == 1
flats$actual = flats$actual == 1

flats <- flats[, c("id", "latitude", "longitude", "price", "resale", "rooms", "total_floors", "floor", "total_area", "living_area", "kitchen_area", "house_type", "year", "balcony", "parking", "ceiling_height", "cottage", "actual")]
flats <- flats[flats$actual,]
flats <- flats[!flats$cottage,]
{'fraction_non_alphanumeric': 0.17011166547873605, 'fraction_numerical': 0.013304822998336897, 'mean_word_length': 6.928436911487759, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '148029', 'n_tokens_mistral': 1735, 'n_tokens_neox': 1576, 'n_words': 278}