<jupyter_start><jupyter_text>Parse and convert Carrier data to parquet<jupyter_code>carriers <- read.df(storage_path("carriers.csv"), "csv", header="true", inferSchema="true")
write.df(carriers, path=full_path("carriers"), source="parquet", mode="overwrite")
createOrReplaceTempView(carriers, "carriers")
head(carriers, 20)<jupyter_output><empty_output><jupyter_text>Parse and convert to parquet Airport data<jupyter_code>airports <- read.df(storage_path("airports.csv"), "csv", header="true", inferSchema="true")
write.df(airports, path=full_path("airports"), source="parquet", mode="overwrite")
createOrReplaceTempView(airports, "airports")
head(airports, 20)<jupyter_output><empty_output><jupyter_text>Parse and convert Flights data to parquet<jupyter_code>flights_w_na <- read.df(storage_path("2008.csv.bz2"), "csv", header="true", inferSchema="true")
flights <- fillna(flights_w_na, 0, cols=colnames(flights_w_na)[c(15, 16, 25:29)])
write.df(flights, path=full_path("flights"), source="parquet", mode="overwrite")
createOrReplaceTempView(flights, "flights")
colnames(flights)
head(flights_w_na, 5)[c("ArrDelay","CarrierDelay","WeatherDelay","Distance")]
head(flights, 5)[c("ArrDelay","CarrierDelay","WeatherDelay","Distance")]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17129262490087233, 'fraction_numerical': 0.01586042823156225, 'mean_word_length': 14.390243902439025, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '314040', 'license': 'Apache-2.0', 'n_tokens_mistral': 463, 'n_tokens_neox': 439, 'n_words': 64, 'path': 'integration-tests/examples/test_templates/jupyter/template_preparation_sparkr.ipynb', 'repo_name': 'AdamsDisturber/incubator-dlab', 'size': '1261'}