<jupyter_start><jupyter_text>Otomoto analysis  Prepare environmentIf you're running this in binder, you will need to install `poetry` using following snippet:<jupyter_code>!curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python -<jupyter_output><empty_output><jupyter_text>Install dependenciesJust to be on the safe side with reproducibility, run following command:<jupyter_code>!$HOME/.poetry/bin/poetry install -q
import pandas as pd
import os
import glob
import math
import csv
import re
from pprint import pprint

import sklearn
import tqdm
import matplotlib.pyplot as plt
import plotly.express as px
from plotly.subplots import make_subplots
from IPython.display import Image
import plotly.graph_objects as go
print(sklearn.__version__)<jupyter_output>0.23.2<jupyter_text>Data preprocessingThis section assumes data is stored in a folder data in subfolders. ```data└── subfolder1    └── file1.csv```<jupyter_code>dfs = []
for f in tqdm.tqdm(glob.glob("data/*/*.csv")):
    make, model = f.split('_')[:2]
    model_df = pd.read_csv(f, quoting=csv.QUOTE_MINIMAL, escapechar='\\')
    model_df['make'] = make
    model_df['model'] = model
    dfs.append(model_df)
df = pd.concat(dfs)
df = df.drop_duplicates(subset=["otomoto_id"], keep="first")
### filter out new cars
df = df[(df.mileage > 1000)]
EPS = 0.001
df = df[df.mileage.between(0, df.mileage.quantile(1-EPS))]
df = df[df.price.between(0, df.price.quantile(1-EPS))]
df.engine = df.engine / 10 - 0.3
MODELS = df.model.unique()

print(f"Available fuel_types:")
pprint(list(df.fuel_type.unique()))
print("Available models")
pprint(list(MODELS))
# filter out broken, RHD, cession
df = df[~df.subtitle.str.contains("uszkodzo", na=False, flags=re.IGNORECASE)]
df = df[~df.subtitle.str.contains("pali", na=False, flags=re.IGNORECASE)]
df = df[~df.subtitle.str.contains("v5c", na=False, flags=re.IGNORECASE)]
df = df[~df.subtitle.str.contains("cesja", na=False, flags=re.IGNORECASE)]
df = df[~df.subtitle.str.contains("przeję", na=False, flags=re.IGNORECASE)]

pd.set_option('display.max_colwidth', -1)

df.subtitle.head()<jupyter_output>/home/tomek/.pyenv/versions/3.7.2/envs/ml/lib/python3.7/site-packages/ipykernel_launcher.py:8: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.<jupyter_text>Plots<jupyter_code>def plot_scatter(df, model, years, fuel_type, engine):
    """Plot scatter plot for a given search criteria"""
    data = df.query(
        "model == @model & "
        "year < @years[1] & "
        "year >= @years[0] & "
        "fuel_type == @fuel_type & "
        "engine < @engine[1] & "
        "engine >= @engine[0]"
    )
    fig = px.scatter(
        data, x="mileage", y="price", color="year",
        render_mode="svg", 
        title=f"Otomoto prices for {model} {engine}",
        marginal_y="box", marginal_x="histogram",
        trendline="ols"
    )
    return fig.to_image(format='png')

Image(plot_scatter(df, "octavia", [2010, 2014], "Benzyna", [1200, 2000]))
from jupyter_dash import JupyterDash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output

app = JupyterDash("otomoto")
app.layout = html.Div([
    html.H1("Otomoto explorer"),
    dcc.Graph(id='graph'),
    html.Label([
        "model",
        dcc.Dropdown(
            id='model-dropdown', clearable=False,
            value='civic', options=[
                {'label': c, 'value': c}
                for c in MODELS
            ])
    ]),
    html.Label([
        "year",
        dcc.RangeSlider(
            id="year-selector",
            min=1998,
            max=2021,
            value=[2014, 2018],
            marks={ix: str(ix) for ix in range(1998, 2022)}
        )
    ]),
    html.Label([
        "fuel_type", 
        dcc.Dropdown(
            id="fuel_type-selector",
            options=[{'label': f, 'value': f} for f in df.fuel_type.unique()],
            value='Benzyna'
        )  
    ]),
    html.Label([
        "engine",
        dcc.RangeSlider(
            id="engine-selector",
            min=df.engine.min(),
            max=df.engine.max(),
            value=[df.engine.min(), df.engine.max()],
            step=100,
        )
    ])
    
])


# Define callback to update graph
@app.callback(
    Output('graph', 'figure'),
    [
        Input("model-dropdown", "value"),
        Input("year-selector", "value"),
        Input("fuel_type-selector", "value"),
        Input("engine-selector", "value")
    ],
)
def update_figure(model, years, fuel_type, engine):
    data = df.query(
        "model == @model & "
        "year < @years[1] & "
        "year >= @years[0] & "
        "fuel_type == @fuel_type & "
        "engine < @engine[1] & "
        "engine >= @engine[0]")
    return px.scatter(
        data, x="mileage", y="price", color="year",
        render_mode="webgl", title=f"Otomoto prices for {model} {engine}",
        marginal_y="box", marginal_x="histogram",
        trendline="ols"
    )



app.run_server(mode='inline')
def plot_price_decline(df, num_years=3):
    EXAMPLE_THRESHOLD = 5
    YEAR_THRESHOLD = 3
    group_criteria = ['model', 'fuel_type', "engine", 'year']
    gdf = df.groupby(by=group_criteria)
    gdf = gdf.filter(lambda x: x.otomoto_id.nunique() > EXAMPLE_THRESHOLD)
    
    gdf = gdf.groupby(by=group_criteria).price.median().to_frame()
    gdf["count"] = df.groupby(by=group_criteria).otomoto_id.nunique()
    gdf[f"price_{num_years}"] = gdf.groupby(group_criteria[:-1]).price.shift(num_years)
    gdf[f"price_diff_{num_years}"] = gdf.price - gdf[f"price_{num_years}"]
    gdf[f"price_ratio_{num_years}"] = (gdf.price - gdf[f"price_{num_years}"])/gdf.price
    gdf = gdf[
        gdf[f"price_ratio_{num_years}"].notnull()
    ].sort_values(
        f'price_diff_{num_years}'
    )[["count", "price", f"price_{num_years}", f"price_diff_{num_years}", f"price_ratio_{num_years}"]]
    return gdf


# For every (fuel_type, engine, year) combination for a given model, calculate the price decline
# for a given time horizon
plot_price_decline(df, num_years=4).xs("tiguan", level=0)
def plot_mileage_hist(query, plot_params):
    mileage = df.query(query)
    mileage.hist(bins=20, **plot_params)

for i_model in ["octavia", "golf"]:
    plot_mileage_hist(f"model == '{i_model}'", {'alpha': 0.5})<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15540749922528665, 'fraction_numerical': 0.01471955376510691, 'mean_word_length': 3.4182067077344285, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '662450', 'license': 'MIT', 'n_tokens_mistral': 2388, 'n_tokens_neox': 2281, 'n_words': 457, 'path': 'otomoto-analysis.ipynb', 'repo_name': 'tomecki/otomoto-explorer', 'size': '6454'}