<jupyter_start><jupyter_text>EXPLORATIVE ANALYSIS<jupyter_code>import pandas as pd
import numpy as np
%matplotlib inline  
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression
import geopandas as gpd<jupyter_output><empty_output><jupyter_text>Analysis of change in LORs<jupyter_code># Model initialization
regression_model = LinearRegression()

df['reg_coef'] = None
df['nan_count'] = 0

for index, row in df.iterrows():
    df.loc[index,'nan_count'] = row[:-2][row[:-2].isna()].shape[0]
    y = row.iloc[2:15].dropna()
    idx = y.index
    x = []
    for key in idx:
        x.append(row.index.get_loc(key))
    if x:
        df.loc[index,'reg_coef'] = regression_model.fit(np.array(x).reshape(-1, 1), y).coef_[0]
df[(df.nan_count < 5) & (df.reg_coef > 0)].sort_values('reg_coef', ascending=False).set_index('Name').iloc[:,1:14].head(10).T.plot()
df[(df.nan_count < 5) & (df.reg_coef < 0)].sort_values('reg_coef', ascending=True).set_index('Name').iloc[:,1:14].head(10).T.plot()
# set coef to nan where there are not enough values
df = df.loc[(df.nan_count < 5)]
df['reg_coef'] = pd.to_numeric(df['reg_coef']).round(2)
df_coef = df.set_index(['Kennung'])[['reg_coef']]

plr = gpd.read_file('lor_planungsraeume.geojson', encoding='utf-8')

data = df_coef.join(plr.set_index('spatial_name'), rsuffix='_r')

gpd.GeoDataFrame(data).to_file('time_change.geojson', driver='GeoJSON', encoding='utf-8')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17869415807560138, 'fraction_numerical': 0.019243986254295534, 'mean_word_length': 5.835680751173709, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '287188', 'license': 'MIT', 'n_tokens_mistral': 604, 'n_tokens_neox': 582, 'n_words': 94, 'path': 'preprocess-data/analysis.ipynb', 'repo_name': 'technologiestiftung/grundsicherung_v2', 'size': '1455'}