<jupyter_start><jupyter_text>Mustererkennung in Funkmessdaten  Aufgabe 1: Laden der Datenbank in Jupyter Notebook<jupyter_code>%matplotlib inline
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pprint as pp
hdfs = pd.HDFStore("../../data/raw/henrik/TestMessungen_NEU.hdf")
hdfs.keys<jupyter_output><empty_output><jupyter_text>Aufgabe 2: Inspektion eines einzelnen Dataframes<jupyter_code>df1 = hdfs.get('/x1/t1/trx_1_2')
df1.head(5)
# Little function to retrieve sender-receiver tuple from df columns
import re
def extract_snd_rcv(df):
    regex = r"trx_[1-4]_[1-4]_ifft_[0-9]*"
    snd_rcv = {x[4:7] for x in df.columns if re.search(regex, x)}
    return [(x[0],x[-1]) for x in snd_rcv]
def get_column_counts(snd_rcv, df):
    col_counts = {}
    for snd,rcv in snd_rcv:
        col_counts['trx_{}_{}_ifft'.format(snd, rcv)] = len([i for i, word in enumerate(list(df.columns)) if word.startswith('trx_{}_{}_ifft'.format(snd, rcv))])
    return col_counts
df1_snd_rcv = extract_snd_rcv(df1)
cc = get_column_counts(df1_snd_rcv, df1)

pp.pprint(cc)
print("Sum of measure columns: %i" % sum(cc.values()))
print("# of other columns: %i" % (len(df1.columns) - sum(cc.values())))
[col for col in df1.columns if 'ifft' not in col]
print(df1['target'].unique())
print("# Unique values in target: %i" % len(df1['target'].unique()))
df2 = hdfs.get('/x1/t1/trx_1_4')
df2.head()
import re
df2_snd_rcv = extract_snd_rcv(df2)
cc = get_column_counts(df2_snd_rcv, df2)
pp.pprint(cc)
print("Sum of measure columns: %i" % sum(cc.values()))
print("# of other columns: %i" % (len(df2.columns) - sum(cc.values())))
[col for col in df2.columns if 'ifft' not in col]
print(df2['target'].unique())
print("# Unique values in target: %i" % len(df2['target'].unique()))<jupyter_output>['Empty_0.0,0.0_0.0,0.0' 'Standing_1.0,1.0_1.0,1.0' 'Step_1.0,1.0_1.0,2.0'
 'Standing_1.0,2.0_1.0,2.0' 'Step_1.0,2.0_2.0,2.0'
 'Standing_2.0,2.0_2.0,2.0' 'Step_2.0,2.0_2.0,1.0'
 'Standing_2.0,1.0_2.0,1.0' 'Step_2.0,1.0_1.0,1.0'
 'Walking_0.0,0.0_0.0,0.0']
# Unique values in target: 10<jupyter_text>Aufgabe 3: Visualisierung & Groundtruth-Label  Visualisierung<jupyter_code>plt.figure(figsize=(20, 15))
ax = sns.heatmap(df1.loc[:,'trx_1_2_ifft_0':'trx_1_2_ifft_1999'].values, cmap='nipy_spectral_r')
plt.figure(figsize=(20, 15))
ax = sns.heatmap(df2.loc[:,'trx_2_4_ifft_0':'trx_2_4_ifft_1999'].values, cmap='YlGnBu')<jupyter_output><empty_output><jupyter_text>Groundtruth-Label anpassen<jupyter_code># Iterating over hdfs data and creating interim data presentation stored in data/interim/henrik/testmessungen_interim.hdf
# Interim data representation contains aditional binary class (binary_target - encoding 0=empty and 1=not empty)
# and multi class target (multi_target - encoding 0-9 for each possible class)
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()

interim_path = '../../data/interim/henrik/01_testmessungen.hdf'

def binary_mapper(df):
    
    def map_binary(target):
        if target.startswith('Empty'):
            return 0
        else:
            return 1
    
    df['binary_target'] = pd.Series(map(map_binary, df['target']))
    
    
def multiclass_mapper(df):
    le.fit(df['target'])
    df['multi_target'] = le.transform(df['target'])
    
for key in hdfs.keys():
    df = hdfs.get(key)
    binary_mapper(df)
    multiclass_mapper(df)
    df.to_hdf(interim_path, key)

hdfs.close()<jupyter_output><empty_output><jupyter_text>Aufgabe 4: Einfacher Erkenner mit Hold-Out-Validierung<jupyter_code>from evaluation import *
from filters import *
from utility import *
from features import *
hdfs = pd.HDFStore('../../data/interim/henrik/01_testmessungen.hdf')
# generate datasets
tst = ['1','2','3']
tst_ds = []

for t in tst:

    df_tst = hdfs.get('/x1/t'+t+'/trx_3_1')
    lst = df_tst.columns[df_tst.columns.str.contains('_ifft_')]
    
    #df_tst_cl,_ = distortion_filter(df_tst_cl)
    
    groups = get_trx_groups(df_tst)
    df_std = rf_grouped(df_tst, groups=groups, fn=rf_std_single, label='target')
    df_mean = rf_grouped(df_tst, groups=groups, fn=rf_mean_single)
    
    df_p2p = rf_grouped(df_tst, groups=groups, fn=rf_ptp_single) # added p2p feature
    
    df_all = pd.concat( [df_std, df_mean, df_p2p], axis=1 ) # added p2p feature
    
    df_all = cf_std_window(df_all, window=4, label='target')
        
    df_tst_sum = generate_class_label_presence(df_all, state_variable='target')
    
    # remove index column
    df_tst_sum = df_tst_sum[df_tst_sum.columns.values[~df_tst_sum.columns.str.contains('index')].tolist()]
    print('Columns in Dataset:',t)
    print(df_tst_sum.columns)
    

    tst_ds.append(df_tst_sum.copy())
# holdout validation
print(hold_out_val(tst_ds, target='target', include_self=False, cl='rf', verbose=False, random_state=1))
hdfs.close()<jupyter_output><empty_output><jupyter_text>Aufgabe 5: Eigener ErkennerIm Rahmen des eigenen Erkenners werden die entsprechenden Preprocessing und Mapping Schritte, anhand des originalen Datasets, erneut durchgeführt im Hinblick auf die Anpassung an den eigenen Erkenner.<jupyter_code># Load hdfs data
hdfs = pd.HDFStore("../../data/raw/henrik/TestMessungen_NEU.hdf")
# Check available keys in hdf5 store
hdfs.keys
# Step-0
# Mapping groundtruth to 0-empty and 1-not empty and prepare for further preprocessing by
# removing additional timestamp columns and index column
# Storing cleaned dataframes (no index, removed _ts columns, mapped multi classes to 0-empty, 1-not empty)
# to new hdfstore to `data/interim/henrik/02_testmessungen.hdf`

hdf_path = "../../data/interim/henrik/02_tesmessungen.hdf"

dfs = []
for key in hdfs.keys():
    df = hdfs.get(key)
    #df['target'] = df['target'].map(lambda x: 0 if x.startswith("Empty") else 1)  
    # drop all time stamp columns who endswith _ts
    cols = [c for c in df.columns if not c.lower().endswith("ts")]
    df = df[cols]
    df = df.drop('index', axis=1)
    df.to_hdf(hdf_path, key)
hdfs.close()
hdfs = pd.HDFStore(hdf_path)
df = hdfs.get("/x1/t1/trx_1_2")
df.head()
# Step-1 repeating the previous taks 4 to get a comparable base result with the now dropped _ts and index column to improve from
# generate datasets
from evaluation import *
from filters import *
from utility import *
from features import *


tst = ['1','2','3']
tst_ds = []

for t in tst:

    df_tst = hdfs.get('/x1/t'+t+'/trx_3_1')
    lst = df_tst.columns[df_tst.columns.str.contains('_ifft_')]
    
    #df_tst_cl,_ = distortion_filter(df_tst_cl)
    
    df_tst,_ = distortion_filter(df_tst)
    
    groups = get_trx_groups(df_tst)
    df_std = rf_grouped(df_tst, groups=groups, fn=rf_std_single, label='target')
    df_mean = rf_grouped(df_tst, groups=groups, fn=rf_mean_single)
    
    df_p2p = rf_grouped(df_tst, groups=groups, fn=rf_ptp_single) # added p2p feature
    
    df_kurt = rf_grouped(df_tst, groups=groups, fn=rf_kurtosis_single)
    
    df_all = pd.concat( [df_std, df_mean, df_p2p, df_kurt], axis=1 ) # added p2p feature
     
    df_all = cf_std_window(df_all, window=4, label='target')
    
    df_all = cf_diff(df_all, label='target')
    
    df_tst_sum = generate_class_label_presence(df_all, state_variable='target')
    
    # remove index column
    # df_tst_sum = df_tst_sum[df_tst_sum.columns.values[~df_tst_sum.columns.str.contains('index')].tolist()]
    print('Columns in Dataset:',t)
    print(df_tst_sum.columns)
    

    tst_ds.append(df_tst_sum.copy())
print(hold_out_val(tst_ds, target='target', include_self=False, cl='dt', verbose=False, random_state=1))
# Evaluating different supervised learning methods provided in eval.py
# added a NN evaluator but there are some problems regarding usage and hidden layers
# For the moment only kurtosis and cf_diff are added to the dataset as well as the distortion filter
# Feature selection is needed right now!
for elem in ['rf', 'dt', 'nb' ,'nn','knn']:
    print(hold_out_val(tst_ds, target='target', include_self=False, cl=elem, verbose=False, random_state=1))<jupyter_output>(0.86225141715993037, 0.049405788187083757)
(0.87454461167643449, 0.055507534644738628)
(0.69467597326646258, 0.037431406160581791)
(0.45908595917510514, 0.083621392018383839)
(0.87454461167643449, 0.055507534644738628)<jupyter_text>Aufgabe 6: Umwandlung in einen Onlineerkenner The following command starts a flask_restful server on localhost port:5444 which answers json post requests. The server is implemented in the file online.py within the ipynb folder and makes use of the final chosen model.Requests can be made as post request to http://localhost:5444/predict with a json file of the following format:{    "row": "features"}be careful that the sent file is valid json. The answer contains the row and the predicted class.{    "row": "features",    "p_class": "predicted class"}For now the online predictor only predicts the class of single sent lines<jupyter_code>%run -i './online.py'<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13007585899152163, 'fraction_numerical': 0.04506916555109326, 'mean_word_length': 4.813878080415045, 'pattern_counts': {'":': 3, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '410981', 'license': 'MIT', 'n_tokens_mistral': 3652, 'n_tokens_neox': 3466, 'n_words': 761, 'path': 'notebooks/henrik_ueb02/Mustererkennung_in_Funkmessdaten.ipynb', 'repo_name': 'hhain/sdap17', 'size': '8964'}