<jupyter_start><jupyter_text>---  Filtering and encoding<jupyter_code>aux = df[~df['zzz'].isna()]
aux = aux.drop(['source_address', 'header_checksum', 'destination_address', 'checksum',
                'source_addressmac', 'destination_addressmac', 'hex_stream'], axis=1)
aux['method'] = aux['zzz'].map(lambda s: np.nan if s.startswith('{') else 'POST')
aux['header'] = aux['zzz'].map(lambda s: np.nan if s.startswith('{') else s)
aux['payload'] = aux['zzz'].map(lambda s: np.nan if not s.startswith('{') else s)
aux['header'] = aux['header'].map(lambda s: s if not isinstance(s, str) else s[5:].split('\r\n'))
def payload_caster(j):
    # sometimes "temperatura" comes empty, which breaks json loading
    try:
        return j if not isinstance(j, str) else json.loads(j)
    except Exception as e:
        return {'temperatura': np.nan}
aux['payload'] = aux['payload'].map(payload_caster)
aux['header_endpoint'] = aux['header'].map(lambda s: s if not isinstance(s, list) else s[0].split()[0])
def get_user_agent(l):
    try:
        return l if not isinstance(l, list) else l[2].split()[1]
    except Exception as e:
        msg = f'error with {l}: {e}'
        print(msg)
        return msg
aux['header_user_agent'] = aux['header'].map(get_user_agent)
aux2 = aux[~aux['header_user_agent'].isna()]
aux3 = aux2[aux2['header_user_agent'].str.contains('error')]<jupyter_output><empty_output><jupyter_text>Removing access attempts from port scanners (this could be useful as 0s in final dataset, but will remove for now)<jupyter_code>aux = aux.drop(aux3.index)
set(aux.header_user_agent)
aux4 = aux[~aux['header_user_agent'].isna()]
aux5 = aux4[aux4['header_user_agent'].str.contains('Mozilla')]
aux5
aux = aux.drop(aux5.index)
set(aux.header_user_agent)
set(aux.header_endpoint)
def define_is_post(method):
    try:
        if np.isnan(method):
            return False
    except:
        pass
    
    return method == 'POST'
aux['is_post'] = aux['method'].map(define_is_post)
aux = aux.drop(['zzz', 'method'], axis=1)
aux.loc[aux.index[0], 'header']
def make_get_from_header(pos, key_name):
    def get_from_header(header):
        try:
            if np.isnan(header):
                return np.nan
            else:
                print('weird', header)
        except:
            pass

        header_data = header[pos]
        if not header_data.startswith(key_name):
            print('weird', header)

        return header_data.split(': ')[1]
    return get_from_header
aux['header_accept_encoding'] = aux['header'].map(make_get_from_header(3, 'Accept-Encoding'))
aux['header_connection'] = aux['header'].map(make_get_from_header(4, 'Connection'))
aux['header_content_type'] = aux['header'].map(make_get_from_header(5, 'Content-Type'))
def get_header(key, caster=str):
    def get_from_header(header):
        try:
            if np.isnan(header):
                return np.nan
            else:
                print('weird', header)
        except:
            pass

        header = {i.split(': ')[0]: i.split(': ')[1] for i in header[1:]}
        try:
            return caster(header[key])
        except KeyError:
            return np.nan

    return get_from_header
aux['header_content_length'] = aux['header'].map(get_header('Content-Length', int))
aux['header_authorization'] = aux['header'].map(get_header('Authorization'))
aux = aux.drop('header', axis=1)
aux2 = aux[~aux['payload'].isna()]
keys = {j for i in aux2['payload'] for j in i}
keys
def get_from_payload(key, caster=str):
    def get(payload):
        try:
            if np.isnan(payload):
                return np.nan
            else:
                print('weird', payload)
        except:
            pass
        
        try:
            return caster(payload[key])
        except KeyError:
            return np.nan

    return get
aux['payload_password'] = aux['payload'].map(get_from_payload('password'))
aux['payload_speed'] = aux['payload'].map(get_from_payload('speed', float))
aux['payload_temperatura'] = aux['payload'].map(get_from_payload('temperatura', float))
aux['payload_username'] = aux['payload'].map(get_from_payload('username'))
aux = aux.drop(['payload', 'payload_username', 'payload_password', 'header_authorization'], axis=1)
# model should not have access to any password information, so this should be useless
set(aux['header_endpoint'])
def make_define_endpoint(desired):
    def define_endpoint(endpoint):
        try:
            if np.isnan(endpoint):
                return False
            else:
                print('weird', endpoint)
        except:
            pass

        return endpoint.endswith(desired)
    return define_endpoint
aux['is_endpoint_signin'] = aux['header_endpoint'].map(make_define_endpoint('signin'))
aux['is_endpoint_message'] = aux['header_endpoint'].map(make_define_endpoint('message'))
aux = aux.drop('header_endpoint', axis=1)
set(aux['header_user_agent'])
def define_esp_user_agent(user_agent):
    def check():
        return user_agent == 'ESP8266HTTPClient'
    
    try:
        if np.isnan(user_agent):
            return False
        else:
            return check()
    except:
        return check()
aux['is_esp8266_user_agent'] = aux['header_user_agent'].map(define_esp_user_agent)
aux = aux.drop('header_user_agent', axis=1)
def make_encoder(default_value):
    def encoder(val):
        def check():
            return val == default_value
        
        try:
            if np.isnan(val):
                return False
            else:
                return check()
        except:
            return check()
    return encoder
aux['is_keep_alive'] = aux['header_connection'].map(make_encoder('keep-alive'))
aux['is_json'] = aux['header_content_type'].map(make_encoder('application/json'))
aux = aux.drop(['header_accept_encoding', 'header_connection', 'header_content_type'], axis=1)
aux['header_content_length'] = aux['header_content_length'].map(lambda d: 0 if np.isnan(d) else d)
aux['payload_speed'] = aux['payload_speed'].map(lambda d: -1 if np.isnan(d) else d)
aux['payload_temperatura'] = aux['payload_temperatura'].map(lambda d: 0 if np.isnan(d) else d)
set(aux.option3)
set(aux.window)
set(aux.option4)
set(aux.option1)
set(aux.option2)
aux = aux.drop(['option3', 'window', 'option4', 'option1', 'option2'], axis=1) # no variation in columns
set(aux.rst)
set(aux.data_offset)
set(aux.psh)
set(aux.fin)
set(aux.syn)
set(aux.reserved)
set(aux.ack)
aux = aux.drop(['rst', 'data_offset', 'psh', 'fin', 'syn', 'reserved', 'ack'], axis=1) # no variation
set(aux.option5)
set(aux.option6)
set(aux.flags_reserved)
set(aux.flags_dont_fragment)
set(aux.flags_more_fragment)
aux = aux.drop(['option5', 'option6', 'flags_reserved', 'flags_dont_fragment', 'flags_more_fragment'], axis=1)
set(aux.option1_length)
set(aux.option2_length)
set(aux.option3_length)
set(aux.option4_length)
set(aux.option5_length)
set(aux.option6_length)
aux = aux.drop(['option1_length', 'option2_length', 'option3_length',
                'option4_length', 'option5_length', 'option6_length'], axis=1)
def encode_booleans(val):
    return 1 if val else 0
aux['is_post'] = aux['is_post'].map(encode_booleans)
aux['is_endpoint_signin'] = aux['is_endpoint_signin'].map(encode_booleans)
aux['is_endpoint_message'] = aux['is_endpoint_message'].map(encode_booleans)
aux['is_esp8266_user_agent'] = aux['is_esp8266_user_agent'].map(encode_booleans)
aux['is_keep_alive'] = aux['is_keep_alive'].map(encode_booleans)
aux['is_json'] = aux['is_json'].map(encode_booleans)
aux
!ls
aux.to_parquet('preped_dataset.parquet')
def try_drop(df, *args, **kwargs):
    try:
        return df.drop(*args, **kwargs)
    except:
        return df
def filter_and_encode(df):
    df = df[~df['zzz'].isna()]
    df = df.drop(['source_address', 'header_checksum', 'destination_address', 'checksum',
                  'source_addressmac', 'destination_addressmac', 'hex_stream'], axis=1)
    df['method'] = df['zzz'].map(lambda s: np.nan if s.startswith('{') else 'POST')
    df['header'] = df['zzz'].map(lambda s: np.nan if s.startswith('{') else s)
    df['payload'] = df['zzz'].map(lambda s: np.nan if not s.startswith('{') else s)
    df['header'] = df['header'].map(lambda s: s if not isinstance(s, str) else s[5:].split('\r\n'))
    df['payload'] = df['payload'].map(payload_caster)
    df['header_endpoint'] = df['header'].map(lambda s: s if not isinstance(s, list) else s[0].split()[0])
    df['header_user_agent'] = df['header'].map(get_user_agent)
    aux2 = df[~df['header_user_agent'].isna()]
    aux3 = aux2[aux2['header_user_agent'].str.contains('error')]
    df = try_drop(df, aux3.index)
    aux4 = df[~df['header_user_agent'].isna()]
    aux5 = aux4[aux4['header_user_agent'].str.contains('Mozilla')]
    df = try_drop(df, aux5.index)
    df['is_post'] = df['method'].map(define_is_post)
    df = df.drop(['zzz', 'method'], axis=1)
    df['header_accept_encoding'] = df['header'].map(make_get_from_header(3, 'Accept-Encoding'))
    df['header_connection'] = df['header'].map(make_get_from_header(4, 'Connection'))
    df['header_content_type'] = df['header'].map(make_get_from_header(5, 'Content-Type'))
    df['header_content_length'] = df['header'].map(get_header('Content-Length', int))
    df['header_authorization'] = df['header'].map(get_header('Authorization'))
    df = df.drop('header', axis=1)
    df['payload_password'] = df['payload'].map(get_from_payload('password'))
    df['payload_speed'] = df['payload'].map(get_from_payload('speed', float))
    df['payload_temperatura'] = df['payload'].map(get_from_payload('temperatura', float))
    df['payload_username'] = df['payload'].map(get_from_payload('username'))
    df = df.drop(['payload', 'payload_username', 'payload_password', 'header_authorization'], axis=1)
    df['is_endpoint_signin'] = df['header_endpoint'].map(make_define_endpoint('signin'))
    df['is_endpoint_message'] = df['header_endpoint'].map(make_define_endpoint('message'))
    df = df.drop('header_endpoint', axis=1)
    df['is_esp8266_user_agent'] = df['header_user_agent'].map(define_esp_user_agent)
    df = df.drop('header_user_agent', axis=1)
    df['is_keep_alive'] = df['header_connection'].map(make_encoder('keep-alive'))
    df['is_json'] = df['header_content_type'].map(make_encoder('application/json'))
    df = df.drop(['header_accept_encoding', 'header_connection', 'header_content_type'], axis=1)
    df['header_content_length'] = df['header_content_length'].map(lambda d: 0 if np.isnan(d) else d)
    df['payload_speed'] = df['payload_speed'].map(lambda d: -1 if np.isnan(d) else d)
    df['payload_temperatura'] = df['payload_temperatura'].map(lambda d: 0 if np.isnan(d) else d)
    df = df.drop(['option3', 'window', 'option4', 'option1', 'option2'], axis=1)
    df = df.drop(['rst', 'data_offset', 'psh', 'fin', 'syn', 'reserved', 'ack'], axis=1)
    df = df.drop(['option5', 'option6', 'flags_reserved', 'flags_dont_fragment', 'flags_more_fragment'], axis=1)
    df = df.drop(['option1_length', 'option2_length', 'option3_length',
                  'option4_length', 'option5_length', 'option6_length'], axis=1)
    for i in ['is_post', 'is_endpoint_signin', 'is_endpoint_message',
              'is_esp8266_user_agent', 'is_keep_alive', 'is_json']:
        df[i] = df[i].map(encode_booleans)
        
    return df
df = pd.read_parquet('requests_python_parsed_df.parquet')
df = filter_and_encode(df)
df.to_parquet('requests_python_filtered_and_encoded.parquet')
df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16713116181849852, 'fraction_numerical': 0.011583347848806828, 'mean_word_length': 4.3359665427509295, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '196005', 'license': 'MIT', 'n_tokens_mistral': 3920, 'n_tokens_neox': 3875, 'n_words': 719, 'path': 'data_prep.ipynb', 'repo_name': 'LRAbbade/IoT_anomaly_detection', 'size': '11482'}