<jupyter_start><jupyter_text>MAPM: Microsoft Azure Predictive Maintenance<jupyter_code>import os
import tqdm
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages<jupyter_output><empty_output><jupyter_text>Overview of Raw Data Error logs (hourly):- encountered by the machines while in operating condition.- don't shut down the machines, not considered as failures.<jupyter_code>pd.read_csv('../datasets/MAPM/PdM_errors.csv.gz')
pd.read_csv('../datasets/MAPM/PdM_failures.csv.gz')
pd.read_csv('../datasets/MAPM/PdM_machines.csv.gz')
pd.read_csv('../datasets/MAPM/PdM_maint.csv.gz')<jupyter_output><empty_output><jupyter_text>Telemetry Time Series Data:  - Hourly average of voltage, rotation, pressure, vibration- Collected from 100 machines for the year 2015<jupyter_code>pd.read_csv('../datasets/MAPM/PdM_telemetry.csv.gz')
def get_sequence_and_events(machine_id=1, error_id=None, comp_id=None,
                             start_date=None, end_date=None):

    sensors = pd.read_csv('../datasets/MAPM/PdM_telemetry.csv.gz')
    sensors = sensors[sensors.machineID == machine_id].reset_index(drop=True)
    # print(sensors.shape)
    errors = pd.read_csv('../datasets/MAPM/PdM_errors.csv.gz')
    errors = errors[errors.machineID == machine_id].reset_index(drop=True)
    # print(errors.shape)
    failures = pd.read_csv('../datasets/MAPM/PdM_failures.csv.gz')
    failures = failures[failures.machineID == machine_id]
    # print(failures.shape)
    data = sensors.merge(errors, how='left', on='datetime').merge(
        failures, how='left', on='datetime')

    data.datetime = pd.to_datetime(data.datetime)
    data = data.set_index('datetime')

    if start_date is not None:
        data = data[start_date:]
    if end_date is not None:
        data = data[:end_date]

    return data
def plot_sequence_and_events(data, machine_id=1):

    data = data[data.machineID == machine_id]
    fig, ax = plt.subplots(4 + 2, figsize=(8, 8))

    data.plot(y='volt', legend=True, ax=ax[0])
    data.plot(y='rotate', legend=True, ax=ax[1])
    data.plot(y='pressure', legend=True, ax=ax[2])
    data.plot(y='vibration', legend=True, ax=ax[3])

    if data.errorID.isnull().sum() < data.errorID.shape[0]:
        pd.get_dummies(data.errorID).plot(ax=ax[4])
    if data.failure.isnull().sum() < data.failure.shape[0]:
        pd.get_dummies(data.failure).plot(ax=ax[5])

    ax[0].set_title('Machine #{}'.format(machine_id))

    for i in range(5):
        ax[i].set_xlabel(None)
        ax[i].set_xticklabels([])

    fig.tight_layout()

    return fig, ax
def gen_summary(outdir='../out/'):
    os.makedirs(outdir, exist_ok=True)
    df = load_data('../datasets/mapm/')
    with PdfPages(outdir + '/mapm_summary.pdf') as pp:
        for i in tqdm.trange(1, 101):
            fig, _ = plot_sequence_and_events(df, machine_id=i)
            fig.savefig(pp, format='pdf')
            plt.clf()
            plt.close()

gen_summary()
# Plot for a single machine
plot_sequence_and_events()<jupyter_output><empty_output><jupyter_text>Dataset Making<jupyter_code>def load_data(fp):
    # Sensor data
    data = pd.read_csv(fp + '/PdM_telemetry.csv.gz')

    # Error alarm logs
    data = data.merge(
        pd.read_csv(fp + '/PdM_errors.csv.gz'),
        how='left', on=['datetime', 'machineID'])

    # Failure logs
    data = data.merge(
        pd.read_csv(fp + '/PdM_failures.csv.gz'),
        how='left', on=['datetime', 'machineID'])
    
    # Formatting
    data.datetime = pd.to_datetime(data.datetime)

    return data
def cleaning(df):

    # NaN values are encoded to -1
    df = df.sort_values('errorID')
    df.errorID = df.errorID.factorize()[0]
    df = df.sort_values('failure')
    df.failure = df.failure.factorize()[0]
    df = df.sort_values(['machineID', 'datetime'])

    df.errorID = df.errorID.astype('category')
    df.failure = df.failure.astype('category')

    df.volt = df.volt.astype('float32')
    df.rotate = df.rotate.astype('float32')
    df.pressure = df.pressure.astype('float32')
    df.vibration = df.vibration.astype('float32')

    df.datetime = pd.to_datetime(df.datetime)
    return df
df = load_data('../datasets/mapm/')
df = cleaning(df)
# df.errorID = df.errorID.sort_values().factorize()[0]
# df.failure = df.failure.sort_values().factorize()[0]
# df = df.sort_values(['machineID', 'datetime'])
df.dtypes
df = load_data('../datasets/mapm/')
df = df.sort_values('errorID')
df.errorID = df.errorID.factorize()[0]
df = df.sort_values('failure')
df.failure = df.failure.factorize()[0]
df = df.sort_values(['machineID', 'datetime'])
df
df_m = [g for _, g in df.groupby('machineID')]
df_m[1]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15734637465447587, 'fraction_numerical': 0.009143100148841165, 'mean_word_length': 3.8246153846153845, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '190133', 'license': 'MIT', 'n_tokens_mistral': 1731, 'n_tokens_neox': 1683, 'n_words': 288, 'path': 'notebooks/view_mapm.ipynb', 'repo_name': 'kokikwbt/predictive-maintenance', 'size': '4703'}