<commit_before>#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
# Copyright (c) 2016 Mozilla Corporation

# Imports auth0.com logs into MozDef

import hjson
import sys
import os
import requests
import traceback

import mozdef_client as mozdef

from mozdef_util.utilities.dot_dict import DotDict
from mozdef_util.utilities.logger import logger


def fatal(msg):
    print(msg)
    sys.exit(1)


# This is from https://auth0.com/docs/api/management/v2#!/Logs/get_logs
# auth0 calls these events with an acronym and description
# The logs have the acronym, but not the description
# but do include a 'description' field that is additional detailed words
# about what happened.
# See also: https://auth0.com/docs/logs#log-data-event-listing
# levels
#     0 = Debug
#     1 = Info
#     2 = Warning
#     3 = Error
#     4 = Critical
log_types = DotDict(
    {
        "s": {"event": "Success Login", "level": 1},
        "slo": {"event": "Success Logout", "level": 1},
        "flo": {"event": "Failed Logout", "level": 3},
        "seacft": {"event": "Success Exchange (Authorization Code for Access Token)", "level": 1},
        "feacft": {"event": "Failed Exchange (Authorization Code for Access Token)", "level": 3},
        "f": {"event": "Failed Login", "level": 3},
        "w": {"event": "Warnings During Login", "level": 2},
        "du": {"event": "Deleted User", "level": 1},
        "fu": {"event": "Failed Login (invalid email/username)", "level": 3},
        "fp": {"event": "Failed Login (wrong password)", "level": 3},
        "fc": {"event": "Failed by Connector", "level": 3},
        "fco": {"event": "Failed by CORS", "level": 3},
        "con": {"event": "Connector Online", "level": 1},
        "coff": {"event": "Connector Offline", "level": 3},
        "fcpro": {"event": "Failed Connector Provisioning", "level": 4},
        "ss": {"event": "Success Signup", "level": 1},
        "fs": {"event": "Failed Signup", "level": 3},
        "cs": {"event": "Code Sent", "level": 0},
        "cls": {"event": "Code/Link Sent", "level": 0},
        "sv": {"event": "Success Verification Email", "level": 0},
        "fv": {"event": "Failed Verification Email", "level": 0},
        "scp": {"event": "Success Change Password", "level": 1},
        "fcp": {"event": "Failed Change Password", "level": 3},
        "sce": {"event": "Success Change Email", "level": 1},
        "fce": {"event": "Failed Change Email", "level": 3},
        "scu": {"event": "Success Change Username", "level": 1},
        "fcu": {"event": "Failed Change Username", "level": 3},
        "scpn": {"event": "Success Change Phone Number", "level": 1},
        "fcpn": {"event": "Failed Change Phone Number", "level": 3},
        "svr": {"event": "Success Verification Email Request", "level": 0},
        "fvr": {"event": "Failed Verification Email Request", "level": 3},
        "scpr": {"event": "Success Change Password Request", "level": 0},
        "fcpr": {"event": "Failed Change Password Request", "level": 3},
        "fn": {"event": "Failed Sending Notification", "level": 3},
        "sapi": {"event": "API Operation", "level": 1},
        "fapi": {"event": "Failed API Operation", "level": 3},
        "limit_wc": {"event": "Blocked Account", "level": 4},
        "limit_ui": {"event": "Too Many Calls to /userinfo", "level": 4},
        "api_limit": {"event": "Rate Limit On API", "level": 4},
        "sdu": {"event": "Successful User Deletion", "level": 1},
        "fdu": {"event": "Failed User Deletion", "level": 3},
        "sd": {"event": "Success Delegation", "level": 3},
        "fd": {"event": "Failed Delegation", "level": 3},
        "seccft": {"event": "Success Exchange (Client Credentials for Access Token)", "level": 1},
        "feccft": {"event": "Failed Exchange (Client Credentials for Access Token)", "level": 1},
        "fsa": {"event": "Failed Silent Auth", "level": 3},
        "ssa": {"event": "Success Silent Auth", "level": 1},
        "fepft": {"event": "Failed Exchange (Password for Access Token)", "level": 3},
        "limit_mu": {"event": "Blocked IP Address", "level": 3},
        "sepft": {"event": "Success Exchange (Password for Access Token)", "level": 1},
        "fcoa": {"event": "Failed Cross Origin Authentication", "level": 3},
        "depnote": {"event": "Deprecation Note", "level": 1},
    }
)


def process_msg(mozmsg, msg):
    """Normalization function for auth0 msg.
    @mozmsg: MozDefEvent (mozdef message as DotDict)
    @msg: DotDict (json with auth0 raw message data).

    All the try-except loops handle cases where the auth0 msg may or may not contain expected fields.
    The msg structure is not garanteed.
    See also https://auth0.com/docs/api/management/v2#!/Logs/get_logs
    """
    details = DotDict({})

    # key words used to set category and success/failure markers
    authentication_words = ["Login", "Logout", "Auth"]
    authorization_words = ["Authorization", "Access", "Delegation"]
    success_words = ["Success"]
    failed_words = ["Failed"]

    # default category (might be modified below to be more specific)
    mozmsg.set_category("iam")
    mozmsg.source = "auth0"
    # fields that should always exist
    mozmsg.timestamp = msg.date
    details["messageid"] = msg._id
    details["sourceipaddress"] = msg.ip

    try:
        details["userid"] = msg.user_id
    except KeyError:
        pass

    try:
        details["username"] = msg.user_name
    except KeyError:
        pass

    try:
        # the details.request/response exist for api calls
        # but not for logins and other events
        # check and prefer them if present.
        details["username"] = msg.details.request.auth.user.name
        details["action"] = msg.details.response.body.name
    except KeyError:
        pass<commit_msg>do not blindly reference a sub-dict without checking the type first<commit_after>#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
# Copyright (c) 2016 Mozilla Corporation

# Imports auth0.com logs into MozDef

import hjson
import sys
import os
import requests
import traceback

import mozdef_client as mozdef

from mozdef_util.utilities.dot_dict import DotDict
from mozdef_util.utilities.logger import logger


def fatal(msg):
    print(msg)
    sys.exit(1)


# This is from https://auth0.com/docs/api/management/v2#!/Logs/get_logs
# auth0 calls these events with an acronym and description
# The logs have the acronym, but not the description
# but do include a 'description' field that is additional detailed words
# about what happened.
# See also: https://auth0.com/docs/logs#log-data-event-listing
# levels
#     0 = Debug
#     1 = Info
#     2 = Warning
#     3 = Error
#     4 = Critical
log_types = DotDict(
    {
        "s": {"event": "Success Login", "level": 1},
        "slo": {"event": "Success Logout", "level": 1},
        "flo": {"event": "Failed Logout", "level": 3},
        "seacft": {"event": "Success Exchange (Authorization Code for Access Token)", "level": 1},
        "feacft": {"event": "Failed Exchange (Authorization Code for Access Token)", "level": 3},
        "f": {"event": "Failed Login", "level": 3},
        "w": {"event": "Warnings During Login", "level": 2},
        "du": {"event": "Deleted User", "level": 1},
        "fu": {"event": "Failed Login (invalid email/username)", "level": 3},
        "fp": {"event": "Failed Login (wrong password)", "level": 3},
        "fc": {"event": "Failed by Connector", "level": 3},
        "fco": {"event": "Failed by CORS", "level": 3},
        "con": {"event": "Connector Online", "level": 1},
        "coff": {"event": "Connector Offline", "level": 3},
        "fcpro": {"event": "Failed Connector Provisioning", "level": 4},
        "ss": {"event": "Success Signup", "level": 1},
        "fs": {"event": "Failed Signup", "level": 3},
        "cs": {"event": "Code Sent", "level": 0},
        "cls": {"event": "Code/Link Sent", "level": 0},
        "sv": {"event": "Success Verification Email", "level": 0},
        "fv": {"event": "Failed Verification Email", "level": 0},
        "scp": {"event": "Success Change Password", "level": 1},
        "fcp": {"event": "Failed Change Password", "level": 3},
        "sce": {"event": "Success Change Email", "level": 1},
        "fce": {"event": "Failed Change Email", "level": 3},
        "scu": {"event": "Success Change Username", "level": 1},
        "fcu": {"event": "Failed Change Username", "level": 3},
        "scpn": {"event": "Success Change Phone Number", "level": 1},
        "fcpn": {"event": "Failed Change Phone Number", "level": 3},
        "svr": {"event": "Success Verification Email Request", "level": 0},
        "fvr": {"event": "Failed Verification Email Request", "level": 3},
        "scpr": {"event": "Success Change Password Request", "level": 0},
        "fcpr": {"event": "Failed Change Password Request", "level": 3},
        "fn": {"event": "Failed Sending Notification", "level": 3},
        "sapi": {"event": "API Operation", "level": 1},
        "fapi": {"event": "Failed API Operation", "level": 3},
        "limit_wc": {"event": "Blocked Account", "level": 4},
        "limit_ui": {"event": "Too Many Calls to /userinfo", "level": 4},
        "api_limit": {"event": "Rate Limit On API", "level": 4},
        "sdu": {"event": "Successful User Deletion", "level": 1},
        "fdu": {"event": "Failed User Deletion", "level": 3},
        "sd": {"event": "Success Delegation", "level": 3},
        "fd": {"event": "Failed Delegation", "level": 3},
        "seccft": {"event": "Success Exchange (Client Credentials for Access Token)", "level": 1},
        "feccft": {"event": "Failed Exchange (Client Credentials for Access Token)", "level": 1},
        "fsa": {"event": "Failed Silent Auth", "level": 3},
        "ssa": {"event": "Success Silent Auth", "level": 1},
        "fepft": {"event": "Failed Exchange (Password for Access Token)", "level": 3},
        "limit_mu": {"event": "Blocked IP Address", "level": 3},
        "sepft": {"event": "Success Exchange (Password for Access Token)", "level": 1},
        "fcoa": {"event": "Failed Cross Origin Authentication", "level": 3},
        "depnote": {"event": "Deprecation Note", "level": 1},
    }
)


def process_msg(mozmsg, msg):
    """Normalization function for auth0 msg.
    @mozmsg: MozDefEvent (mozdef message as DotDict)
    @msg: DotDict (json with auth0 raw message data).

    All the try-except loops handle cases where the auth0 msg may or may not contain expected fields.
    The msg structure is not garanteed.
    See also https://auth0.com/docs/api/management/v2#!/Logs/get_logs
    """
    details = DotDict({})

    # key words used to set category and success/failure markers
    authentication_words = ["Login", "Logout", "Auth"]
    authorization_words = ["Authorization", "Access", "Delegation"]
    success_words = ["Success"]
    failed_words = ["Failed"]

    # default category (might be modified below to be more specific)
    mozmsg.set_category("iam")
    mozmsg.source = "auth0"
    # fields that should always exist
    mozmsg.timestamp = msg.date
    details["messageid"] = msg._id
    details["sourceipaddress"] = msg.ip

    try:
        details["userid"] = msg.user_id
    except KeyError:
        pass

    try:
        details["username"] = msg.user_name
    except KeyError:
        pass

    try:
        # the details.request/response exist for api calls
        # but not for logins and other events
        # check and prefer them if present.
        details["username"] = msg.details.request.auth.user.name
        if type(msg.details.response.body) is not list:
            details["action"] = msg.details.response.body.name
    except KeyError:
        pass
{'fraction_non_alphanumeric': 0.17225078785868303, 'fraction_numerical': 0.012771603914413667, 'mean_word_length': 3.2193841847445768, 'pattern_counts': {'":': 312, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7016981', 'n_tokens_mistral': 3793, 'n_tokens_neox': 3602, 'n_words': 1294}