<commit_before>"""Mission evaluation."""

import itertools
import logging
from auvsi_suas.models.map import Map
from auvsi_suas.models.mission_judge_feedback import MissionJudgeFeedback
from auvsi_suas.models.odlc import Odlc
from auvsi_suas.models.odlc import OdlcEvaluator
from auvsi_suas.models.takeoff_or_landing_event import TakeoffOrLandingEvent
from auvsi_suas.models.uas_telemetry import UasTelemetry
from auvsi_suas.proto import interop_admin_api_pb2
from auvsi_suas.proto import interop_api_pb2
from django.contrib.auth.models import User

logger = logging.getLogger(__name__)

# The time between interop telemetry posts that's a prereq for other tasks.
INTEROP_TELEM_THRESHOLD_TIME_SEC = 1.0

# Weight of timeline points for mission time.
MISSION_TIME_WEIGHT = 0.8
# Weight of timeline points for not taking a timeout.
TIMEOUT_WEIGHT = 0.2
# Minimum amount of flight time without losing points.
FLIGHT_TIME_MIN_SEC = 20 * 60
# Maximum amount of flight time allowed.
FLIGHT_TIME_MAX_SEC = 30 * 60
# Maximum amount of post process time allowed.
PROCESS_TIME_MAX_SEC = 10 * 60
# Mission time points lost due for every second over max time.
MISSION_TIME_PENALTY_FROM_SEC = 0.03
# Weighting of flight time relative to post process time.
FLIGHT_TO_PROCESS_TIME_WEIGHT = 5.0

# Ratio of points lost per takeover.
AUTONOMOUS_FLIGHT_TAKEOVER = 0.10
# Ratio of points lost per out of bounds.
BOUND_PENALTY = 0.1
SAFETY_BOUND_PENALTY = 1.0
# Ratio of points lost for TFOA and crash.
TFOA_PENALTY = 0.25
CRASH_PENALTY = 0.50
# Weight of accuracy points to all autonomous flight.
WAYPOINT_ACCURACY_WEIGHT = 1.0

# Map quality to point ratio.
MAP_QUALITY_POINT_RATIO = {
    interop_admin_api_pb2.MapEvaluation.MapQuality.INSUFFICIENT: 0,
    interop_admin_api_pb2.MapEvaluation.MapQuality.MEDIUM: 0.5,
    interop_admin_api_pb2.MapEvaluation.MapQuality.HIGH: 1.0,
}

# Weight of air drop points for accuracy.
AIR_DROP_ACCURACY_WEIGHT = 0.5
# Weight of air drop points for UGV driving to location.
AIR_DROP_DRIVE_WEIGHT = 0.5
# Air drop to point ratio.
AIR_DROP_DISTANCE_POINT_RATIO = {
    interop_admin_api_pb2.MissionJudgeFeedback.NO_POINTS: 0,
    interop_admin_api_pb2.MissionJudgeFeedback.WITHIN_05_FT: 1,
    interop_admin_api_pb2.MissionJudgeFeedback.WITHIN_15_FT: 0.5,
    interop_admin_api_pb2.MissionJudgeFeedback.WITHIN_40_FT: 0.25,
}

# Scoring weights.
TIMELINE_WEIGHT = 0.1
AUTONOMOUS_WEIGHT = 0.2
OBSTACLE_WEIGHT = 0.1
OBJECT_WEIGHT = 0.2
MAP_WEIGHT = 0.1
AIR_DROP_WEIGHT = 0.2
OPERATIONAL_WEIGHT = 0.1


def generate_feedback(mission_config, user, team_eval):
    """Generates mission feedback for the given team and mission.

    Args:
        mission_config: The mission to evaluate the team against.
        user: The team user object for which to evaluate and provide feedback.
        team_eval: The team evaluation to fill.
    """
    feedback = team_eval.feedback

    # Find the user's flights.
    flight_periods = TakeoffOrLandingEvent.flights(mission_config, user)
    for period in flight_periods:
        if period.duration() is None:
            team_eval.warnings.append(
                'Infinite flight period, may be missing TakeoffOrLandingEvent.'
            )
            break
    uas_period_logs = [
        list(UasTelemetry.dedupe(logs))
        for logs in UasTelemetry.by_time_period(user, flight_periods)
    ]
    uas_logs = list(itertools.chain.from_iterable(uas_period_logs))

    # Determine interop telemetry rates.
    telem_max, telem_avg = UasTelemetry.rates(user,
                                              flight_periods,
                                              time_period_logs=uas_period_logs)
    if telem_max:
        feedback.uas_telemetry_time_max_sec = telem_max
    if telem_avg:
        feedback.uas_telemetry_time_avg_sec = telem_avg

    # Determine if the uas hit the waypoints.
    feedback.waypoints.extend(
        UasTelemetry.satisfied_waypoints(
            mission_config.home_pos,
            mission_config.mission_waypoints.order_by('order'), uas_logs))

    # Evaluate the object detections.
    user_odlcs = Odlc.objects.filter(user=user).filter(
        mission=mission_config.pk).all()
    for odlc in user_odlcs:
        if odlc.thumbnail and odlc.thumbnail_approved is None:
            team_eval.warnings.append(
                'Odlc thumbnail review not set, may need to review ODLCs.')
            break
    evaluator = OdlcEvaluator(user_odlcs, mission_config.odlcs.all(),
                              flight_periods)
    feedback.odlc.CopyFrom(evaluator.evaluate())

    # Add map feedback if it exists.
    try:
        m = Map.objects.get(mission_id=mission_config.pk, user=user)
        feedback.map.quality = m.quality
    except Map.DoesNotExist:
        pass<commit_msg>Check for unreviewed maps and add warning. (#529)<commit_after>"""Mission evaluation."""

import itertools
import logging
from auvsi_suas.models.map import Map
from auvsi_suas.models.mission_judge_feedback import MissionJudgeFeedback
from auvsi_suas.models.odlc import Odlc
from auvsi_suas.models.odlc import OdlcEvaluator
from auvsi_suas.models.takeoff_or_landing_event import TakeoffOrLandingEvent
from auvsi_suas.models.uas_telemetry import UasTelemetry
from auvsi_suas.proto import interop_admin_api_pb2
from auvsi_suas.proto import interop_api_pb2
from django.contrib.auth.models import User

logger = logging.getLogger(__name__)

# The time between interop telemetry posts that's a prereq for other tasks.
INTEROP_TELEM_THRESHOLD_TIME_SEC = 1.0

# Weight of timeline points for mission time.
MISSION_TIME_WEIGHT = 0.8
# Weight of timeline points for not taking a timeout.
TIMEOUT_WEIGHT = 0.2
# Minimum amount of flight time without losing points.
FLIGHT_TIME_MIN_SEC = 20 * 60
# Maximum amount of flight time allowed.
FLIGHT_TIME_MAX_SEC = 30 * 60
# Maximum amount of post process time allowed.
PROCESS_TIME_MAX_SEC = 10 * 60
# Mission time points lost due for every second over max time.
MISSION_TIME_PENALTY_FROM_SEC = 0.03
# Weighting of flight time relative to post process time.
FLIGHT_TO_PROCESS_TIME_WEIGHT = 5.0

# Ratio of points lost per takeover.
AUTONOMOUS_FLIGHT_TAKEOVER = 0.10
# Ratio of points lost per out of bounds.
BOUND_PENALTY = 0.1
SAFETY_BOUND_PENALTY = 1.0
# Ratio of points lost for TFOA and crash.
TFOA_PENALTY = 0.25
CRASH_PENALTY = 0.50
# Weight of accuracy points to all autonomous flight.
WAYPOINT_ACCURACY_WEIGHT = 1.0

# Map quality to point ratio.
MAP_QUALITY_POINT_RATIO = {
    interop_admin_api_pb2.MapEvaluation.MapQuality.INSUFFICIENT: 0,
    interop_admin_api_pb2.MapEvaluation.MapQuality.MEDIUM: 0.5,
    interop_admin_api_pb2.MapEvaluation.MapQuality.HIGH: 1.0,
}

# Weight of air drop points for accuracy.
AIR_DROP_ACCURACY_WEIGHT = 0.5
# Weight of air drop points for UGV driving to location.
AIR_DROP_DRIVE_WEIGHT = 0.5
# Air drop to point ratio.
AIR_DROP_DISTANCE_POINT_RATIO = {
    interop_admin_api_pb2.MissionJudgeFeedback.NO_POINTS: 0,
    interop_admin_api_pb2.MissionJudgeFeedback.WITHIN_05_FT: 1,
    interop_admin_api_pb2.MissionJudgeFeedback.WITHIN_15_FT: 0.5,
    interop_admin_api_pb2.MissionJudgeFeedback.WITHIN_40_FT: 0.25,
}

# Scoring weights.
TIMELINE_WEIGHT = 0.1
AUTONOMOUS_WEIGHT = 0.2
OBSTACLE_WEIGHT = 0.1
OBJECT_WEIGHT = 0.2
MAP_WEIGHT = 0.1
AIR_DROP_WEIGHT = 0.2
OPERATIONAL_WEIGHT = 0.1


def generate_feedback(mission_config, user, team_eval):
    """Generates mission feedback for the given team and mission.

    Args:
        mission_config: The mission to evaluate the team against.
        user: The team user object for which to evaluate and provide feedback.
        team_eval: The team evaluation to fill.
    """
    feedback = team_eval.feedback

    # Find the user's flights.
    flight_periods = TakeoffOrLandingEvent.flights(mission_config, user)
    for period in flight_periods:
        if period.duration() is None:
            team_eval.warnings.append(
                'Infinite flight period, may be missing TakeoffOrLandingEvent.'
            )
            break
    uas_period_logs = [
        list(UasTelemetry.dedupe(logs))
        for logs in UasTelemetry.by_time_period(user, flight_periods)
    ]
    uas_logs = list(itertools.chain.from_iterable(uas_period_logs))

    # Determine interop telemetry rates.
    telem_max, telem_avg = UasTelemetry.rates(user,
                                              flight_periods,
                                              time_period_logs=uas_period_logs)
    if telem_max:
        feedback.uas_telemetry_time_max_sec = telem_max
    if telem_avg:
        feedback.uas_telemetry_time_avg_sec = telem_avg

    # Determine if the uas hit the waypoints.
    feedback.waypoints.extend(
        UasTelemetry.satisfied_waypoints(
            mission_config.home_pos,
            mission_config.mission_waypoints.order_by('order'), uas_logs))

    # Evaluate the object detections.
    user_odlcs = Odlc.objects.filter(user=user).filter(
        mission=mission_config.pk).all()
    for odlc in user_odlcs:
        if odlc.thumbnail and odlc.thumbnail_approved is None:
            team_eval.warnings.append(
                'Odlc thumbnail review not set, may need to review ODLCs.')
            break
    evaluator = OdlcEvaluator(user_odlcs, mission_config.odlcs.all(),
                              flight_periods)
    feedback.odlc.CopyFrom(evaluator.evaluate())

    # Add map feedback if it exists.
    try:
        m = Map.objects.get(mission_id=mission_config.pk, user=user)
        if m.quality is None:
            team_eval.warnings.append(
                'Map review not set, may need to review maps.')
        else:
            feedback.map.quality = m.quality
    except Map.DoesNotExist:
        pass
{'fraction_non_alphanumeric': 0.06524196034110757, 'fraction_numerical': 0.017363608342751465, 'mean_word_length': 3.3689407540394973, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5099240', 'n_tokens_mistral': 3400, 'n_tokens_neox': 3211, 'n_words': 801}