<commit_before>package org.mwc.debrief.dis.core;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.mwc.debrief.dis.listeners.IDISCollisionListener;
import org.mwc.debrief.dis.listeners.IDISDetonationListener;
import org.mwc.debrief.dis.listeners.IDISEventListener;
import org.mwc.debrief.dis.listeners.IDISFireListener;
import org.mwc.debrief.dis.listeners.IDISFixListener;
import org.mwc.debrief.dis.listeners.IDISGeneralPDUListener;
import org.mwc.debrief.dis.listeners.IDISScenarioListener;
import org.mwc.debrief.dis.listeners.IDISStartResumeListener;
import org.mwc.debrief.dis.listeners.IDISStopListener;
import org.mwc.debrief.dis.providers.IPDUProvider;

import edu.nps.moves.dis.CollisionPdu;
import edu.nps.moves.dis.DetonationPdu;
import edu.nps.moves.dis.EntityStatePdu;
import edu.nps.moves.dis.EntityType;
import edu.nps.moves.dis.EventReportPdu;
import edu.nps.moves.dis.FirePdu;
import edu.nps.moves.dis.OneByteChunk;
import edu.nps.moves.dis.Orientation;
import edu.nps.moves.dis.Pdu;
import edu.nps.moves.dis.StartResumePdu;
import edu.nps.moves.dis.StopFreezePdu;
import edu.nps.moves.dis.VariableDatum;
import edu.nps.moves.dis.Vector3Double;
import edu.nps.moves.dis.Vector3Float;
import edu.nps.moves.disutil.CoordinateConversions;

public class DISModule implements IDISModule, IDISGeneralPDUListener
{
  private static final int ESTIMATED_KIND = 1;
  private List<IDISFixListener> _fixListeners =
      new ArrayList<IDISFixListener>();
  private Map<Integer, List<IDISEventListener>> _eventListeners =
      new HashMap<Integer, List<IDISEventListener>>();
  private List<IDISDetonationListener> _detonationListeners =
      new ArrayList<IDISDetonationListener>();
  private List<IDISGeneralPDUListener> _generalListeners =
      new ArrayList<IDISGeneralPDUListener>();
  private List<IDISScenarioListener> _scenarioListeners =
      new ArrayList<IDISScenarioListener>();
  private List<IDISStopListener> _stopListeners =
      new ArrayList<IDISStopListener>();
  private List<IDISCollisionListener> _collisionListeners =
      new ArrayList<IDISCollisionListener>();
  private List<IDISStartResumeListener> _startResumeListeners =
      new ArrayList<IDISStartResumeListener>();
  private boolean _newStart = false;
  private List<IDISFireListener> _fireListeners =
      new ArrayList<IDISFireListener>();

  private Map<Integer, String> _entityNames = new HashMap<Integer, String>();
  final private IDISEventListener _nameListener;

  public DISModule()
  {
    // SPECIAL PROCESSING - declare ourselves as the first event listener
    // so we can intercept launch events (and retrieve the name)
    _nameListener = new IDISEventListener()
    {

      @Override
      public void add(long time, short exerciseId, long senderId,
          String hisName, int eventType, String message)
      {
        if (eventType == IDISEventListener.EVENT_LAUNCH)
        {
          if (_entityNames.get(senderId) == null)
          {
            // ok, extract the message
            String name = extractNameFor(message);

            // did we manage it?
            if (name != null)
            {
              _entityNames.put((int) senderId, name);
            }
          }
        }
      }
    };
  }

  protected String extractNameFor(String message)
  {
    // Entity 1 called SubmarineSouth has been created or launched.
    final String called = "called ";
    final String has = "has been";

    String res = null;

    if (message.contains(called) && message.contains(has))
    {
      int nameStart = message.indexOf(called) + called.length();
      int nameEnd = message.indexOf(has) - 1;
      res = message.substring(nameStart, nameEnd);
    }
    return res;
  }

  /**
   * retrieve the name for this entity id, or generate a default one
   * 
   * @param id
   *          the id we're looking against
   * @return its name (or a generated one)
   */
  private String nameFor(long id)
  {
    String name = _entityNames.get((Integer) (int) id);
    if (name == null)
    {
      name = "DIS_" + id;
    }

    return name;
  }

  @Override
  public void addFixListener(IDISFixListener handler)
  {
    _fixListeners.add(handler);
  }

  @Override
  public void addCollisionListener(IDISCollisionListener handler)
  {
    _collisionListeners.add(handler);
  }

  @Override
  public void addEventListener(IDISEventListener handler)
  {
    addEventListener(handler, null);
  }

  @Override
  public void addStartResumeListener(IDISStartResumeListener handler)
  {
    _startResumeListeners.add(handler);
  }

  @Override
  public void addEventListener(IDISEventListener handler, Integer eType)
  {
    List<IDISEventListener> matches = _eventListeners.get(null);
    if (matches == null)
    {
      matches = new ArrayList<IDISEventListener>();
      _eventListeners.put(null, matches);
    }
    matches.add(handler);
  }

  @Override
  public void addFireListener(IDISFireListener handler)
  {
    _fireListeners.add(handler);
  }

  @Override
  public void addDetonationListener(IDISDetonationListener handler)
  {
    _detonationListeners.add(handler);
  }

  @Override
  public void addScenarioListener(IDISScenarioListener handler)
  {
    _scenarioListeners.add(handler);
  }

  @Override
  public void setProvider(IPDUProvider provider)
  {
    // remember we're restarting
    _newStart = true;

    // register as a listener, to hear about new data
    provider.addListener(this);
  }

  private void handleFix(EntityStatePdu pdu)
  {
    // unpack the data
    final short eid = pdu.getExerciseID();
    final short force = pdu.getForceId();
    final long hisId = pdu.getEntityID().getEntity();

    boolean isEstimated = pdu.getEntityType().getEntityKind() == ESTIMATED_KIND;

    long time = convertTime(pdu.getTimestamp());
    Vector3Double loc = pdu.getEntityLocation();
    double[] locArr = new double[]
    {loc.getX(), loc.getY(), loc.getZ()};
    double[] worldCoords = CoordinateConversions.xyzToLatLonDegrees(locArr);
    Orientation orientation = pdu.getEntityOrientation();
    Vector3Float velocity = pdu.getEntityLinearVelocity();

    double speedMs =
        Math.sqrt(velocity.getX() * velocity.getX() + velocity.getY()
            * velocity.getY());
    
    // entity state
    String hisName = nameFor(hisId);

    EntityType cat = pdu.getEntityType();
    short kind = cat.getEntityKind();
    short domain = cat.getDomain();
    short category = cat.getCategory();

    Iterator<IDISFixListener> fIter = _fixListeners.iterator();
    while (fIter.hasNext())
    {
      IDISFixListener thisF = (IDISFixListener) fIter.next();
      thisF.add(time, eid, hisId, hisName, force, kind, domain, category,
          isEstimated, worldCoords[0], worldCoords[1], -worldCoords[2],
          orientation.getPhi(), speedMs, pdu.getEntityAppearance_damage());
    }
  }

  private void handleEvent(EventReportPdu pdu)
  {
    short eid = pdu.getExerciseID();
    long time = convertTime(pdu.getTimestamp());
    int originator = pdu.getOriginatingEntityID().getEntity();
    int eType = (int) pdu.getEventType();
    String msg = "";

    // try to get the data
    List<VariableDatum> items = pdu.getVariableDatums();
    for (int i = 0; i < items.size(); i++)
    {
      VariableDatum val = items.get(i);
      List<OneByteChunk> chunks = val.getVariableData();
      final int thisLen = (int) val.getVariableDatumLength();
      byte[] bytes = new byte[thisLen];
      Iterator<OneByteChunk> iter = chunks.iterator();
      int ctr = 0;
      for (int l = 0; l < thisLen; l++)
      {
        OneByteChunk thisB = (OneByteChunk) iter.next();
        final byte thisByte = thisB.getOtherParameters()[0];

        // if (eType == 10004)
        // {
        // System.out.println("byte:" + thisByte + " str:" + new String(new byte[]{thisByte}));
        // }

        if (thisByte > 10)
        {
          bytes[ctr++] = thisByte;
        }
        else if (thisByte == 1 || thisByte == 9)
        {
          bytes[ctr++] = 32;
        }
      }
      String newS = new String(bytes);
      msg += newS;
    }

    if (msg.length() == 0)
    {
      msg = "Unset";
    }

    // sort out his name
    _nameListener.add(time, eid, originator, null, eType, msg);

    // now try to retrieve name
    String hisName = nameFor(originator);

    // first send out to specific listeners
    List<IDISEventListener> specificListeners = _eventListeners.get(eType);
    if (specificListeners != null)
    {
      Iterator<IDISEventListener> eIter = specificListeners.iterator();
      while (eIter.hasNext())<commit_msg>trim the output, to remove nulls<commit_after>package org.mwc.debrief.dis.core;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.mwc.debrief.dis.listeners.IDISCollisionListener;
import org.mwc.debrief.dis.listeners.IDISDetonationListener;
import org.mwc.debrief.dis.listeners.IDISEventListener;
import org.mwc.debrief.dis.listeners.IDISFireListener;
import org.mwc.debrief.dis.listeners.IDISFixListener;
import org.mwc.debrief.dis.listeners.IDISGeneralPDUListener;
import org.mwc.debrief.dis.listeners.IDISScenarioListener;
import org.mwc.debrief.dis.listeners.IDISStartResumeListener;
import org.mwc.debrief.dis.listeners.IDISStopListener;
import org.mwc.debrief.dis.providers.IPDUProvider;

import edu.nps.moves.dis.CollisionPdu;
import edu.nps.moves.dis.DetonationPdu;
import edu.nps.moves.dis.EntityStatePdu;
import edu.nps.moves.dis.EntityType;
import edu.nps.moves.dis.EventReportPdu;
import edu.nps.moves.dis.FirePdu;
import edu.nps.moves.dis.OneByteChunk;
import edu.nps.moves.dis.Orientation;
import edu.nps.moves.dis.Pdu;
import edu.nps.moves.dis.StartResumePdu;
import edu.nps.moves.dis.StopFreezePdu;
import edu.nps.moves.dis.VariableDatum;
import edu.nps.moves.dis.Vector3Double;
import edu.nps.moves.dis.Vector3Float;
import edu.nps.moves.disutil.CoordinateConversions;

public class DISModule implements IDISModule, IDISGeneralPDUListener
{
  private static final int ESTIMATED_KIND = 1;
  private List<IDISFixListener> _fixListeners =
      new ArrayList<IDISFixListener>();
  private Map<Integer, List<IDISEventListener>> _eventListeners =
      new HashMap<Integer, List<IDISEventListener>>();
  private List<IDISDetonationListener> _detonationListeners =
      new ArrayList<IDISDetonationListener>();
  private List<IDISGeneralPDUListener> _generalListeners =
      new ArrayList<IDISGeneralPDUListener>();
  private List<IDISScenarioListener> _scenarioListeners =
      new ArrayList<IDISScenarioListener>();
  private List<IDISStopListener> _stopListeners =
      new ArrayList<IDISStopListener>();
  private List<IDISCollisionListener> _collisionListeners =
      new ArrayList<IDISCollisionListener>();
  private List<IDISStartResumeListener> _startResumeListeners =
      new ArrayList<IDISStartResumeListener>();
  private boolean _newStart = false;
  private List<IDISFireListener> _fireListeners =
      new ArrayList<IDISFireListener>();

  private Map<Integer, String> _entityNames = new HashMap<Integer, String>();
  final private IDISEventListener _nameListener;

  public DISModule()
  {
    // SPECIAL PROCESSING - declare ourselves as the first event listener
    // so we can intercept launch events (and retrieve the name)
    _nameListener = new IDISEventListener()
    {

      @Override
      public void add(long time, short exerciseId, long senderId,
          String hisName, int eventType, String message)
      {
        if (eventType == IDISEventListener.EVENT_LAUNCH)
        {
          if (_entityNames.get(senderId) == null)
          {
            // ok, extract the message
            String name = extractNameFor(message);

            // did we manage it?
            if (name != null)
            {
              _entityNames.put((int) senderId, name);
            }
          }
        }
      }
    };
  }

  protected String extractNameFor(String message)
  {
    // Entity 1 called SubmarineSouth has been created or launched.
    final String called = "called ";
    final String has = "has been";

    String res = null;

    if (message.contains(called) && message.contains(has))
    {
      int nameStart = message.indexOf(called) + called.length();
      int nameEnd = message.indexOf(has) - 1;
      res = message.substring(nameStart, nameEnd);
    }
    return res;
  }

  /**
   * retrieve the name for this entity id, or generate a default one
   * 
   * @param id
   *          the id we're looking against
   * @return its name (or a generated one)
   */
  private String nameFor(long id)
  {
    String name = _entityNames.get((Integer) (int) id);
    if (name == null)
    {
      name = "DIS_" + id;
    }

    return name;
  }

  @Override
  public void addFixListener(IDISFixListener handler)
  {
    _fixListeners.add(handler);
  }

  @Override
  public void addCollisionListener(IDISCollisionListener handler)
  {
    _collisionListeners.add(handler);
  }

  @Override
  public void addEventListener(IDISEventListener handler)
  {
    addEventListener(handler, null);
  }

  @Override
  public void addStartResumeListener(IDISStartResumeListener handler)
  {
    _startResumeListeners.add(handler);
  }

  @Override
  public void addEventListener(IDISEventListener handler, Integer eType)
  {
    List<IDISEventListener> matches = _eventListeners.get(null);
    if (matches == null)
    {
      matches = new ArrayList<IDISEventListener>();
      _eventListeners.put(null, matches);
    }
    matches.add(handler);
  }

  @Override
  public void addFireListener(IDISFireListener handler)
  {
    _fireListeners.add(handler);
  }

  @Override
  public void addDetonationListener(IDISDetonationListener handler)
  {
    _detonationListeners.add(handler);
  }

  @Override
  public void addScenarioListener(IDISScenarioListener handler)
  {
    _scenarioListeners.add(handler);
  }

  @Override
  public void setProvider(IPDUProvider provider)
  {
    // remember we're restarting
    _newStart = true;

    // register as a listener, to hear about new data
    provider.addListener(this);
  }

  private void handleFix(EntityStatePdu pdu)
  {
    // unpack the data
    final short eid = pdu.getExerciseID();
    final short force = pdu.getForceId();
    final long hisId = pdu.getEntityID().getEntity();

    boolean isEstimated = pdu.getEntityType().getEntityKind() == ESTIMATED_KIND;

    long time = convertTime(pdu.getTimestamp());
    Vector3Double loc = pdu.getEntityLocation();
    double[] locArr = new double[]
    {loc.getX(), loc.getY(), loc.getZ()};
    double[] worldCoords = CoordinateConversions.xyzToLatLonDegrees(locArr);
    Orientation orientation = pdu.getEntityOrientation();
    Vector3Float velocity = pdu.getEntityLinearVelocity();

    double speedMs =
        Math.sqrt(velocity.getX() * velocity.getX() + velocity.getY()
            * velocity.getY());

    // entity state
    String hisName = nameFor(hisId);

    EntityType cat = pdu.getEntityType();
    short kind = cat.getEntityKind();
    short domain = cat.getDomain();
    short category = cat.getCategory();

    Iterator<IDISFixListener> fIter = _fixListeners.iterator();
    while (fIter.hasNext())
    {
      IDISFixListener thisF = (IDISFixListener) fIter.next();
      thisF.add(time, eid, hisId, hisName, force, kind, domain, category,
          isEstimated, worldCoords[0], worldCoords[1], -worldCoords[2],
          orientation.getPhi(), speedMs, pdu.getEntityAppearance_damage());
    }
  }

  private void handleEvent(EventReportPdu pdu)
  {
    short eid = pdu.getExerciseID();
    long time = convertTime(pdu.getTimestamp());
    int originator = pdu.getOriginatingEntityID().getEntity();
    int eType = (int) pdu.getEventType();
    String msg = "";

    // try to get the data
    List<VariableDatum> items = pdu.getVariableDatums();
    for (int i = 0; i < items.size(); i++)
    {
      VariableDatum val = items.get(i);
      List<OneByteChunk> chunks = val.getVariableData();
      final int thisLen = (int) val.getVariableDatumLength();
      byte[] bytes = new byte[thisLen];
      Iterator<OneByteChunk> iter = chunks.iterator();
      int ctr = 0;
      for (int l = 0; l < thisLen; l++)
      {
        OneByteChunk thisB = (OneByteChunk) iter.next();
        final byte thisByte = thisB.getOtherParameters()[0];

        // if (eType == 10004)
        // {
        // System.out.println("byte:" + thisByte + " str:" + new String(new byte[]{thisByte}));
        // }

        if (thisByte > 10)
        {
          bytes[ctr++] = thisByte;
        }
        else if (thisByte == 1 || thisByte == 9)
        {
          bytes[ctr++] = 32;
        }
      }
      String newS = new String(bytes).trim();

      msg += newS;
    }

    if (msg.length() == 0)
    {
      msg = "Unset";
    }

    // sort out his name
    _nameListener.add(time, eid, originator, null, eType, msg);

    // now try to retrieve name
    String hisName = nameFor(originator);

    // first send out to specific listeners
    List<IDISEventListener> specificListeners = _eventListeners.get(eType);
    if (specificListeners != null)
    {
      Iterator<IDISEventListener> eIter = specificListeners.iterator();
      while (eIter.hasNext())
{'fraction_non_alphanumeric': 0.1088145546663596, 'fraction_numerical': 0.0029938395992860846, 'mean_word_length': 3.2720118052139693, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5526597', 'n_tokens_mistral': 5540, 'n_tokens_neox': 5393, 'n_words': 1192}