Cannot read field "durationInMicroseconds" because "right" is null
Running a scheduling goal twice in a row for Banananation on develop gives the error:

```text
java.lang.NullPointerException: Cannot read field "durationInMicroseconds" because "right" is null
	at gov.nasa.jpl.aerie.merlin.protocol.types.Duration.add(Duration.java:230)
	at gov.nasa.jpl.aerie.merlin.protocol.types.Duration.plus(Duration.java:355)
	at gov.nasa.jpl.aerie.scheduler.model.ActivityInstance.getEndTime(ActivityInstance.java:127)
	at gov.nasa.jpl.aerie.scheduler.solver.PrioritySolver.satisfyGoalGeneral(PrioritySolver.java:506)
	at gov.nasa.jpl.aerie.scheduler.solver.PrioritySolver.satisfyGoal(PrioritySolver.java:332)
	at gov.nasa.jpl.aerie.scheduler.solver.PrioritySolver.solve(PrioritySolver.java:291)
	at gov.nasa.jpl.aerie.scheduler.solver.PrioritySolver.getNextSolution(PrioritySolver.java:127)
	at gov.nasa.jpl.aerie.scheduler.worker.services.SynchronousSchedulerAgent.schedule(SynchronousSchedulerAgent.java:198)
	at gov.nasa.jpl.aerie.scheduler.worker.SchedulerWorkerAppDriver.main(SchedulerWorkerAppDriver.java:91)
```

Here is the goal: 

```ts
export default (): Goal =>
  Goal.ActivityRecurrenceGoal({
    activityTemplate: ActivityTemplates.BakeBananaBread({ temperature: 325.0, tbSugar: 2, glutenFree: false }),
    interval: Temporal.Duration.from({ hours: 12 }),
  });
```

This is causing the UI e2e tests to fail in PR.
diff --git a/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/model/ActivityInstance.java b/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/model/ActivityInstance.java
index 1ca2eda70..a2f85e713 100644
--- a/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/model/ActivityInstance.java
+++ b/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/model/ActivityInstance.java
@@ -101,6 +101,24 @@ public record ActivityInstance(
                                 parameters, topParent);
   }
 
+  private static ActivityInstance of(SchedulingActivityInstanceId id, ActivityType type, Duration start, Duration duration, Map<String, SerializedValue> parameters, SchedulingActivityInstanceId topParent) {
+    return new ActivityInstance(id,
+                                type,
+                                start,
+                                duration,
+                                parameters,
+                                topParent);
+  }
+
+  public static ActivityInstance copyOf(ActivityInstance activityInstance, Duration duration){
+    return ActivityInstance.of(activityInstance.id,
+            activityInstance.type,
+            activityInstance.startTime,
+            duration,
+            new HashMap<>(activityInstance.arguments),
+            activityInstance.topParent);
+  }
+
   /**
    * create an activity instance based on the provided one (but adifferent id)
    *
diff --git a/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/SimulationFacade.java b/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/SimulationFacade.java
index 49cf30172..a9ce14a97 100644
--- a/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/SimulationFacade.java
+++ b/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/SimulationFacade.java
@@ -130,6 +130,28 @@ public class SimulationFacade {
     }
   }
 
+  /**
+   * Replaces an activity instance with another, strictly when they have the same id
+   * @param toBeReplaced the activity to be replaced
+   * @param replacement the replacement activity
+   */
+  public void replaceActivityFromSimulation(final ActivityInstance toBeReplaced, final ActivityInstance replacement){
+    if(toBeReplaced.type() != replacement.type()||
+       toBeReplaced.startTime() != replacement.startTime()||
+       !(toBeReplaced.arguments().equals(replacement.arguments()))) {
+      throw new IllegalArgumentException("When replacing an activity, you can only update the duration");
+    }
+    if(!insertedActivities.containsKey(toBeReplaced)){
+      throw new IllegalArgumentException("Trying to replace an activity that has not been previously simulated");
+    }
+    final var associated = insertedActivities.get(toBeReplaced);
+    insertedActivities.remove(toBeReplaced);
+    insertedActivities.put(replacement, associated);
+    final var simulationId = this.planActInstanceIdToSimulationActInstanceId.get(toBeReplaced.id());
+    this.planActInstanceIdToSimulationActInstanceId.remove(toBeReplaced.id());
+    this.planActInstanceIdToSimulationActInstanceId.put(replacement.id(), simulationId);
+  }
+
   public void simulateActivities(final Collection<ActivityInstance> activities)
   throws SimulationException {
     final var activitiesSortedByStartTime =
diff --git a/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/solver/PrioritySolver.java b/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/solver/PrioritySolver.java
index f9ac857ed..400b8bdd9 100644
--- a/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/solver/PrioritySolver.java
+++ b/scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/solver/PrioritySolver.java
@@ -188,6 +188,7 @@ public class PrioritySolver implements Solver {
       }
       final var allGeneratedActivities = simulationFacade.getAllGeneratedActivities(simulationFacade.getCurrentSimulationEndTime());
       processNewGeneratedActivities(allGeneratedActivities);
+      pullActivityDurationsIfNecessary();
     } else{
       //update simulation with regard to plan
       try {
@@ -223,9 +224,38 @@ public class PrioritySolver implements Solver {
       simulationFacade.simulateActivities(plan.getActivities());
       final var allGeneratedActivities = simulationFacade.getAllGeneratedActivities(problem.getPlanningHorizon().getEndAerie());
       processNewGeneratedActivities(allGeneratedActivities);
+      pullActivityDurationsIfNecessary();
     }
   }
 
+  /**
+   * For activities that have a null duration (in an initial plan for example) and that have been simulated, we pull the duration and
+   * replace the original instance with a new instance that includes the duraiton, both in the plan and the simulation facade
+   */
+  public void pullActivityDurationsIfNecessary() {
+    final var toRemoveFromPlan = new ArrayList<ActivityInstance>();
+    final var toAddToPlan = new ArrayList<ActivityInstance>();
+    for (final var activity : plan.getActivities()) {
+      if (activity.duration() == null) {
+        final var duration = simulationFacade.getActivityDuration(activity);
+        if (duration.isPresent()) {
+          final var replacementAct = ActivityInstance.copyOf(
+              activity,
+              duration.get()
+              );
+          simulationFacade.replaceActivityFromSimulation(activity, replacementAct);
+          toAddToPlan.add(replacementAct);
+          toRemoveFromPlan.add(activity);
+          generatedActivityInstances = generatedActivityInstances.stream().map(pair -> pair.getLeft().equals(activity) ? Pair.of(replacementAct, pair.getRight()): pair).collect(Collectors.toList());
+          generatedActivityInstances = generatedActivityInstances.stream().map(pair -> pair.getRight().equals(activity) ? Pair.of(pair.getLeft(), replacementAct): pair).collect(Collectors.toList());
+        }
+      }
+    }
+
+    plan.remove(toRemoveFromPlan);
+    plan.add(toAddToPlan);
+  }
+
   /**
    * Filters generated activities and makes sure that simulations are only adding activities and not removing them
    * @param allNewGeneratedActivities all the generated activities from the last simulation results.
diff --git a/scheduler-worker/src/test/java/gov/nasa/jpl/aerie/scheduler/worker/services/SchedulingIntegrationTests.java b/scheduler-worker/src/test/java/gov/nasa/jpl/aerie/scheduler/worker/services/SchedulingIntegrationTests.java
index dd5f3268d..af8e28fee 100644
--- a/scheduler-worker/src/test/java/gov/nasa/jpl/aerie/scheduler/worker/services/SchedulingIntegrationTests.java
+++ b/scheduler-worker/src/test/java/gov/nasa/jpl/aerie/scheduler/worker/services/SchedulingIntegrationTests.java
@@ -1348,6 +1348,35 @@ public class SchedulingIntegrationTests {
     assertEquals(96, results.updatedPlan().size());
   }
 
+  /**
+   * If you passed activities without duration to the scheduler in an initial plan, it would fail
+   */
+  @Test
+  public void testBugDurationInMicroseconds(){
+    final var results = runScheduler(
+        BANANANATION,
+        List.of(),
+        List.of(new SchedulingGoal(new GoalId(0L), """
+            export default (): Goal =>
+              Goal.ActivityRecurrenceGoal({
+                activityTemplate: ActivityTemplates.BakeBananaBread({ temperature: 325.0, tbSugar: 2, glutenFree: false }),
+                interval: Temporal.Duration.from({ hours: 12 }),
+              });
+            """, true)),
+        PLANNING_HORIZON);
+    final var results2 = runScheduler(
+        BANANANATION,
+        results.updatedPlan.stream().toList(),
+        List.of(new SchedulingGoal(new GoalId(0L), """
+            export default (): Goal =>
+              Goal.ActivityRecurrenceGoal({
+                activityTemplate: ActivityTemplates.BakeBananaBread({ temperature: 325.0, tbSugar: 2, glutenFree: false }),
+                interval: Temporal.Duration.from({ hours: 12 }),
+              });
+            """, true)),
+        PLANNING_HORIZON);
+    assertEquals(8, results.updatedPlan().size());
+  }
 
   @Test
   void test_inf_loop(){
['scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/simulation/SimulationFacade.java', 'scheduler-worker/src/test/java/gov/nasa/jpl/aerie/scheduler/worker/services/SchedulingIntegrationTests.java', 'scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/model/ActivityInstance.java', 'scheduler-driver/src/main/java/gov/nasa/jpl/aerie/scheduler/solver/PrioritySolver.java']