[BUG] Cannot connect classification confusion matrix to App
On `fiftyone==0.16.5`, running this code to connect a confusion matrix for a `Classification` task to the App should be valid:

```py
import fiftyone as fo

filepath = "/path/to/image.jpg"

sample1 = fo.Sample(
    filepath=filepath,
    gt=fo.Classification(label="cat"),
    pred=fo.Classification(label="cat", confidence=0.99),
)
sample2 = fo.Sample(
    filepath=filepath,
    gt=fo.Classification(label="dog"),
    pred=fo.Classification(label="dog", confidence=0.8),
)
sample3 = fo.Sample(
    filepath=filepath,
    gt=fo.Classification(label="dog"),
    pred=fo.Classification(label="cat", confidence=0.1),
)

dataset = fo.Dataset()
dataset.add_samples([sample1, sample2, sample3])

results = dataset.evaluate_classifications("pred", gt_field="gt", eval_key="eval")

plot = results.plot_confusion_matrix()
plot.show()

session = fo.launch_app(dataset)

# Raises the error shown below
session.plots.attach(plot)
```

but currently raises the following error:

```
ValueError: Invalid label field type <class 'fiftyone.core.labels.Classification'>. Extracting patches is only supported for the following types: (<class 'fiftyone.core.labels.Detections'>, <class 'fiftyone.core.labels.Polylines'>)
```

The problem is that the plot's `selection_mode` defaults to `"patches"`, but that shouldn't happen for classification tasks.

Therefore, one can workaround this issue by manually setting the plot's `selection_mode` to `"select"` or `"match"` prior to attaching the plot to the App:

```py
# Avoids the error
plot.selection_mode = "match"
session.plots.attach(plot)
```

diff --git a/fiftyone/core/plots/base.py b/fiftyone/core/plots/base.py
index 1afd66080..90ed6d7d0 100644
--- a/fiftyone/core/plots/base.py
+++ b/fiftyone/core/plots/base.py
@@ -8,6 +8,9 @@ Base plotting definitions.
 import logging
 
 import fiftyone.core.context as foc
+import fiftyone.core.labels as fol
+import fiftyone.core.patches as fop
+import fiftyone.core.video as fov
 
 
 logger = logging.getLogger(__name__)
@@ -998,6 +1001,48 @@ class InteractivePlot(ResponsivePlot):
     def _selected_ids(self):
         raise NotImplementedError("Subclass must implement _selected_ids")
 
+    @staticmethod
+    def recommend_link_type(label_field=None, samples=None):
+        """Recommends a link type for the given info.
+
+        Args:
+            label_field (None): the label field, if any
+            samples (None): the
+                :class:`fiftyone.core.collections.SampleCollection`, if known
+
+        Returns:
+            a ``(link_type, label_fields, selection_mode, init_fcn)`` tuple
+        """
+        link_type = None
+        label_fields = label_field
+        selection_mode = None
+        init_fcn = None
+
+        if label_field is None:
+            if isinstance(samples, fov.FramesView):
+                link_type = "frames"
+            elif isinstance(samples, fop.PatchesView):
+                link_type = "labels"
+                label_fields = samples._label_fields
+            else:
+                link_type = "samples"
+        elif label_field == "frames":
+            if isinstance(samples, fov.FramesView):
+                link_type = "frames"
+            else:
+                link_type = "frames"
+                init_fcn = lambda view: view.to_frames()
+        else:
+            link_type = "labels"
+
+            if samples is not None:
+                label_type = samples._get_label_field_type(label_field)
+                if issubclass(label_type, (fol.Detections, fol.Polylines)):
+                    selection_mode = "patches"
+                    init_fcn = lambda view: view.to_patches(label_field)
+
+        return link_type, label_fields, selection_mode, init_fcn
+
     def register_selection_callback(self, callback):
         """Registers a selection callback for this plot.
 
diff --git a/fiftyone/core/plots/matplotlib.py b/fiftyone/core/plots/matplotlib.py
index 3a903e752..f02c5ebff 100644
--- a/fiftyone/core/plots/matplotlib.py
+++ b/fiftyone/core/plots/matplotlib.py
@@ -614,27 +614,15 @@ def lines(
             plt.tight_layout()
             return fig
 
-        selection_mode = None
-        init_fcn = None
-
-        if link_field is None:
-            if isinstance(samples, fov.FramesView):
-                link_type = "frames"
-            elif isinstance(samples, fop.PatchesView):
-                link_type = "labels"
-                link_field = samples._label_fields
-            else:
-                link_type = "samples"
-        elif link_field == "frames":
-            if isinstance(samples, fov.FramesView):
-                link_type = "frames"
-            else:
-                link_type = "frames"
-                init_fcn = lambda view: view.to_frames()
-        else:
-            link_type = "labels"
-            selection_mode = "patches"
-            init_fcn = lambda view: view.to_patches(link_field)
+        (
+            link_type,
+            label_fields,
+            selection_mode,
+            init_fcn,
+        ) = InteractiveCollection.recommend_link_type(
+            label_field=link_field,
+            samples=samples,
+        )
 
         return InteractiveCollection(
             collection,
@@ -642,7 +630,7 @@ def lines(
             buttons=buttons,
             link_type=link_type,
             init_view=samples,
-            label_fields=link_field,
+            label_fields=label_fields,
             selection_mode=selection_mode,
             init_fcn=init_fcn,
         )
@@ -785,27 +773,15 @@ def scatterplot(
         if ids is not None and inds is not None:
             ids = ids[inds]
 
-        selection_mode = None
-        init_fcn = None
-
-        if link_field is None:
-            if isinstance(samples, fov.FramesView):
-                link_type = "frames"
-            elif isinstance(samples, fop.PatchesView):
-                link_type = "labels"
-                link_field = samples._label_fields
-            else:
-                link_type = "samples"
-        elif link_field == "frames":
-            if isinstance(samples, fov.FramesView):
-                link_type = "frames"
-            else:
-                link_type = "frames"
-                init_fcn = lambda view: view.to_frames()
-        else:
-            link_type = "labels"
-            selection_mode = "patches"
-            init_fcn = lambda view: view.to_patches(link_field)
+        (
+            link_type,
+            label_fields,
+            selection_mode,
+            init_fcn,
+        ) = InteractiveCollection.recommend_link_type(
+            label_field=link_field,
+            samples=samples,
+        )
 
         return InteractiveCollection(
             collection,
@@ -813,7 +789,7 @@ def scatterplot(
             buttons=buttons,
             link_type=link_type,
             init_view=samples,
-            label_fields=link_field,
+            label_fields=label_fields,
             selection_mode=selection_mode,
             init_fcn=init_fcn,
         )
diff --git a/fiftyone/core/plots/plotly.py b/fiftyone/core/plots/plotly.py
index 7abbd7caf..e0cc56e82 100644
--- a/fiftyone/core/plots/plotly.py
+++ b/fiftyone/core/plots/plotly.py
@@ -196,7 +196,18 @@ def _plot_confusion_matrix_interactive(
     num_rows, num_cols = confusion_matrix.shape
     zlim = [0, confusion_matrix.max()]
 
-    if gt_field and pred_field:
+    if samples is not None and eval_key is not None:
+        if gt_field is None or pred_field is None:
+            eval_info = samples.get_evaluation_info(eval_key)
+            gt_field = eval_info.config.gt_field
+            pred_field = eval_info.config.pred_field
+
+        label_type = samples._get_label_field_type(gt_field)
+        use_patches = issubclass(label_type, (fol.Detections, fol.Polylines))
+    else:
+        use_patches = False
+
+    if gt_field is not None and pred_field is not None:
         label_fields = [gt_field, pred_field]
     else:
         label_fields = None
@@ -211,7 +222,7 @@ def _plot_confusion_matrix_interactive(
     ids = np.flip(ids, axis=0)
     ylabels = np.flip(ylabels)
 
-    if eval_key is not None:
+    if use_patches:
         selection_mode = "patches"
         init_fcn = lambda view: view.to_evaluation_patches(eval_key)
     else:
@@ -941,33 +952,21 @@ def lines(
 
         return figure
 
-    selection_mode = None
-    init_fcn = None
-
-    if link_field is None:
-        if isinstance(samples, fov.FramesView):
-            link_type = "frames"
-        elif isinstance(samples, fop.PatchesView):
-            link_type = "labels"
-            link_field = samples._label_fields
-        else:
-            link_type = "samples"
-    elif link_field == "frames":
-        if isinstance(samples, fov.FramesView):
-            link_type = "frames"
-        else:
-            link_type = "frames"
-            init_fcn = lambda view: view.to_frames()
-    else:
-        link_type = "labels"
-        selection_mode = "patches"
-        init_fcn = lambda view: view.to_patches(link_field)
+    (
+        link_type,
+        label_fields,
+        selection_mode,
+        init_fcn,
+    ) = InteractiveScatter.recommend_link_type(
+        label_field=link_field,
+        samples=samples,
+    )
 
     return InteractiveScatter(
         figure,
         link_type=link_type,
         init_view=samples,
-        label_fields=link_field,
+        label_fields=label_fields,
         selection_mode=selection_mode,
         init_fcn=init_fcn,
     )
@@ -1195,33 +1194,21 @@ def scatterplot(
 
         return figure
 
-    selection_mode = None
-    init_fcn = None
-
-    if link_field is None:
-        if isinstance(samples, fov.FramesView):
-            link_type = "frames"
-        elif isinstance(samples, fop.PatchesView):
-            link_type = "labels"
-            link_field = samples._label_fields
-        else:
-            link_type = "samples"
-    elif link_field == "frames":
-        if isinstance(samples, fov.FramesView):
-            link_type = "frames"
-        else:
-            link_type = "frames"
-            init_fcn = lambda view: view.to_frames()
-    else:
-        link_type = "labels"
-        selection_mode = "patches"
-        init_fcn = lambda view: view.to_patches(link_field)
+    (
+        link_type,
+        label_fields,
+        selection_mode,
+        init_fcn,
+    ) = InteractiveScatter.recommend_link_type(
+        label_field=link_field,
+        samples=samples,
+    )
 
     return InteractiveScatter(
         figure,
         link_type=link_type,
         init_view=samples,
-        label_fields=link_field,
+        label_fields=label_fields,
         selection_mode=selection_mode,
         init_fcn=init_fcn,
     )
['fiftyone/core/plots/matplotlib.py', 'fiftyone/core/plots/plotly.py', 'fiftyone/core/plots/base.py']