REST response time does not scale properly with `limit` param while including config_context
### NetBox version

v3.2.2

### Python version

3.9

### Steps to Reproduce

1. GET devices from `/api/dcim/devices`, including config_context, with a resulting `count` field less than 100, and `limit=100` (for example, use a query parameter that would result in a low number of matches)
2. GET devices from `/api/dcim/devices` including config_context, with a resulting `count` field greater than 100,000, and `limit=100` (for example, use a query parameter that would result in a high number of matches)

### Expected Behavior

1. Similar API response times from either query, scaling with `limit`

### Observed Behavior

1. When the `exclude=config_context` parameter is included in a query, there is a similar response time between queries that return a large dataset, and queries that return a small dataset
2. When `exclude=config_context` is omitted (i.e. config context is fetched), queries that return a large dataset increase in response time dramatically vs. smaller queries, regardless of `limit`

diff --git a/netbox/dcim/api/views.py b/netbox/dcim/api/views.py
index e99ef333a..c4c25f654 100644
--- a/netbox/dcim/api/views.py
+++ b/netbox/dcim/api/views.py
@@ -19,6 +19,7 @@ from ipam.models import Prefix, VLAN
 from netbox.api.authentication import IsAuthenticatedOrLoginNotRequired
 from netbox.api.exceptions import ServiceUnavailable
 from netbox.api.metadata import ContentTypeMetadata
+from netbox.api.pagination import StripCountAnnotationsPaginator
 from netbox.api.viewsets import NetBoxModelViewSet
 from netbox.config import get_config
 from utilities.api import get_serializer_for_model
@@ -392,6 +393,7 @@ class DeviceViewSet(ConfigContextQuerySetMixin, NetBoxModelViewSet):
         'virtual_chassis__master', 'primary_ip4__nat_outside', 'primary_ip6__nat_outside', 'tags',
     )
     filterset_class = filtersets.DeviceFilterSet
+    pagination_class = StripCountAnnotationsPaginator
 
     def get_serializer_class(self):
         """
diff --git a/netbox/netbox/api/pagination.py b/netbox/netbox/api/pagination.py
index d89e32124..5ecade264 100644
--- a/netbox/netbox/api/pagination.py
+++ b/netbox/netbox/api/pagination.py
@@ -16,7 +16,7 @@ class OptionalLimitOffsetPagination(LimitOffsetPagination):
     def paginate_queryset(self, queryset, request, view=None):
 
         if isinstance(queryset, QuerySet):
-            self.count = queryset.count()
+            self.count = self.get_queryset_count(queryset)
         else:
             # We're dealing with an iterable, not a QuerySet
             self.count = len(queryset)
@@ -52,6 +52,9 @@ class OptionalLimitOffsetPagination(LimitOffsetPagination):
 
         return self.default_limit
 
+    def get_queryset_count(self, queryset):
+        return queryset.count()
+
     def get_next_link(self):
 
         # Pagination has been disabled
@@ -67,3 +70,16 @@ class OptionalLimitOffsetPagination(LimitOffsetPagination):
             return None
 
         return super().get_previous_link()
+
+
+class StripCountAnnotationsPaginator(OptionalLimitOffsetPagination):
+    """
+    Strips the annotations on the queryset before getting the count
+    to optimize pagination of complex queries.
+    """
+    def get_queryset_count(self, queryset):
+        # Clone the queryset to avoid messing up the actual query
+        cloned_queryset = queryset.all()
+        cloned_queryset.query.annotations.clear()
+
+        return cloned_queryset.count()
['netbox/netbox/api/pagination.py', 'netbox/dcim/api/views.py']