Greatly improve prefetching and reduce # of db calls for each trade_offer

docker-compose.yml

@@ -7,29 +7,32 @@ services:
     restart: always
     volumes:
       - ./seed:/seed:ro
+      # DANGEROUS DUE TO DOCKERFILE PACKAGE BUILDING/INSTALLATION
+      #- ./src/pkmntrade_club:/app/lib/python3.12/site-packages/pkmntrade_club:ro
     env_file:
       - .env
     environment:
       - DEBUG=true
       - PUBLIC_HOST=localhost
       - ALLOWED_HOSTS=127.0.0.1,localhost
-    depends_on:
+      - DISABLE_CACHE=false
-      db:
+#     depends_on:
-        condition: service_healthy
+#       db:
-  db:
+#         condition: service_healthy
-    image: postgres:16
+#   db:
-    restart: always
+#     image: postgres:16
-    ports:
+#     restart: always
-      - 5432:5432
+#     ports:
-    volumes:
+#       - 5432:5432
-      - postgres_data:/var/lib/postgresql/data/
+#     volumes:
-    environment:
+#       - postgres_data:/var/lib/postgresql/data/
-      - "POSTGRES_HOST_AUTH_METHOD=trust"
+#     environment:
-    healthcheck:
+#       - "POSTGRES_HOST_AUTH_METHOD=trust"
-      test: ["CMD", "pg_isready", "-U", "postgres", "-d", "postgres"]
+#     healthcheck:
-      interval: 10s
+#       test: ["CMD", "pg_isready", "-U", "postgres", "-d", "postgres"]
-      timeout: 5s
+#       interval: 10s
-      retries: 5
+#       timeout: 5s
+#       retries: 5
 
-volumes:
+# volumes:
-  postgres_data:
+#   postgres_data:

src/pkmntrade_club/home/views.py

@@ -89,27 +89,30 @@ class HomePageView(TemplateView):
                 logger.error(f"Error fetching 'All' featured offers: {str(e)}")
                 featured["All"] = []
                 
-            try:
+            # *** we only show All Featured Offers for now, 
-                # Pull out distinct (rarity_level, rarity_icon) tuples
+            # *** we will add rarity-tabbed featured offers later
-                distinct_rarities = base_offer_qs.values_list("rarity_level", "rarity_icon").distinct()
+            # try:
+            #     # Pull out distinct (rarity_level, rarity_icon) tuples
+            #     distinct_rarities = base_offer_qs.values_list("rarity_level", "rarity_icon").distinct()
                 
-                # Prepare a list that holds tuples of (rarity_level, rarity_icon, offers)
+            #     # Prepare a list that holds tuples of (rarity_level, rarity_icon, offers)
-                rarity_offers = []
+            #     rarity_offers = []
-                for rarity_level, rarity_icon in distinct_rarities:
+            #     for rarity_level, rarity_icon in distinct_rarities:
-                    offers = base_offer_qs.filter(rarity_level=rarity_level).order_by("created_at")[:6]
+            #         offers = base_offer_qs.filter(rarity_level=rarity_level).order_by("created_at")[:6]
-                    rarity_offers.append((rarity_level, rarity_icon, offers))
+            #         rarity_offers.append((rarity_level, rarity_icon, offers))
                 
-                # Sort by rarity_level (from greatest to least)
+            #     # Sort by rarity_level (from greatest to least)
-                rarity_offers.sort(key=lambda x: x[0], reverse=True)
+            #     rarity_offers.sort(key=lambda x: x[0], reverse=True)
             
-                # Add the sorted offers to the OrderedDict
+            #     # Add the sorted offers to the OrderedDict
-                for rarity_level, rarity_icon, offers in rarity_offers:
+            #     for rarity_level, rarity_icon, offers in rarity_offers:
-                    featured[rarity_icon] = offers
+            #         featured[rarity_icon] = offers
-            except Exception as e:
+            # except Exception as e:
-                logger.error(f"Error processing rarity-based featured offers: {str(e)}")
+            #     logger.error(f"Error processing rarity-based featured offers: {str(e)}")
 
             context["featured_offers"] = featured
             # Generate a cache key based on the pks and updated_at timestamps of all featured offers
+            # *** we will separate cache keys for each featured section later
             all_offer_identifiers = []
             for section_name,section_offers in featured.items():
                 # featured_section is a QuerySet. Fetch (pk, updated_at) tuples.

src/pkmntrade_club/trades/forms.py

@@ -55,15 +55,13 @@ class TradeAcceptanceCreateForm(forms.ModelForm):
         elif default_friend_code and friend_codes.filter(pk=default_friend_code.pk).exists():
             self.initial["accepted_by"] = default_friend_code.pk
 
-        # Fix: Convert available 'have' cards (from through model) to Card objects.
+        available_have_items = trade_offer.have_cards_available
-        self.fields["requested_card"].queryset = Card.objects.filter(
+        requested_card_pks = [item.card.pk for item in available_have_items]
-            pk__in=trade_offer.have_cards_available_qs.values_list("card_id", flat=True)
+        self.fields["requested_card"].queryset = Card.objects.filter(pk__in=requested_card_pks).order_by('name')
-        )
 
-        # Similarly for offered_card.
+        available_want_items = trade_offer.want_cards_available
-        self.fields["offered_card"].queryset = Card.objects.filter(
+        offered_card_pks = [item.card.pk for item in available_want_items]
-            pk__in=trade_offer.want_cards_available_qs.values_list("card_id", flat=True)
+        self.fields["offered_card"].queryset = Card.objects.filter(pk__in=offered_card_pks).order_by('name')
-        )
 
     def clean(self):
         """

src/pkmntrade_club/trades/models.py

@@ -25,20 +25,46 @@ def generate_tradeacceptance_hash():
 class TradeOfferManager(models.Manager):
 
     def get_queryset(self):
-        qs = super().get_queryset().select_related(
+        qs = super().get_queryset()
-            "initiated_by",
+
-            "initiated_by__user",
+        # Prefetch for have_cards (through model: TradeOfferHaveCard)
-        ).prefetch_related(
+        # Ensures 'card' is select_related and 'Meta.ordering' is respected/applied.
-            "trade_offer_have_cards__card",
+        prefetch_have_cards = Prefetch(
-            "trade_offer_want_cards__card",
+            'trade_offer_have_cards',
-            "acceptances",
+            queryset=TradeOfferHaveCard.objects.select_related('card').order_by('card__name')
-            "acceptances__accepted_by",
-            "acceptances__requested_card",
-            "acceptances__offered_card",
-            "acceptances__accepted_by__user",
         )
 
-        return qs.order_by("-updated_at")
+        # Prefetch for want_cards (through model: TradeOfferWantCard)
+        # Ensures 'card' is select_related and 'Meta.ordering' is respected/applied.
+        prefetch_want_cards = Prefetch(
+            'trade_offer_want_cards',
+            queryset=TradeOfferWantCard.objects.select_related('card').order_by('card__name')
+        )
+
+        # Prefetch for acceptances
+        # Ensures related 'accepted_by__user', 'requested_card', 'offered_card' are fetched.
+        prefetch_acceptances = Prefetch(
+            'acceptances',
+            queryset=TradeAcceptance.objects.select_related(
+                'accepted_by__user',
+                'requested_card',
+                'offered_card'
+            ).order_by('-created_at') # Sensible default ordering for acceptances
+        )
+
+        qs = qs.select_related(
+            "initiated_by__user",  # Fetches FriendCode and its related CustomUser
+        ).prefetch_related(
+            prefetch_have_cards,
+            prefetch_want_cards,
+            prefetch_acceptances,
+            # If direct access like offer.have_cards.all() (the M2M to Card, not through model)
+            # is heavily used AND causes N+1s (e.g. via __str__), uncomment these:
+            Prefetch('have_cards'),
+            Prefetch('want_cards'),
+        )
+        
+        return qs.order_by("-updated_at") # Default ordering for TradeOffer querysets
 
 class TradeOffer(models.Model):
     objects = TradeOfferManager()
@@ -112,16 +138,6 @@ class TradeOffer(models.Model):
         # Returns the list of want_cards (through objects) that still have available quantity.
         return [item for item in self.trade_offer_want_cards.all() if item.quantity > item.qty_accepted]
 
-    @property
-    def have_cards_available_qs(self):
-        # Returns a queryset of TradeOfferHaveCard objects that still have available quantity
-        return self.trade_offer_have_cards.filter(quantity__gt=F("qty_accepted")).select_related("card")
-
-    @property
-    def want_cards_available_qs(self):
-        # Returns a queryset of TradeOfferWantCard objects that still have available quantity
-        return self.trade_offer_want_cards.filter(quantity__gt=F("qty_accepted")).select_related("card")
-
 class TradeOfferHaveCard(models.Model):
     """
     Through model for TradeOffer.have_cards.

src/pkmntrade_club/trades/templatetags/trade_offer_tags.py

@@ -7,11 +7,13 @@ register = template.Library()
 def render_trade_offer(context, offer):
     """
     Renders a trade offer including detailed trade acceptance information.
-    Freezes the through-model querysets to avoid extra DB hits.
+    Assumes 'offer' has related fields prefetched by the view.
     """
-    trade_offer_have_cards = list(offer.trade_offer_have_cards.select_related('card').all())
+    # Access prefetched data directly.
-    trade_offer_want_cards = list(offer.trade_offer_want_cards.select_related('card').all())
+    # The TradeOfferManager should have already handled select_related and prefetch_related.
-    acceptances = list(offer.acceptances.select_related('accepted_by__user', 'requested_card', 'offered_card').all())
+    trade_offer_have_cards = list(offer.trade_offer_have_cards.all())
+    trade_offer_want_cards = list(offer.trade_offer_want_cards.all())
+    acceptances = list(offer.acceptances.all())
 
 
     have_cards_available = [