ai_ml_operations/Example-TCG-Site
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Last bit of major changes

Browse Source 
Closes #1
Closes #5
Closes #6
Closes #8
Closes #9
Closes #10
This commit is contained in:
Ryan Westfall
2026-01-26 04:11:38 -06:00
parent 1cd87156bd
commit 739d136209
24 changed files with 1157 additions and 410 deletions
Download Patch File Download Diff File Expand all files Collapse all files

264
store/management/commands/populate_db.py
View File

@@ -1,139 +1,201 @@
import random
import requests
from django.core.management import call_command
from django.core.management.base import BaseCommand
from django.utils.text import slugify
from django.contrib.auth import get_user_model
from store.models import Game, Set, Card, CardListing
from faker import Faker
from store.models import Game, Set, Card, CardListing, Seller
User = get_user_model()
fake = Faker()
class Command(BaseCommand):
help = 'Populate database with MTG data from Scryfall and fake data for other games'
help = 'Populate database with specific test stores and listings, leveraging external scripts for card data.'
def handle(self, *args, **options):
self.stdout.write('Starting database population...')
# Create Games
mtg, _ = Game.objects.get_or_create(name='Magic: The Gathering', slug='mtg')
pokemon, _ = Game.objects.get_or_create(name='Pokemon TCG', slug='pokemon')
lorcana, _ = Game.objects.get_or_create(name='Disney Lorcana', slug='lorcana')
# Populate MTG (Real Data)
self.populate_mtg(mtg)
# Populate Pokemon (Fake Data)
self.populate_fake_game(pokemon, 'Pokemon')
# Populate Lorcana (Fake Data)
self.populate_fake_game(lorcana, 'Lorcana')
# Create Superuser
if not User.objects.filter(username='admin').exists():
User.objects.create_superuser('admin', 'admin@example.com', 'admin')
self.stdout.write(self.style.SUCCESS('Created superuser: admin/admin'))
# 1. Ensure Games and Cards exist
self.ensure_games_populated()
# Create Demo Users
self.create_demo_users()
self.stdout.write(self.style.SUCCESS('Database populated successfully!'))
# 2. Get Game Objects
try:
mtg = Game.objects.get(slug='magic-the-gathering')
pokemon = Game.objects.get(slug='pokemon-tcg')
lorcana = Game.objects.get(slug='disney-lorcana')
except Game.DoesNotExist as e:
self.stdout.write(self.style.ERROR(f"Missing a required game after population attempts! {e}"))
return
def populate_mtg(self, game):
self.stdout.write('Fetching MTG data from Scryfall...')
# Get a few sets
# 3. Create Stores
self.create_stores_and_listings(mtg, pokemon, lorcana)
self.stdout.write(self.style.SUCCESS('Database population complete!'))
def ensure_games_populated(self):
# MTG - Native (kept simple from original) or call a script if we had one.
# Since the original had MTG logic, I'll keep a simplified version here or call it if I haven't deleted it.
# But wait, I'm replacing the whole file. I should probably keep the MTG fetcher or rely on what's there?
# The prompt implies I should just "overhaul" it. I'll re-implement a robust MTG fetcher or reuse the old one's logic if I want
# but to be safe and clean, I'll just check if cards exist, if not, fetch some.
# Check MTG
if not Game.objects.filter(slug='magic-the-gathering').exists() or not Card.objects.filter(set__game__slug='magic-the-gathering').exists():
self.stdout.write("Populating MTG Cards...")
self.populate_mtg()
else:
self.stdout.write("MTG cards found, skipping population.")
# Check Pokemon
if not Game.objects.filter(slug='pokemon-tcg').exists() or not Card.objects.filter(set__game__slug='pokemon-tcg').exists():
self.stdout.write("Populating Pokemon Cards via script...")
call_command('populate_pokemon_cards')
else:
self.stdout.write("Pokemon cards found, skipping population.")
# Check Lorcana
if not Game.objects.filter(slug='disney-lorcana').exists() or not Card.objects.filter(set__game__slug='disney-lorcana').exists():
self.stdout.write("Populating Lorcana Cards via script...")
call_command('populate_lorcana_cards')
else:
self.stdout.write("Lorcana cards found, skipping population.")
def populate_mtg(self):
# Simplified reused logic from original
game, _ = Game.objects.get_or_create(
slug='magic-the-gathering',
defaults={'name': 'Magic: The Gathering'}
)
sets_api = "https://api.scryfall.com/sets"
try:
resp = requests.get(sets_api).json()
# Pick top 3 recent expansion sets
target_sets = [s for s in resp['data'] if s['set_type'] == 'expansion'][:3]
for s_data in target_sets:
set_obj, created = Set.objects.get_or_create(
set_obj, _ = Set.objects.get_or_create(
game=game,
name=s_data['name'],
code=s_data['code'],
defaults={'release_date': s_data.get('released_at')}
)
if created:
self.stdout.write(f"Created set: {set_obj.name}")
# Fetch cards for this set
cards_url = s_data['search_uri']
cards_resp = requests.get(cards_url).json()
cards_url = s_data['search_uri']
cards_resp = requests.get(cards_url).json()
for card_data in cards_resp.get('data', [])[:50]:
image = None
if 'image_uris' in card_data:
image = card_data['image_uris'].get('normal')
elif 'card_faces' in card_data and 'image_uris' in card_data['card_faces'][0]:
image = card_data['card_faces'][0]['image_uris'].get('normal')
for card_data in cards_resp.get('data', [])[:20]: # Limit to 20 cards per set to be fast
if 'image_uris' in card_data:
image = card_data['image_uris'].get('normal')
elif 'card_faces' in card_data and 'image_uris' in card_data['card_faces'][0]:
image = card_data['card_faces'][0]['image_uris'].get('normal')
else:
continue
if not image: continue
card, _ = Card.objects.get_or_create(
set=set_obj,
name=card_data['name'],
collector_number=card_data['collector_number'],
defaults={
'rarity': card_data['rarity'].capitalize(),
'image_url': image,
'scryfall_id': card_data['id'],
'tcgplayer_id': card_data.get('tcgplayer_id'),
}
)
# Create Listings
self.create_listings_for_card(card)
Card.objects.get_or_create(
set=set_obj,
name=card_data['name'],
collector_number=card_data['collector_number'],
defaults={
'rarity': card_data['rarity'].capitalize(),
'image_url': image,
'scryfall_id': card_data['id'],
'tcgplayer_id': card_data.get('tcgplayer_id'),
}
)
except Exception as e:
self.stdout.write(self.style.ERROR(f"Failed to fetch MTG data: {e}"))
self.stdout.write(self.style.ERROR(f"Error populating MTG: {e}"))
def populate_fake_game(self, game, prefix):
self.stdout.write(f'Generating data for {game.name}...')
for i in range(3): # 3 Sets
set_name = f"{prefix} Set {i+1}"
set_obj, _ = Set.objects.get_or_create(
game=game,
name=set_name,
code=f"{prefix[:3].upper()}{i+1}",
defaults={'release_date': fake.date_between(start_date='-2y', end_date='today')}
def create_stores_and_listings(self, mtg, pokemon, lorcana):
# Define Store Configs
stores_config = [
# Store 1: 100 Magic the gathering single card listings
{
'name': 'Mystic Magic',
'slug': 'mystic-magic',
'listings': [(mtg, 100)]
},
# Store 2: 80 lorcana single card listings
{
'name': 'Inkborn Illumineers',
'slug': 'inkborn-illumineers',
'listings': [(lorcana, 80)]
},
# Store 3: 200 pokemon single card listings
{
'name': 'Poke Mart',
'slug': 'poke-mart',
'listings': [(pokemon, 200)]
},
# Store 4: 50 Magic the gathering and lorcana listings (Split 25/25)
{
'name': 'Wizards and Wanderers',
'slug': 'wizards-and-wanderers',
'listings': [(mtg, 25), (lorcana, 25)]
},
# Store 5: 40 Magic the gathering and 20 pokemon listings
{
'name': 'Mana & Mons',
'slug': 'mana-and-mons',
'listings': [(mtg, 40), (pokemon, 20)]
},
# Store 6: 100 lorcana and 10 pokemon listings
{
'name': 'Disney Duelists',
'slug': 'disney-duelists',
'listings': [(lorcana, 100), (pokemon, 10)]
},
# Store 7: 100 cards for all three games (Split ~33 each)
{
'name': 'The Collector Trove',
'slug': 'collector-trove',
'listings': [(mtg, 33), (lorcana, 33), (pokemon, 34)]
}
]
for config in stores_config:
self.stdout.write(f"Setting up store: {config['name']}...")
# Create User and Seller
username = config['slug'].replace('-', '')
user, created = User.objects.get_or_create(username=username, defaults={'email': f"{username}@example.com"})
if created:
user.set_password('password')
user.save()
seller, _ = Seller.objects.get_or_create(
user=user,
defaults={
'store_name': config['name'],
'slug': config['slug'],
'description': f"Welcome to {config['name']}!",
'contact_email': f"{username}@example.com"
}
)
for j in range(15): # 15 Cards per set
card, _ = Card.objects.get_or_create(
set=set_obj,
name=f"{prefix} Monster {fake.word().capitalize()}",
defaults={
'rarity': random.choice(['Common', 'Uncommon', 'Rare', 'Ultra Rare']),
'image_url': f"https://placehold.co/400x600?text={prefix}+{j}",
'collector_number': str(j+1)
}
)
self.create_listings_for_card(card)
# Create Listings
for game, count in config['listings']:
self.create_listings_for_store(seller, game, count)
def create_listings_for_card(self, card):
# Create 1-5 listings per card with different conditions
for _ in range(random.randint(1, 4)):
def create_listings_for_store(self, seller, game, count):
cards = list(Card.objects.filter(set__game=game))
if not cards:
self.stdout.write(self.style.WARNING(f"No cards found for {game.name}, cannot create listings."))
return
listing_count = 0
while listing_count < count:
card = random.choice(cards)
# Randomize attributes
price = round(random.uniform(1.00, 150.00), 2)
quantity = random.randint(1, 10)
condition = random.choice(['NM', 'LP', 'MP', 'HP'])
price = round(random.uniform(0.50, 100.00), 2)
CardListing.objects.create(
card=card,
condition=condition,
seller=seller,
price=price,
quantity=random.randint(1, 20),
market_price=price, # Simplified
is_foil=random.choice([True, False])
quantity=quantity,
condition=condition,
status='listed'
)
def create_demo_users(self):
# Create a Pro user
if not User.objects.filter(username='prouser').exists():
u = User.objects.create_user('prouser', 'pro@example.com', 'password')
u.profile.is_pro = True
u.profile.save()
self.stdout.write("Created prouser/password")
listing_count += 1
# Create a Basic user
if not User.objects.filter(username='basicuser').exists():
User.objects.create_user('basicuser', 'basic@example.com', 'password')
self.stdout.write("Created basicuser/password")
self.stdout.write(f" - Created {count} listings for {game.name}")

243
store/management/commands/populate_pokemon_cards.py
View File

@@ -1,12 +1,11 @@
import requests
import time
import os
from django.core.management.base import BaseCommand
from django.utils.dateparse import parse_date
from store.models import Game, Set, Card
class Command(BaseCommand):
help = 'Populates the database with Pokémon TCG sets and cards using the Pokémon TCG API.'
help = 'Populates the database with Pokémon TCG sets and cards using the TCGDex REST API (English).'
def add_arguments(self, parser):
parser.add_argument(
@@ -15,21 +14,17 @@ class Command(BaseCommand):
help='Clear existing Pokémon TCG cards and sets before populating.'
)
parser.add_argument(
'--duration',
default='7',
help='Duration in days to look back for new sets/cards. Use "all" to fetch everything. Default is 7 days.'
)
parser.add_argument(
'--api-key',
default=os.getenv('POKEMONTCG_API_KEY', None),
help='Optional API Key for higher rate limits.'
'--duration',
default='7',
help='(Not full supported by TCGDex) Duration in days to look back. For now, this will just fetch all sets as TCGDex sets endpoint is not sorted by date.'
)
def handle(self, *args, **options):
self.stdout.write(self.style.SUCCESS('Starting Pokémon TCG population...'))
# Setup Headers for API (Rate limits are better with a key)
self.headers = {'X-Api-Key': options['api_key']} if options['api_key'] else {}
self.stdout.write(self.style.SUCCESS('Starting Pokémon TCG population (via TCGDex)...'))
# User Agent is good practice
self.headers = {'User-Agent': 'ExampleTCGSite/1.0'}
base_url = "https://api.tcgdex.net/v2/en"
# 1. Ensure Game exists
game, created = Game.objects.get_or_create(
@@ -48,177 +43,99 @@ class Command(BaseCommand):
Set.objects.filter(game=game).delete()
self.stdout.write(self.style.SUCCESS('Cleared Pokémon data.'))
# Handle --duration
duration = options['duration']
start_date_str = None
if duration != 'all':
try:
days = int(duration)
from django.utils import timezone
from datetime import timedelta
start_date = timezone.now().date() - timedelta(days=days)
start_date_str = start_date.strftime('%Y/%m/%d') # API uses YYYY/MM/DD
self.stdout.write(f'Fetching data released since {start_date_str}...')
except ValueError:
self.stdout.write(self.style.ERROR('Invalid duration. Must be an integer or "all".'))
return
# 2. Fetch Sets
self.stdout.write('Fetching sets from Pokémon TCG API...')
self.stdout.write('Fetching sets from TCGDex...')
# Build query for sets
# If duration is set, we use the Lucene search syntax provided by the API
params = {'orderBy': '-releaseDate', 'pageSize': 250}
if start_date_str:
params['q'] = f'releaseDate:>=("{start_date_str}")'
try:
# Note: /v2/sets does not usually require pagination for < 250 sets if filtering by recent date
# But "all" will require pagination.
sets_data = self.fetch_all_pages('https://api.pokemontcg.io/v2/sets', params)
# TCGDex /sets returns a list of minimal set objects
response = requests.get(f"{base_url}/sets", headers=self.headers)
response.raise_for_status()
sets_data = response.json()
except Exception as e:
self.stdout.write(self.style.ERROR(f'Failed to fetch sets: {e}'))
return
self.stdout.write(self.style.ERROR(f'Failed to fetch sets: {e}'))
return
self.stdout.write(f'Found {len(sets_data)} sets. Processing...')
processed_sets = []
for set_data in sets_data:
release_date = parse_date(set_data.get('releaseDate', '').replace('/', '-'))
# Pokémon sets have an 'id' (e.g., 'swsh1') and 'ptcgoCode' (e.g., 'SSH').
# We use 'id' as the unique code.
set_code = set_data.get('id')
for s_data in sets_data:
# s_data example: {"id": "base1", "name": "Base Set", ...}
# TCGDex sets don't consistently provide releaseDate in the list view,
# so we'll leave it null or updated if we fetched details (which we might do for cards).
# For efficiency we might not fetch set details just for date if unnecessary.
set_obj, created = Set.objects.update_or_create(
code=set_code,
code=s_data.get('id'),
game=game,
defaults={
'name': set_data.get('name'),
'release_date': release_date,
'name': s_data.get('name'),
# 'release_date': None # Not available in simple list
}
)
processed_sets.append(set_obj)
self.stdout.write(self.style.SUCCESS(f'Processed {len(processed_sets)} sets.'))
# 3. Fetch Cards
# Strategy: To be efficient, if we have a specific duration, we query cards by date.
# If we are doing "all", we iterate through the sets we just found (or all sets) to ensure we get everything structured.
self.stdout.write('Fetching cards...')
card_params = {'pageSize': 250}
# We must iterate sets to get cards, as there isn't a robust "all cards new" stream without pagination headaches
# on some APIs, and TCGDex structure favors set traversal.
if start_date_str:
# Fetch all cards released after date (cross-set)
card_params['q'] = f'set.releaseDate:>=("{start_date_str}")'
self.fetch_and_process_cards(card_params, game)
else:
# Fetch by set to allow for better progress tracking/chunking if doing a full import
total_sets = len(processed_sets)
for idx, set_obj in enumerate(processed_sets):
self.stdout.write(f' [{idx+1}/{total_sets}] Fetching cards for set: {set_obj.name} ({set_obj.code})...')
total_sets = len(processed_sets)
for idx, set_obj in enumerate(processed_sets):
self.stdout.write(f' [{idx+1}/{total_sets}] Fetching cards for set: {set_obj.name} ({set_obj.code})...')
try:
# Fetch Set Detail to get cards
# Endpoint: /sets/{id}
set_resp = requests.get(f"{base_url}/sets/{set_obj.code}", headers=self.headers)
if set_resp.status_code == 404:
self.stdout.write(self.style.WARNING(f' Set {set_obj.code} detail not found. Skipping.'))
continue
set_resp.raise_for_status()
# Filter by specific set ID
set_card_params = {'pageSize': 250, 'q': f'set.id:{set_obj.code}'}
self.fetch_and_process_cards(set_card_params, game, specific_set=set_obj)
set_detail = set_resp.json()
cards = set_detail.get('cards', [])
# Sleep briefly to respect rate limits (60/min without key, 1000/min with key)
time.sleep(0.5 if options['api_key'] else 1.5)
except Exception as e:
self.stdout.write(self.style.ERROR(f' Failed to fetch cards for {set_obj.name}: {e}'))
continue
self.stdout.write(f' Found {len(cards)} cards.')
for c_data in cards:
# c_data example: {"id": "base1-1", "localId": "1", "name": "Alakazam", "image": "..."}
# Rarity is NOT in this list usually, requires fetching card detail. Skipping for speed.
# Image URL: TCGDex gives a base URL usually, e.g. ".../base1/1"
# Sometimes it has /high.png or /low.png supported. The provided 'image' field often works as is.
# It might have extension like .png or just be the base.
# The user-provided example curl showed "image": "https://assets.tcgdex.net/en/base/base1/1"
# Those usually redirect to an image or handle extension. Let's append /high.png if we want best quality or try as is.
# Actually, TCGDex assets usually need an extension. Let's assume the API provides a valid URL or we append.
# Inspecting typical TCGDex response: "image": ".../1" (no extension).
# Browsers handle it, but for our backend saving it might be tricky if it's not a direct file.
# Let's save the URL as provided + "/high.png" as a guess for better quality if it doesn't have extension,
# Or just use the provided one.
# Update: TCGDex documentation often says: {image}/high.webp or {image}/low.webp
base_image = c_data.get('image')
image_url = f"{base_image}/high.webp" if base_image else ''
Card.objects.update_or_create(
scryfall_id=c_data.get('id'),
defaults={
'set': set_obj,
'name': c_data.get('name'),
'rarity': '', # specific call needed, simplifying
'image_url': image_url,
'collector_number': c_data.get('localId', ''),
'external_url': f"https://tcgdex.dev/cards/{c_data.get('id')}", # simplified assumption
}
)
# Rate limiting check - TCGDex is generous but good validation to not slam
# time.sleep(0.1)
self.stdout.write(self.style.SUCCESS('Finished Pokémon TCG population!'))
def fetch_all_pages(self, url, params):
"""Helper to handle API pagination"""
results = []
page = 1
has_more = True
while has_more:
params['page'] = page
response = requests.get(url, params=params, headers=self.headers)
if response.status_code == 429:
self.stdout.write(self.style.WARNING('Rate limit hit. Sleeping for 10 seconds...'))
time.sleep(10)
continue
if response.status_code != 200:
raise Exception(f"API Error {response.status_code}: {response.text}")
data = response.json()
batch = data.get('data', [])
results.extend(batch)
# Check if we need more pages
total_count = data.get('totalCount', 0)
count = data.get('count', 0)
if len(results) >= total_count or count == 0:
has_more = False
else:
page += 1
return results
def fetch_and_process_cards(self, params, game, specific_set=None):
try:
cards_data = self.fetch_all_pages('https://api.pokemontcg.io/v2/cards', params)
except Exception as e:
self.stdout.write(self.style.ERROR(f' Failed to fetch cards: {e}'))
return
self.stdout.write(f' Processing {len(cards_data)} cards...')
# Cache sets if we are doing a bulk mixed query
sets_map = {}
if not specific_set:
sets_map = {s.code: s for s in Set.objects.filter(game=game)}
for card_data in cards_data:
# Determine Set
if specific_set:
set_obj = specific_set
else:
set_code = card_data.get('set', {}).get('id')
if set_code in sets_map:
set_obj = sets_map[set_code]
else:
# If set missing (rare if we synced sets first), try to fetch/create or skip
# For speed, we skip if not found in our pre-fetched map
continue
# Extract Image URL (Prefer Hi-Res)
image_url = ''
if 'images' in card_data:
image_url = card_data['images'].get('large', card_data['images'].get('small', ''))
# TCGPlayer ID (Sometimes provided in tcgplayer field)
tcgplayer_url = card_data.get('tcgplayer', {}).get('url', '')
# Extract ID from URL if possible, or store URL.
# Model expects 'tcgplayer_id' (integer usually).
# The API doesn't always give a clean ID field, often just the URL.
# We will try to parse or leave null if your model requires int.
# Assuming model handles null or we just store nothing.
tcgplayer_id = None
# External URL
external_url = tcgplayer_url if tcgplayer_url else f"https://pkmncards.com/card/{card_data.get('id')}"
# Collector Number
collector_number = card_data.get('number', '')
Card.objects.update_or_create(
scryfall_id=card_data.get('id'), # Using API ID as unique identifier
defaults={
'set': set_obj,
'name': card_data.get('name'),
'rarity': card_data.get('rarity', 'Common'),
'image_url': image_url,
'tcgplayer_id': tcgplayer_id, # Can be updated if you add parsing logic
'collector_number': collector_number,
'external_url': external_url,
}
)