owain/MileVault
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use ON CONFLICT upsert for health metrics - fixes concurrent worker race condition
Build and push images / build-backend (push) Successful in 5s
Details
Build and push images / build-worker (push) Successful in 5s
Details
Build and push images / build-frontend (push) Successful in 5s
Details

Browse Source
This commit is contained in:
Owain Jarrett
2026-06-06 15:53:56 +01:00
parent 8104ca5ed0
commit 38632cfe4f
1 changed files with 61 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/workers/tasks.py
+61 -86
View File
@@ -181,7 +181,7 @@ def parse_wellness_fit(file_path: str, user_id: int):
import fitparse import fitparse
from app.core.database import SyncSessionLocal from app.core.database import SyncSessionLocal
from app.models.user import HealthMetric from app.models.user import HealthMetric
from sqlalchemy import select, func from sqlalchemy import text
from datetime import datetime, timezone, date from datetime import datetime, timezone, date
try: try:
@@ -269,10 +269,9 @@ def parse_wellness_fit(file_path: str, user_id: int):
if not daily: if not daily:
return {"status": "no_data", "file": file_path} return {"status": "no_data", "file": file_path}
# Upsert into health_metrics # Upsert into health_metrics using ON CONFLICT to handle concurrent workers
with SyncSessionLocal() as db: with SyncSessionLocal() as db:
for day_date, data in daily.items(): for day_date, data in daily.items():
# Compute averages from raw readings
hrs = data.pop("heart_rates", []) hrs = data.pop("heart_rates", [])
stresses = data.pop("stress_values", []) stresses = data.pop("stress_values", [])
spo2s = data.pop("spo2_readings", []) spo2s = data.pop("spo2_readings", [])
@@ -283,7 +282,6 @@ def parse_wellness_fit(file_path: str, user_id: int):
avg_stress = (sum(stresses) / len(stresses)) if stresses else None avg_stress = (sum(stresses) / len(stresses)) if stresses else None
spo2_avg = (sum(spo2s) / len(spo2s)) if spo2s else None spo2_avg = (sum(spo2s) / len(spo2s)) if spo2s else None
# Rough sleep stage breakdown from level codes
# Garmin sleep levels: 0=unmeasurable, 1=awake, 2=light, 3=deep, 4=rem # Garmin sleep levels: 0=unmeasurable, 1=awake, 2=light, 3=deep, 4=rem
sleep_deep_s = sum(30 for l in sleep_levels if l == 3) if sleep_levels else None sleep_deep_s = sum(30 for l in sleep_levels if l == 3) if sleep_levels else None
sleep_light_s = sum(30 for l in sleep_levels if l == 2) if sleep_levels else None sleep_light_s = sum(30 for l in sleep_levels if l == 2) if sleep_levels else None
@@ -295,56 +293,40 @@ def parse_wellness_fit(file_path: str, user_id: int):
date_dt = datetime(day_date.year, day_date.month, day_date.day, tzinfo=timezone.utc) date_dt = datetime(day_date.year, day_date.month, day_date.day, tzinfo=timezone.utc)
# Check for existing record # ON CONFLICT upsert - race-condition safe, COALESCE preserves existing data
existing = db.execute( db.execute(text("""
select(HealthMetric).where( INSERT INTO health_metrics (user_id, date, resting_hr, avg_hr_day, avg_stress,
HealthMetric.user_id == user_id, spo2_avg, hrv_nightly_avg, hrv_5min_high, hrv_status, steps,
func.date(HealthMetric.date) == day_date, sleep_duration_s, sleep_deep_s, sleep_light_s, sleep_rem_s, sleep_awake_s)
) VALUES (:user_id, :date, :resting_hr, :avg_hr, :avg_stress,
).scalar_one_or_none() :spo2_avg, :hrv_avg, :hrv_high, :hrv_status, :steps,
:sleep_dur, :sleep_deep, :sleep_light, :sleep_rem, :sleep_awake)
if existing: ON CONFLICT (user_id, date) DO UPDATE SET
# Update only fields we have data for resting_hr = COALESCE(EXCLUDED.resting_hr, health_metrics.resting_hr),
if resting_hr: avg_hr_day = COALESCE(EXCLUDED.avg_hr_day, health_metrics.avg_hr_day),
existing.resting_hr = resting_hr avg_stress = COALESCE(EXCLUDED.avg_stress, health_metrics.avg_stress),
if avg_hr: spo2_avg = COALESCE(EXCLUDED.spo2_avg, health_metrics.spo2_avg),
existing.avg_hr_day = avg_hr hrv_nightly_avg = COALESCE(EXCLUDED.hrv_nightly_avg, health_metrics.hrv_nightly_avg),
if avg_stress: hrv_5min_high = COALESCE(EXCLUDED.hrv_5min_high, health_metrics.hrv_5min_high),
existing.avg_stress = avg_stress hrv_status = COALESCE(EXCLUDED.hrv_status, health_metrics.hrv_status),
if spo2_avg: steps = COALESCE(EXCLUDED.steps, health_metrics.steps),
existing.spo2_avg = spo2_avg sleep_duration_s = COALESCE(EXCLUDED.sleep_duration_s, health_metrics.sleep_duration_s),
if data.get("hrv_nightly_avg"): sleep_deep_s = COALESCE(EXCLUDED.sleep_deep_s, health_metrics.sleep_deep_s),
existing.hrv_nightly_avg = data["hrv_nightly_avg"] sleep_light_s = COALESCE(EXCLUDED.sleep_light_s, health_metrics.sleep_light_s),
if data.get("hrv_5min_high"): sleep_rem_s = COALESCE(EXCLUDED.sleep_rem_s, health_metrics.sleep_rem_s),
existing.hrv_5min_high = data["hrv_5min_high"] sleep_awake_s = COALESCE(EXCLUDED.sleep_awake_s, health_metrics.sleep_awake_s)
if data.get("hrv_status"): """), {
existing.hrv_status = data["hrv_status"] "user_id": user_id, "date": date_dt,
if data.get("steps"): "resting_hr": resting_hr, "avg_hr": avg_hr,
existing.steps = data["steps"] "avg_stress": avg_stress, "spo2_avg": spo2_avg,
if sleep_duration_s: "hrv_avg": data.get("hrv_nightly_avg"),
existing.sleep_duration_s = sleep_duration_s "hrv_high": data.get("hrv_5min_high"),
existing.sleep_deep_s = sleep_deep_s "hrv_status": data.get("hrv_status"),
existing.sleep_light_s = sleep_light_s "steps": data.get("steps"),
existing.sleep_rem_s = sleep_rem_s "sleep_dur": sleep_duration_s, "sleep_deep": sleep_deep_s,
existing.sleep_awake_s = sleep_awake_s "sleep_light": sleep_light_s, "sleep_rem": sleep_rem_s,
else: "sleep_awake": sleep_awake_s,
db.add(HealthMetric( })
user_id=user_id,
date=date_dt,
resting_hr=resting_hr,
avg_hr_day=avg_hr,
avg_stress=avg_stress,
spo2_avg=spo2_avg,
hrv_nightly_avg=data.get("hrv_nightly_avg"),
hrv_5min_high=data.get("hrv_5min_high"),
hrv_status=data.get("hrv_status"),
steps=data.get("steps"),
sleep_duration_s=sleep_duration_s,
sleep_deep_s=sleep_deep_s,
sleep_light_s=sleep_light_s,
sleep_rem_s=sleep_rem_s,
sleep_awake_s=sleep_awake_s,
))
db.commit() db.commit()
@@ -406,7 +388,6 @@ def process_garmin_health_zip(zip_path: str, user_id: int):
import json import json
from app.core.database import SyncSessionLocal from app.core.database import SyncSessionLocal
from app.models.user import HealthMetric from app.models.user import HealthMetric
from sqlalchemy import select, func
from datetime import datetime, timezone from datetime import datetime, timezone
with SyncSessionLocal() as db: with SyncSessionLocal() as db:
@@ -425,39 +406,33 @@ def process_garmin_health_zip(zip_path: str, user_id: int):
continue continue
try: try:
date = datetime.fromisoformat(date_str).replace(tzinfo=timezone.utc) date_dt = datetime.fromisoformat(date_str).replace(tzinfo=timezone.utc)
except ValueError: except ValueError:
continue continue
existing = db.execute( from sqlalchemy import text as _text
select(HealthMetric).where( db.execute(_text("""
HealthMetric.user_id == user_id, INSERT INTO health_metrics (user_id, date, resting_hr, steps,
func.date(HealthMetric.date) == date.date(), floors_climbed, active_calories, total_calories, avg_stress, spo2_avg)
) VALUES (:user_id, :date, :resting_hr, :steps,
).scalar_one_or_none() :floors, :active_cal, :total_cal, :stress, :spo2)
ON CONFLICT (user_id, date) DO UPDATE SET
if existing: resting_hr = COALESCE(EXCLUDED.resting_hr, health_metrics.resting_hr),
if data.get("restingHeartRate"): steps = COALESCE(EXCLUDED.steps, health_metrics.steps),
existing.resting_hr = data["restingHeartRate"] floors_climbed = COALESCE(EXCLUDED.floors_climbed, health_metrics.floors_climbed),
if data.get("totalSteps"): active_calories = COALESCE(EXCLUDED.active_calories, health_metrics.active_calories),
existing.steps = data["totalSteps"] total_calories = COALESCE(EXCLUDED.total_calories, health_metrics.total_calories),
if data.get("activeKilocalories"): avg_stress = COALESCE(EXCLUDED.avg_stress, health_metrics.avg_stress),
existing.active_calories = data["activeKilocalories"] spo2_avg = COALESCE(EXCLUDED.spo2_avg, health_metrics.spo2_avg)
if data.get("averageStressLevel"): """), {
existing.avg_stress = data["averageStressLevel"] "user_id": user_id, "date": date_dt,
if data.get("avgSpo2"): "resting_hr": data.get("restingHeartRate"),
existing.spo2_avg = data["avgSpo2"] "steps": data.get("totalSteps"),
else: "floors": data.get("floorsAscended"),
db.add(HealthMetric( "active_cal": data.get("activeKilocalories"),
user_id=user_id, "total_cal": data.get("totalKilocalories"),
date=date, "stress": data.get("averageStressLevel"),
resting_hr=data.get("restingHeartRate"), "spo2": data.get("avgSpo2"),
steps=data.get("totalSteps"), })
floors_climbed=data.get("floorsAscended"),
active_calories=data.get("activeKilocalories"),
total_calories=data.get("totalKilocalories"),
avg_stress=data.get("averageStressLevel"),
spo2_avg=data.get("avgSpo2"),
))
db.commit() db.commit()