Fix wellness parser: field names, sleep epoch durations, HRV, sleep score

The Garmin FIT SDK returns snake_case field names but the parser was
looking for camelCase. Sleep epoch durations were wrong (fixed 30s each
instead of computing from timestamp gaps). HRV is in message 370 not 275
(275 now carries sleep levels in modern firmware). Multiple fixes:

- msg 55: use 'steps', 'heart_rate', 'active_calories' (not numeric keys)
- msg 211: use 'resting_heart_rate' (not msg.get(0))
- msg 227: use 'stress_level_time'/'stress_level_value' for named fields
- msg 132: use snake_case 'stress_level_time'/'stress_level_value'
- msg 275: detect sleep_level field → handle as sleep epoch (modern),
           fall back to HRV handling for older firmware
- msg 370: new handler for modern hrv_status_summary (last_night_average,
           last_night_5_min_high, status)
- msg 346: new handler for sleep_assessment → overall_sleep_score
- msg 21:  new handler for sleep session start/stop events to close the
           last sleep epoch and record sleep_start/sleep_end timestamps
- Sleep duration: computed from epoch timestamp gaps instead of 30s/epoch
- Celery task SQL: add sleep_score, sleep_start, sleep_end to INSERT/UPDATE;
  use GREATEST for total_calories so most-complete value wins across files

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

backend/app/services/wellness_parser.py

@@ -1,6 +1,11 @@
 """
 Garmin wellness FIT file parser using the official Garmin FIT Python SDK.
-SDK field names are camelCase as per the SDK documentation.
+The SDK with convert_types_to_strings=True returns snake_case field names.
+
+Sleep stages: message 275 (modern) or 269 (older) each carry a start timestamp
+and a stage name. Duration of each stage = gap to the next stage's timestamp.
+The sleep session stop time (from event message 21, event_type='stop') closes
+the last stage.
 """
 from datetime import datetime, timezone, date
 from typing import Optional
@@ -8,6 +13,7 @@ from garmin_fit_sdk import Decoder, Stream
 
 
 FIT_EPOCH_S = 631065600
+SLEEP_LEVEL_MAP = {"unmeasurable": 0, "awake": 1, "light": 2, "deep": 3, "rem": 4}
 
 
 def _fit_ts(raw) -> Optional[datetime]:
@@ -35,12 +41,21 @@ def _to_date(val) -> Optional[date]:
     return None
 
 
+def _to_dt(val) -> Optional[datetime]:
+    if isinstance(val, datetime):
+        return val.replace(tzinfo=timezone.utc) if val.tzinfo is None else val
+    if isinstance(val, (int, float)):
+        return _fit_ts(val)
+    return None
+
+
 def parse_wellness_fit(file_path: str) -> dict:
     """
     Parse a Garmin wellness/monitoring FIT file.
     Returns {"days": {date: metrics_dict}, "error": str|None}
     """
     daily = {}
+    last_date_seen = [None]
 
     def ensure_day(d: date) -> dict:
         if d not in daily:
@@ -48,154 +63,212 @@ def parse_wellness_fit(file_path: str) -> dict:
                 "heart_rates": [],
                 "stress_values": [],
                 "spo2_readings": [],
-                "sleep_levels": [],
+                # Each entry: (datetime, level_int) — duration computed from gaps
+                "sleep_epochs": [],
+                "sleep_start": None,
+                "sleep_end": None,
                 "steps": None,
                 "floors_climbed": None,
                 "active_calories": None,
-                "total_calories": None,
+                "bmr": None,
                 "resting_hr": None,
                 "hrv_nightly_avg": None,
                 "hrv_5min_high": None,
                 "hrv_status": None,
+                "sleep_score": None,
             }
         return daily[d]
 
+    def _add_sleep_epoch(ts: datetime, level_raw):
+        d = _to_date(ts)
+        if not d:
+            return
+        last_date_seen[0] = d
+        if isinstance(level_raw, str):
+            level = SLEEP_LEVEL_MAP.get(level_raw.lower())
+        else:
+            level = level_raw
+        if level is not None:
+            ensure_day(d)["sleep_epochs"].append((ts, int(level)))
+
     def listener(mesg_num: int, msg: dict):
 
-        # monitoring_info (147)
+        # ── monitoring_info (147) - older firmware ─────────────────────────
         if mesg_num == 147:
-            d = _to_date(msg.get("timestamp") or msg.get("localTimestamp"))
+            d = _to_date(msg.get("timestamp") or msg.get("local_timestamp"))
-            rhr = msg.get("restingHeartRate")
+            rhr = msg.get("resting_heart_rate")
             if d and rhr and 20 < rhr < 120:
+                last_date_seen[0] = d
                 ensure_day(d)["resting_hr"] = int(rhr)
 
-        # monitoring (148)
+        # ── monitoring (148) - older firmware ──────────────────────────────
         elif mesg_num == 148:
-            d = _to_date(msg.get("timestamp") or msg.get("localTimestamp"))
+            d = _to_date(msg.get("timestamp") or msg.get("local_timestamp"))
             if not d:
                 return
+            last_date_seen[0] = d
             entry = ensure_day(d)
-            hr = msg.get("heartRate")
+            hr = msg.get("heart_rate")
             if hr and 20 < hr < 250:
                 entry["heart_rates"].append(int(hr))
             steps = msg.get("steps") or msg.get("cycles")
             if steps and steps > 0:
                 entry["steps"] = max(entry["steps"] or 0, int(steps))
-            stress = msg.get("stressLevelValue")
+            stress = msg.get("stress_level_value")
             if stress is not None and stress >= 0:
                 entry["stress_values"].append(int(stress))
 
-        # hrv_status_summary (275)
+        # ── monitoring (55) - modern, per-interval running totals ──────────
-        elif mesg_num == 275:
+        elif mesg_num == 55:
             d = _to_date(msg.get("timestamp"))
             if not d:
                 return
+            last_date_seen[0] = d
             entry = ensure_day(d)
-            for key in ("weeklyAverage", "lastNightAvg", "hrvNightlyAvg"):
+            hr = msg.get("heart_rate")
-                v = msg.get(key)
+            if hr and 20 < hr < 250:
-                if v and v > 0:
+                entry["heart_rates"].append(int(hr))
-                    entry["hrv_nightly_avg"] = float(v)
+            steps = msg.get("steps")
-                    break
+            if steps and steps > 0:
-            high = msg.get("lastNight5MinHigh")
+                entry["steps"] = max(entry["steps"] or 0, int(steps))
-            if high:
+            active_cal = msg.get("active_calories")
-                entry["hrv_5min_high"] = float(high)
+            if active_cal and active_cal > 0:
-            status = msg.get("hrvStatus")
+                entry["active_calories"] = max(entry["active_calories"] or 0, float(active_cal))
+            ascent = msg.get("ascent")
+            if ascent and ascent > 0:
+                # Garmin counts 1 floor ≈ 3 m of ascent
+                floors = max(1, round(float(ascent) / 3))
+                entry["floors_climbed"] = max(entry["floors_climbed"] or 0, floors)
+
+        # ── monitoring_info (103) - calibration; carries BMR ───────────────
+        elif mesg_num == 103:
+            d = _to_date(msg.get("timestamp"))
+            if not d:
+                return
+            last_date_seen[0] = d
+            bmr = msg.get("resting_metabolic_rate")
+            if bmr and bmr > 0:
+                ensure_day(d)["bmr"] = int(bmr)
+
+        # ── hrv_status_summary (370) - modern HRV ─────────────────────────
+        elif mesg_num == 370:
+            d = _to_date(msg.get("timestamp"))
+            if not d:
+                return
+            last_date_seen[0] = d
+            entry = ensure_day(d)
+            hrv_avg = msg.get("last_night_average")
+            if hrv_avg and hrv_avg > 0:
+                entry["hrv_nightly_avg"] = float(hrv_avg)
+            hrv_high = msg.get("last_night_5_min_high")
+            if hrv_high and hrv_high > 0:
+                entry["hrv_5min_high"] = float(hrv_high)
+            status = msg.get("status")
             if status:
                 entry["hrv_status"] = str(status)
 
-        # stress_level (132)
+        # ── message 275 - sleep epochs (modern) or HRV (older firmware) ───
-        elif mesg_num == 132:
+        elif mesg_num == 275:
-            d = _to_date(msg.get("stressLevelTime") or msg.get("timestamp"))
+            sleep_level = msg.get("sleep_level")
+            ts = _to_dt(msg.get("timestamp"))
+            if sleep_level is not None and ts:
+                _add_sleep_epoch(ts, sleep_level)
+            elif ts:
+                # Older firmware: HRV summary in message 275
+                d = _to_date(ts)
+                if d:
+                    last_date_seen[0] = d
+                    entry = ensure_day(d)
+                    for key in ("weekly_average", "last_night_avg", "hrv_nightly_avg"):
+                        v = msg.get(key)
+                        if v and v > 0:
+                            entry["hrv_nightly_avg"] = float(v)
+                            break
+                    high = msg.get("last_night_5_min_high")
+                    if high:
+                        entry["hrv_5min_high"] = float(high)
+                    status = msg.get("hrv_status") or msg.get("status")
+                    if status:
+                        entry["hrv_status"] = str(status)
+
+        # ── sleep_level (269) - older firmware sleep epochs ────────────────
+        elif mesg_num == 269:
+            ts = _to_dt(msg.get("timestamp"))
+            level = msg.get("sleep_level")
+            if ts and level is not None:
+                _add_sleep_epoch(ts, level)
+
+        # ── event (21) - sleep session start / stop ────────────────────────
+        elif mesg_num == 21:
+            ts = _to_dt(msg.get("timestamp"))
+            if not ts:
+                return
+            d = _to_date(ts)
             if not d:
                 return
-            stress = msg.get("stressLevelValue")
+            event_type = msg.get("event_type")
+            if event_type == "start":
+                last_date_seen[0] = d
+                ensure_day(d)["sleep_start"] = ts
+            elif event_type == "stop":
+                last_date_seen[0] = d
+                ensure_day(d)["sleep_end"] = ts
+
+        # ── sleep_assessment (346) - overall sleep score, no timestamp ────
+        elif mesg_num == 346:
+            d = last_date_seen[0]
+            if not d:
+                return
+            score = msg.get("overall_sleep_score")
+            if score and score > 0:
+                ensure_day(d)["sleep_score"] = int(score)
+
+        # ── stress_level (132) ─────────────────────────────────────────────
+        elif mesg_num == 132:
+            d = _to_date(msg.get("stress_level_time") or msg.get("timestamp"))
+            if not d:
+                return
+            last_date_seen[0] = d
+            stress = msg.get("stress_level_value")
             if stress is not None and stress >= 0:
                 ensure_day(d)["stress_values"].append(int(stress))
 
-        # spo2_data (258)
+        # ── spo2_data (258) ────────────────────────────────────────────────
         elif mesg_num == 258:
             d = _to_date(msg.get("timestamp"))
             if not d:
                 return
-            spo2 = msg.get("spo2Percent") or msg.get("readingSpo2")
+            last_date_seen[0] = d
+            spo2 = msg.get("spo2_percent") or msg.get("reading_spo2")
             if spo2 and 50 < spo2 <= 100:
                 ensure_day(d)["spo2_readings"].append(float(spo2))
 
-        # sleep_level (269)
+        # ── per-minute stress + HR (227) proprietary ───────────────────────
-        elif mesg_num == 269:
+        elif mesg_num == 227:
+            d = _to_date(msg.get("stress_level_time") or msg.get("timestamp"))
+            if not d:
+                return
+            last_date_seen[0] = d
+            entry = ensure_day(d)
+            hr_raw = msg.get(2)
+            if hr_raw and isinstance(hr_raw, (int, float)) and 20 < hr_raw < 250:
+                entry["heart_rates"].append(int(hr_raw))
+            stress = msg.get("stress_level_value")
+            if stress is None:
+                stress = msg.get(0)
+            if stress is not None and isinstance(stress, (int, float)) and stress >= 0:
+                entry["stress_values"].append(int(stress))
+
+        # ── daily resting HR (211) proprietary ─────────────────────────────
+        elif mesg_num == 211:
             d = _to_date(msg.get("timestamp"))
             if not d:
                 return
-            level = msg.get("sleepLevel")
+            last_date_seen[0] = d
-            if level is not None:
-                if isinstance(level, str):
-                    level_map = {"unmeasurable": 0, "awake": 1, "light": 2, "deep": 3, "rem": 4}
-                    level = level_map.get(level.lower())
-                if level is not None:
-                    ensure_day(d)["sleep_levels"].append(int(level))
-
-        # Proprietary 227: per-minute stress + HR
-        elif mesg_num == 227:
-            ts_raw = msg.get(1) or msg.get("1")
-            hr_raw = msg.get(2) or msg.get("2")
-            stress_raw = msg.get(0) or msg.get("0")
-            d = _to_date(ts_raw)
-            if not d:
-                return
             entry = ensure_day(d)
-            if hr_raw and isinstance(hr_raw, (int, float)) and 20 < hr_raw < 250:
+            rhr = msg.get("resting_heart_rate") or msg.get("current_day_resting_heart_rate")
-                entry["heart_rates"].append(int(hr_raw))
-            if stress_raw is not None and isinstance(stress_raw, (int, float)) and stress_raw >= 0:
-                entry["stress_values"].append(int(stress_raw))
-
-        # Proprietary 103: daily totals
-        elif mesg_num == 103:
-            ts_raw = msg.get(253) or msg.get("253") or msg.get("timestamp")
-            d = _to_date(ts_raw)
-            if not d:
-                return
-            entry = ensure_day(d)
-            steps = msg.get(3) or msg.get("3")
-            if steps and isinstance(steps, (int, float)) and steps > 0:
-                entry["steps"] = int(steps)
-            floors = msg.get(4) or msg.get("4")
-            if floors and isinstance(floors, (int, float)) and floors > 0:
-                f = float(floors)
-                entry["floors_climbed"] = round(f / 100 if f > 1000 else f, 1)
-            active_cal = msg.get(5) or msg.get("5")
-            if active_cal and isinstance(active_cal, (int, float)) and active_cal > 0:
-                entry["active_calories"] = float(active_cal)
-            total_cal = msg.get(7) or msg.get("7")
-            if total_cal and isinstance(total_cal, (int, float)) and total_cal > 0:
-                entry["total_calories"] = float(total_cal)
-
-        # Proprietary 211: resting HR + HRV
-        elif mesg_num == 211:
-            ts_raw = msg.get(253) or msg.get("253") or msg.get("timestamp")
-            d = _to_date(ts_raw)
-            if not d:
-                return
-            entry = ensure_day(d)
-            rhr = msg.get(0) or msg.get("0")
             if rhr and isinstance(rhr, (int, float)) and 20 < rhr < 120:
                 entry["resting_hr"] = int(rhr)
-            hrv = msg.get(1) or msg.get("1")
-            if hrv and isinstance(hrv, (int, float)) and 5 < hrv < 300:
-                entry["hrv_nightly_avg"] = float(hrv)
-
-        # Proprietary 55: activity accumulations
-        elif mesg_num == 55:
-            ts_raw = msg.get(253) or msg.get("253") or msg.get("timestamp")
-            d = _to_date(ts_raw)
-            if not d:
-                return
-            entry = ensure_day(d)
-            steps = msg.get(2) or msg.get("2")
-            if steps and isinstance(steps, (int, float)) and steps > 0:
-                entry["steps"] = max(entry["steps"] or 0, int(steps))
-            hr = msg.get(19) or msg.get("19")
-            if hr and isinstance(hr, (int, float)) and 20 < hr < 250:
-                entry["heart_rates"].append(int(hr))
 
     try:
         stream = Stream.from_file(file_path)
@@ -218,22 +291,42 @@ def parse_wellness_fit(file_path: str) -> dict:
         hrs = data.pop("heart_rates", [])
         stresses = data.pop("stress_values", [])
         spo2s = data.pop("spo2_readings", [])
-        sleep_levels = data.pop("sleep_levels", [])
+        sleep_epochs = data.pop("sleep_epochs", [])
+        sleep_end_ts = data.pop("sleep_end", None)
+        sleep_start_ts = data.pop("sleep_start", None)
 
         avg_hr = round(sum(hrs) / len(hrs), 1) if hrs else None
         max_hr = max(hrs) if hrs else None
         avg_stress = round(sum(s for s in stresses if s >= 0) / len(stresses), 1) if stresses else None
         spo2_avg = round(sum(spo2s) / len(spo2s), 1) if spo2s else None
 
-        if sleep_levels:
+        # Compute sleep stage durations from epoch timestamps
-            sleep_deep_s = sum(30 for l in sleep_levels if l == 3) or None
+        if sleep_epochs:
-            sleep_light_s = sum(30 for l in sleep_levels if l == 2) or None
+            epochs_sorted = sorted(sleep_epochs, key=lambda x: x[0])
-            sleep_rem_s = sum(30 for l in sleep_levels if l == 4) or None
+            level_secs = {1: 0, 2: 0, 3: 0, 4: 0}  # awake, light, deep, rem
-            sleep_awake_s = sum(30 for l in sleep_levels if l == 1) or None
+            for i, (ts, level) in enumerate(epochs_sorted):
-            sleep_duration_s = (sleep_deep_s or 0) + (sleep_light_s or 0) + (sleep_rem_s or 0) or None
+                if i + 1 < len(epochs_sorted):
+                    next_ts = epochs_sorted[i + 1][0]
+                elif sleep_end_ts:
+                    next_ts = sleep_end_ts
+                else:
+                    continue
+                dur = (next_ts - ts).total_seconds()
+                if level in level_secs and dur > 0:
+                    level_secs[level] += dur
+            sleep_deep_s = level_secs[3] or None
+            sleep_light_s = level_secs[2] or None
+            sleep_rem_s = level_secs[4] or None
+            sleep_awake_s = level_secs[1] or None
+            sleep_duration_s = (level_secs[2] + level_secs[3] + level_secs[4]) or None
         else:
             sleep_deep_s = sleep_light_s = sleep_rem_s = sleep_awake_s = sleep_duration_s = None
 
+        active_cal = data.get("active_calories")
+        bmr = data.get("bmr")
+        # Require active_cal so we don't store BMR-only as "total" calories
+        total_cal = float(bmr + active_cal) if (bmr and active_cal) else None
+
         result[day_date] = {
             "resting_hr": data.get("resting_hr"),
             "avg_hr_day": avg_hr,
@@ -245,13 +338,16 @@ def parse_wellness_fit(file_path: str) -> dict:
             "hrv_status": data.get("hrv_status"),
             "steps": data.get("steps"),
             "floors_climbed": data.get("floors_climbed"),
-            "active_calories": data.get("active_calories"),
+            "active_calories": active_cal,
-            "total_calories": data.get("total_calories"),
+            "total_calories": total_cal,
             "sleep_duration_s": sleep_duration_s,
             "sleep_deep_s": sleep_deep_s,
             "sleep_light_s": sleep_light_s,
             "sleep_rem_s": sleep_rem_s,
             "sleep_awake_s": sleep_awake_s,
+            "sleep_score": data.get("sleep_score"),
+            "sleep_start": sleep_start_ts,
+            "sleep_end": sleep_end_ts,
         }
 
-    return {"days": result, "error": None}
+    return {"days": result, "error": None}

backend/app/workers/tasks.py

@@ -215,11 +215,13 @@ def parse_wellness_fit(file_path: str, user_id: int):
                 INSERT INTO health_metrics (user_id, date, resting_hr, avg_hr_day, max_hr_day,
                     avg_stress, spo2_avg, hrv_nightly_avg, hrv_5min_high, hrv_status,
                     steps, floors_climbed, active_calories, total_calories,
-                    sleep_duration_s, sleep_deep_s, sleep_light_s, sleep_rem_s, sleep_awake_s)
+                    sleep_duration_s, sleep_deep_s, sleep_light_s, sleep_rem_s, sleep_awake_s,
+                    sleep_score, sleep_start, sleep_end)
                 VALUES (:user_id, :date, :resting_hr, :avg_hr, :max_hr,
                     :avg_stress, :spo2_avg, :hrv_avg, :hrv_high, :hrv_status,
                     :steps, :floors, :active_cal, :total_cal,
-                    :sleep_dur, :sleep_deep, :sleep_light, :sleep_rem, :sleep_awake)
+                    :sleep_dur, :sleep_deep, :sleep_light, :sleep_rem, :sleep_awake,
+                    :sleep_score, :sleep_start, :sleep_end)
                 ON CONFLICT (user_id, date) DO UPDATE SET
                     resting_hr       = COALESCE(EXCLUDED.resting_hr, health_metrics.resting_hr),
                     avg_hr_day       = COALESCE(EXCLUDED.avg_hr_day, health_metrics.avg_hr_day),
@@ -232,12 +234,15 @@ def parse_wellness_fit(file_path: str, user_id: int):
                     steps            = COALESCE(EXCLUDED.steps, health_metrics.steps),
                     floors_climbed   = COALESCE(EXCLUDED.floors_climbed, health_metrics.floors_climbed),
                     active_calories  = COALESCE(EXCLUDED.active_calories, health_metrics.active_calories),
-                    total_calories   = COALESCE(EXCLUDED.total_calories, health_metrics.total_calories),
+                    total_calories   = GREATEST(EXCLUDED.total_calories, health_metrics.total_calories),
                     sleep_duration_s = COALESCE(EXCLUDED.sleep_duration_s, health_metrics.sleep_duration_s),
                     sleep_deep_s     = COALESCE(EXCLUDED.sleep_deep_s, health_metrics.sleep_deep_s),
                     sleep_light_s    = COALESCE(EXCLUDED.sleep_light_s, health_metrics.sleep_light_s),
                     sleep_rem_s      = COALESCE(EXCLUDED.sleep_rem_s, health_metrics.sleep_rem_s),
-                    sleep_awake_s    = COALESCE(EXCLUDED.sleep_awake_s, health_metrics.sleep_awake_s)
+                    sleep_awake_s    = COALESCE(EXCLUDED.sleep_awake_s, health_metrics.sleep_awake_s),
+                    sleep_score      = COALESCE(EXCLUDED.sleep_score, health_metrics.sleep_score),
+                    sleep_start      = COALESCE(EXCLUDED.sleep_start, health_metrics.sleep_start),
+                    sleep_end        = COALESCE(EXCLUDED.sleep_end, health_metrics.sleep_end)
             """), {
                 "user_id": user_id, "date": date_dt,
                 "resting_hr": data.get("resting_hr"),
@@ -257,6 +262,9 @@ def parse_wellness_fit(file_path: str, user_id: int):
                 "sleep_light": data.get("sleep_light_s"),
                 "sleep_rem": data.get("sleep_rem_s"),
                 "sleep_awake": data.get("sleep_awake_s"),
+                "sleep_score": data.get("sleep_score"),
+                "sleep_start": data.get("sleep_start"),
+                "sleep_end": data.get("sleep_end"),
             })
         db.commit()