Fix wellness parser: field names, sleep epoch durations, HRV, sleep score
Build and push images / validate (push) Successful in 2s
Build and push images / build-backend (push) Successful in 6s
Build and push images / build-worker (push) Successful in 6s
Build and push images / build-frontend (push) Successful in 5s

The Garmin FIT SDK returns snake_case field names but the parser was
looking for camelCase. Sleep epoch durations were wrong (fixed 30s each
instead of computing from timestamp gaps). HRV is in message 370 not 275
(275 now carries sleep levels in modern firmware). Multiple fixes:

- msg 55: use 'steps', 'heart_rate', 'active_calories' (not numeric keys)
- msg 211: use 'resting_heart_rate' (not msg.get(0))
- msg 227: use 'stress_level_time'/'stress_level_value' for named fields
- msg 132: use snake_case 'stress_level_time'/'stress_level_value'
- msg 275: detect sleep_level field → handle as sleep epoch (modern),
           fall back to HRV handling for older firmware
- msg 370: new handler for modern hrv_status_summary (last_night_average,
           last_night_5_min_high, status)
- msg 346: new handler for sleep_assessment → overall_sleep_score
- msg 21:  new handler for sleep session start/stop events to close the
           last sleep epoch and record sleep_start/sleep_end timestamps
- Sleep duration: computed from epoch timestamp gaps instead of 30s/epoch
- Celery task SQL: add sleep_score, sleep_start, sleep_end to INSERT/UPDATE;
  use GREATEST for total_calories so most-complete value wins across files

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-06-06 23:33:50 +01:00
parent 95f704cb54
commit c3637fa3fa
2 changed files with 214 additions and 110 deletions
+201 -105
View File
@@ -1,6 +1,11 @@
"""
Garmin wellness FIT file parser using the official Garmin FIT Python SDK.
SDK field names are camelCase as per the SDK documentation.
The SDK with convert_types_to_strings=True returns snake_case field names.
Sleep stages: message 275 (modern) or 269 (older) each carry a start timestamp
and a stage name. Duration of each stage = gap to the next stage's timestamp.
The sleep session stop time (from event message 21, event_type='stop') closes
the last stage.
"""
from datetime import datetime, timezone, date
from typing import Optional
@@ -8,6 +13,7 @@ from garmin_fit_sdk import Decoder, Stream
FIT_EPOCH_S = 631065600
SLEEP_LEVEL_MAP = {"unmeasurable": 0, "awake": 1, "light": 2, "deep": 3, "rem": 4}
def _fit_ts(raw) -> Optional[datetime]:
@@ -35,12 +41,21 @@ def _to_date(val) -> Optional[date]:
return None
def _to_dt(val) -> Optional[datetime]:
if isinstance(val, datetime):
return val.replace(tzinfo=timezone.utc) if val.tzinfo is None else val
if isinstance(val, (int, float)):
return _fit_ts(val)
return None
def parse_wellness_fit(file_path: str) -> dict:
"""
Parse a Garmin wellness/monitoring FIT file.
Returns {"days": {date: metrics_dict}, "error": str|None}
"""
daily = {}
last_date_seen = [None]
def ensure_day(d: date) -> dict:
if d not in daily:
@@ -48,154 +63,212 @@ def parse_wellness_fit(file_path: str) -> dict:
"heart_rates": [],
"stress_values": [],
"spo2_readings": [],
"sleep_levels": [],
# Each entry: (datetime, level_int) — duration computed from gaps
"sleep_epochs": [],
"sleep_start": None,
"sleep_end": None,
"steps": None,
"floors_climbed": None,
"active_calories": None,
"total_calories": None,
"bmr": None,
"resting_hr": None,
"hrv_nightly_avg": None,
"hrv_5min_high": None,
"hrv_status": None,
"sleep_score": None,
}
return daily[d]
def _add_sleep_epoch(ts: datetime, level_raw):
d = _to_date(ts)
if not d:
return
last_date_seen[0] = d
if isinstance(level_raw, str):
level = SLEEP_LEVEL_MAP.get(level_raw.lower())
else:
level = level_raw
if level is not None:
ensure_day(d)["sleep_epochs"].append((ts, int(level)))
def listener(mesg_num: int, msg: dict):
# monitoring_info (147)
# ── monitoring_info (147) - older firmware ─────────────────────────
if mesg_num == 147:
d = _to_date(msg.get("timestamp") or msg.get("localTimestamp"))
rhr = msg.get("restingHeartRate")
d = _to_date(msg.get("timestamp") or msg.get("local_timestamp"))
rhr = msg.get("resting_heart_rate")
if d and rhr and 20 < rhr < 120:
last_date_seen[0] = d
ensure_day(d)["resting_hr"] = int(rhr)
# monitoring (148)
# ── monitoring (148) - older firmware ──────────────────────────────
elif mesg_num == 148:
d = _to_date(msg.get("timestamp") or msg.get("localTimestamp"))
d = _to_date(msg.get("timestamp") or msg.get("local_timestamp"))
if not d:
return
last_date_seen[0] = d
entry = ensure_day(d)
hr = msg.get("heartRate")
hr = msg.get("heart_rate")
if hr and 20 < hr < 250:
entry["heart_rates"].append(int(hr))
steps = msg.get("steps") or msg.get("cycles")
if steps and steps > 0:
entry["steps"] = max(entry["steps"] or 0, int(steps))
stress = msg.get("stressLevelValue")
stress = msg.get("stress_level_value")
if stress is not None and stress >= 0:
entry["stress_values"].append(int(stress))
# hrv_status_summary (275)
elif mesg_num == 275:
# ── monitoring (55) - modern, per-interval running totals ──────────
elif mesg_num == 55:
d = _to_date(msg.get("timestamp"))
if not d:
return
last_date_seen[0] = d
entry = ensure_day(d)
for key in ("weeklyAverage", "lastNightAvg", "hrvNightlyAvg"):
v = msg.get(key)
if v and v > 0:
entry["hrv_nightly_avg"] = float(v)
break
high = msg.get("lastNight5MinHigh")
if high:
entry["hrv_5min_high"] = float(high)
status = msg.get("hrvStatus")
hr = msg.get("heart_rate")
if hr and 20 < hr < 250:
entry["heart_rates"].append(int(hr))
steps = msg.get("steps")
if steps and steps > 0:
entry["steps"] = max(entry["steps"] or 0, int(steps))
active_cal = msg.get("active_calories")
if active_cal and active_cal > 0:
entry["active_calories"] = max(entry["active_calories"] or 0, float(active_cal))
ascent = msg.get("ascent")
if ascent and ascent > 0:
# Garmin counts 1 floor ≈ 3 m of ascent
floors = max(1, round(float(ascent) / 3))
entry["floors_climbed"] = max(entry["floors_climbed"] or 0, floors)
# ── monitoring_info (103) - calibration; carries BMR ───────────────
elif mesg_num == 103:
d = _to_date(msg.get("timestamp"))
if not d:
return
last_date_seen[0] = d
bmr = msg.get("resting_metabolic_rate")
if bmr and bmr > 0:
ensure_day(d)["bmr"] = int(bmr)
# ── hrv_status_summary (370) - modern HRV ─────────────────────────
elif mesg_num == 370:
d = _to_date(msg.get("timestamp"))
if not d:
return
last_date_seen[0] = d
entry = ensure_day(d)
hrv_avg = msg.get("last_night_average")
if hrv_avg and hrv_avg > 0:
entry["hrv_nightly_avg"] = float(hrv_avg)
hrv_high = msg.get("last_night_5_min_high")
if hrv_high and hrv_high > 0:
entry["hrv_5min_high"] = float(hrv_high)
status = msg.get("status")
if status:
entry["hrv_status"] = str(status)
# stress_level (132)
elif mesg_num == 132:
d = _to_date(msg.get("stressLevelTime") or msg.get("timestamp"))
# ── message 275 - sleep epochs (modern) or HRV (older firmware) ───
elif mesg_num == 275:
sleep_level = msg.get("sleep_level")
ts = _to_dt(msg.get("timestamp"))
if sleep_level is not None and ts:
_add_sleep_epoch(ts, sleep_level)
elif ts:
# Older firmware: HRV summary in message 275
d = _to_date(ts)
if d:
last_date_seen[0] = d
entry = ensure_day(d)
for key in ("weekly_average", "last_night_avg", "hrv_nightly_avg"):
v = msg.get(key)
if v and v > 0:
entry["hrv_nightly_avg"] = float(v)
break
high = msg.get("last_night_5_min_high")
if high:
entry["hrv_5min_high"] = float(high)
status = msg.get("hrv_status") or msg.get("status")
if status:
entry["hrv_status"] = str(status)
# ── sleep_level (269) - older firmware sleep epochs ────────────────
elif mesg_num == 269:
ts = _to_dt(msg.get("timestamp"))
level = msg.get("sleep_level")
if ts and level is not None:
_add_sleep_epoch(ts, level)
# ── event (21) - sleep session start / stop ────────────────────────
elif mesg_num == 21:
ts = _to_dt(msg.get("timestamp"))
if not ts:
return
d = _to_date(ts)
if not d:
return
stress = msg.get("stressLevelValue")
event_type = msg.get("event_type")
if event_type == "start":
last_date_seen[0] = d
ensure_day(d)["sleep_start"] = ts
elif event_type == "stop":
last_date_seen[0] = d
ensure_day(d)["sleep_end"] = ts
# ── sleep_assessment (346) - overall sleep score, no timestamp ────
elif mesg_num == 346:
d = last_date_seen[0]
if not d:
return
score = msg.get("overall_sleep_score")
if score and score > 0:
ensure_day(d)["sleep_score"] = int(score)
# ── stress_level (132) ─────────────────────────────────────────────
elif mesg_num == 132:
d = _to_date(msg.get("stress_level_time") or msg.get("timestamp"))
if not d:
return
last_date_seen[0] = d
stress = msg.get("stress_level_value")
if stress is not None and stress >= 0:
ensure_day(d)["stress_values"].append(int(stress))
# spo2_data (258)
# ── spo2_data (258) ────────────────────────────────────────────────
elif mesg_num == 258:
d = _to_date(msg.get("timestamp"))
if not d:
return
spo2 = msg.get("spo2Percent") or msg.get("readingSpo2")
last_date_seen[0] = d
spo2 = msg.get("spo2_percent") or msg.get("reading_spo2")
if spo2 and 50 < spo2 <= 100:
ensure_day(d)["spo2_readings"].append(float(spo2))
# sleep_level (269)
elif mesg_num == 269:
# ── per-minute stress + HR (227) proprietary ───────────────────────
elif mesg_num == 227:
d = _to_date(msg.get("stress_level_time") or msg.get("timestamp"))
if not d:
return
last_date_seen[0] = d
entry = ensure_day(d)
hr_raw = msg.get(2)
if hr_raw and isinstance(hr_raw, (int, float)) and 20 < hr_raw < 250:
entry["heart_rates"].append(int(hr_raw))
stress = msg.get("stress_level_value")
if stress is None:
stress = msg.get(0)
if stress is not None and isinstance(stress, (int, float)) and stress >= 0:
entry["stress_values"].append(int(stress))
# ── daily resting HR (211) proprietary ─────────────────────────────
elif mesg_num == 211:
d = _to_date(msg.get("timestamp"))
if not d:
return
level = msg.get("sleepLevel")
if level is not None:
if isinstance(level, str):
level_map = {"unmeasurable": 0, "awake": 1, "light": 2, "deep": 3, "rem": 4}
level = level_map.get(level.lower())
if level is not None:
ensure_day(d)["sleep_levels"].append(int(level))
# Proprietary 227: per-minute stress + HR
elif mesg_num == 227:
ts_raw = msg.get(1) or msg.get("1")
hr_raw = msg.get(2) or msg.get("2")
stress_raw = msg.get(0) or msg.get("0")
d = _to_date(ts_raw)
if not d:
return
last_date_seen[0] = d
entry = ensure_day(d)
if hr_raw and isinstance(hr_raw, (int, float)) and 20 < hr_raw < 250:
entry["heart_rates"].append(int(hr_raw))
if stress_raw is not None and isinstance(stress_raw, (int, float)) and stress_raw >= 0:
entry["stress_values"].append(int(stress_raw))
# Proprietary 103: daily totals
elif mesg_num == 103:
ts_raw = msg.get(253) or msg.get("253") or msg.get("timestamp")
d = _to_date(ts_raw)
if not d:
return
entry = ensure_day(d)
steps = msg.get(3) or msg.get("3")
if steps and isinstance(steps, (int, float)) and steps > 0:
entry["steps"] = int(steps)
floors = msg.get(4) or msg.get("4")
if floors and isinstance(floors, (int, float)) and floors > 0:
f = float(floors)
entry["floors_climbed"] = round(f / 100 if f > 1000 else f, 1)
active_cal = msg.get(5) or msg.get("5")
if active_cal and isinstance(active_cal, (int, float)) and active_cal > 0:
entry["active_calories"] = float(active_cal)
total_cal = msg.get(7) or msg.get("7")
if total_cal and isinstance(total_cal, (int, float)) and total_cal > 0:
entry["total_calories"] = float(total_cal)
# Proprietary 211: resting HR + HRV
elif mesg_num == 211:
ts_raw = msg.get(253) or msg.get("253") or msg.get("timestamp")
d = _to_date(ts_raw)
if not d:
return
entry = ensure_day(d)
rhr = msg.get(0) or msg.get("0")
rhr = msg.get("resting_heart_rate") or msg.get("current_day_resting_heart_rate")
if rhr and isinstance(rhr, (int, float)) and 20 < rhr < 120:
entry["resting_hr"] = int(rhr)
hrv = msg.get(1) or msg.get("1")
if hrv and isinstance(hrv, (int, float)) and 5 < hrv < 300:
entry["hrv_nightly_avg"] = float(hrv)
# Proprietary 55: activity accumulations
elif mesg_num == 55:
ts_raw = msg.get(253) or msg.get("253") or msg.get("timestamp")
d = _to_date(ts_raw)
if not d:
return
entry = ensure_day(d)
steps = msg.get(2) or msg.get("2")
if steps and isinstance(steps, (int, float)) and steps > 0:
entry["steps"] = max(entry["steps"] or 0, int(steps))
hr = msg.get(19) or msg.get("19")
if hr and isinstance(hr, (int, float)) and 20 < hr < 250:
entry["heart_rates"].append(int(hr))
try:
stream = Stream.from_file(file_path)
@@ -218,22 +291,42 @@ def parse_wellness_fit(file_path: str) -> dict:
hrs = data.pop("heart_rates", [])
stresses = data.pop("stress_values", [])
spo2s = data.pop("spo2_readings", [])
sleep_levels = data.pop("sleep_levels", [])
sleep_epochs = data.pop("sleep_epochs", [])
sleep_end_ts = data.pop("sleep_end", None)
sleep_start_ts = data.pop("sleep_start", None)
avg_hr = round(sum(hrs) / len(hrs), 1) if hrs else None
max_hr = max(hrs) if hrs else None
avg_stress = round(sum(s for s in stresses if s >= 0) / len(stresses), 1) if stresses else None
spo2_avg = round(sum(spo2s) / len(spo2s), 1) if spo2s else None
if sleep_levels:
sleep_deep_s = sum(30 for l in sleep_levels if l == 3) or None
sleep_light_s = sum(30 for l in sleep_levels if l == 2) or None
sleep_rem_s = sum(30 for l in sleep_levels if l == 4) or None
sleep_awake_s = sum(30 for l in sleep_levels if l == 1) or None
sleep_duration_s = (sleep_deep_s or 0) + (sleep_light_s or 0) + (sleep_rem_s or 0) or None
# Compute sleep stage durations from epoch timestamps
if sleep_epochs:
epochs_sorted = sorted(sleep_epochs, key=lambda x: x[0])
level_secs = {1: 0, 2: 0, 3: 0, 4: 0} # awake, light, deep, rem
for i, (ts, level) in enumerate(epochs_sorted):
if i + 1 < len(epochs_sorted):
next_ts = epochs_sorted[i + 1][0]
elif sleep_end_ts:
next_ts = sleep_end_ts
else:
continue
dur = (next_ts - ts).total_seconds()
if level in level_secs and dur > 0:
level_secs[level] += dur
sleep_deep_s = level_secs[3] or None
sleep_light_s = level_secs[2] or None
sleep_rem_s = level_secs[4] or None
sleep_awake_s = level_secs[1] or None
sleep_duration_s = (level_secs[2] + level_secs[3] + level_secs[4]) or None
else:
sleep_deep_s = sleep_light_s = sleep_rem_s = sleep_awake_s = sleep_duration_s = None
active_cal = data.get("active_calories")
bmr = data.get("bmr")
# Require active_cal so we don't store BMR-only as "total" calories
total_cal = float(bmr + active_cal) if (bmr and active_cal) else None
result[day_date] = {
"resting_hr": data.get("resting_hr"),
"avg_hr_day": avg_hr,
@@ -245,13 +338,16 @@ def parse_wellness_fit(file_path: str) -> dict:
"hrv_status": data.get("hrv_status"),
"steps": data.get("steps"),
"floors_climbed": data.get("floors_climbed"),
"active_calories": data.get("active_calories"),
"total_calories": data.get("total_calories"),
"active_calories": active_cal,
"total_calories": total_cal,
"sleep_duration_s": sleep_duration_s,
"sleep_deep_s": sleep_deep_s,
"sleep_light_s": sleep_light_s,
"sleep_rem_s": sleep_rem_s,
"sleep_awake_s": sleep_awake_s,
"sleep_score": data.get("sleep_score"),
"sleep_start": sleep_start_ts,
"sleep_end": sleep_end_ts,
}
return {"days": result, "error": None}
+12 -4
View File
@@ -215,11 +215,13 @@ def parse_wellness_fit(file_path: str, user_id: int):
INSERT INTO health_metrics (user_id, date, resting_hr, avg_hr_day, max_hr_day,
avg_stress, spo2_avg, hrv_nightly_avg, hrv_5min_high, hrv_status,
steps, floors_climbed, active_calories, total_calories,
sleep_duration_s, sleep_deep_s, sleep_light_s, sleep_rem_s, sleep_awake_s)
sleep_duration_s, sleep_deep_s, sleep_light_s, sleep_rem_s, sleep_awake_s,
sleep_score, sleep_start, sleep_end)
VALUES (:user_id, :date, :resting_hr, :avg_hr, :max_hr,
:avg_stress, :spo2_avg, :hrv_avg, :hrv_high, :hrv_status,
:steps, :floors, :active_cal, :total_cal,
:sleep_dur, :sleep_deep, :sleep_light, :sleep_rem, :sleep_awake)
:sleep_dur, :sleep_deep, :sleep_light, :sleep_rem, :sleep_awake,
:sleep_score, :sleep_start, :sleep_end)
ON CONFLICT (user_id, date) DO UPDATE SET
resting_hr = COALESCE(EXCLUDED.resting_hr, health_metrics.resting_hr),
avg_hr_day = COALESCE(EXCLUDED.avg_hr_day, health_metrics.avg_hr_day),
@@ -232,12 +234,15 @@ def parse_wellness_fit(file_path: str, user_id: int):
steps = COALESCE(EXCLUDED.steps, health_metrics.steps),
floors_climbed = COALESCE(EXCLUDED.floors_climbed, health_metrics.floors_climbed),
active_calories = COALESCE(EXCLUDED.active_calories, health_metrics.active_calories),
total_calories = COALESCE(EXCLUDED.total_calories, health_metrics.total_calories),
total_calories = GREATEST(EXCLUDED.total_calories, health_metrics.total_calories),
sleep_duration_s = COALESCE(EXCLUDED.sleep_duration_s, health_metrics.sleep_duration_s),
sleep_deep_s = COALESCE(EXCLUDED.sleep_deep_s, health_metrics.sleep_deep_s),
sleep_light_s = COALESCE(EXCLUDED.sleep_light_s, health_metrics.sleep_light_s),
sleep_rem_s = COALESCE(EXCLUDED.sleep_rem_s, health_metrics.sleep_rem_s),
sleep_awake_s = COALESCE(EXCLUDED.sleep_awake_s, health_metrics.sleep_awake_s)
sleep_awake_s = COALESCE(EXCLUDED.sleep_awake_s, health_metrics.sleep_awake_s),
sleep_score = COALESCE(EXCLUDED.sleep_score, health_metrics.sleep_score),
sleep_start = COALESCE(EXCLUDED.sleep_start, health_metrics.sleep_start),
sleep_end = COALESCE(EXCLUDED.sleep_end, health_metrics.sleep_end)
"""), {
"user_id": user_id, "date": date_dt,
"resting_hr": data.get("resting_hr"),
@@ -257,6 +262,9 @@ def parse_wellness_fit(file_path: str, user_id: int):
"sleep_light": data.get("sleep_light_s"),
"sleep_rem": data.get("sleep_rem_s"),
"sleep_awake": data.get("sleep_awake_s"),
"sleep_score": data.get("sleep_score"),
"sleep_start": data.get("sleep_start"),
"sleep_end": data.get("sleep_end"),
})
db.commit()