From 492418586a59ccff4312256ef76bdd63116cfd88 Mon Sep 17 00:00:00 2001 From: owain Date: Sun, 7 Jun 2026 18:15:07 +0100 Subject: [PATCH] Fixed Garmin sync progress bar granularity, timeout issue, and lookback days input, plus redesigned the sleep timeline with taller bars and yellow Awake colour. --- CLAUDE.md | 15 +++- backend/app/services/garmin_connect_sync.py | 19 ++++- backend/app/workers/tasks.py | 12 ++- frontend/src/pages/HealthPage.jsx | 47 ++++++----- frontend/src/pages/ProfilePage.jsx | 86 ++++++++++++++------- 5 files changed, 120 insertions(+), 59 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8d0cbe9..a53c71d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -63,11 +63,12 @@ docker compose -f docker-compose.deploy.yml up -d - `main.py` — FastAPI app, DB init on startup (creates tables, seeds admin user, creates TimescaleDB hypertable) - `core/` — `config.py` (pydantic-settings from env), `database.py` (async engine for FastAPI + sync engine for Celery), `security.py` (JWT, bcrypt) -- `api/` — routers: `auth`, `activities`, `routes`, `health`, `records`, `upload`, `profile` -- `models/user.py` — all SQLAlchemy models: `User`, `Activity`, `ActivityDataPoint`, `ActivityLap`, `NamedRoute`, `RouteSegment`, `PersonalRecord`, `HealthMetric`, `WeightLog` +- `api/` — routers: `auth`, `activities`, `routes`, `health`, `records`, `upload`, `profile`, `garmin_sync` +- `models/user.py` — all SQLAlchemy models: `User`, `Activity`, `ActivityDataPoint`, `ActivityLap`, `NamedRoute`, `RouteSegment`, `PersonalRecord`, `HealthMetric`, `WeightLog`, `GarminConnectConfig` - `services/fit_parser.py` — parses Garmin FIT and GPX files; handles raw FIT timestamps (FIT epoch offset 631065600s) and semicircle→degree conversion - `services/wellness_parser.py` — parses Garmin wellness FIT files (metrics, sleep, HRV, SPO2, etc.) - `services/route_matcher.py` — bounding-box pre-filter + DTW (Dynamic Time Warping) for GPS track similarity +- `services/garmin_connect_sync.py` — Garmin Connect API integration; `authenticate_garmin()` tries stored OAuth token first, falls back to email/password; Garmin credentials stored Fernet-encrypted using `SECRET_KEY` as the key - `workers/tasks.py` — Celery tasks: `process_activity_file`, `parse_wellness_fit`, `detect_route`, `compute_personal_records`, `process_garmin_health_zip` ### Key design decisions @@ -76,7 +77,7 @@ docker compose -f docker-compose.deploy.yml up -d **File routing in Celery**: `process_activity_file` inspects the filename; files matching wellness suffixes (`_METRICS.fit`, `_WELLNESS.fit`, `_SLEEP.fit`, etc.) are routed to `parse_wellness_fit` instead. -**Schema management**: No Alembic migrations are used in production. `Base.metadata.create_all` runs at startup with retry logic to handle multi-worker races. Health metrics upserts use raw SQL `ON CONFLICT ... DO UPDATE SET ... COALESCE(EXCLUDED.x, existing.x)` to merge data from multiple file sources without overwriting. +**Schema management**: No Alembic migrations are used in production. `Base.metadata.create_all` runs at startup with retry logic to handle multi-worker races. Post-initial schema changes (new columns, constraint changes) are applied as `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` / `DROP CONSTRAINT IF EXISTS` statements in `init_db()` in `main.py` — this is the only place schema migrations happen. Health metrics upserts use raw SQL `ON CONFLICT ... DO UPDATE SET ... COALESCE(EXCLUDED.x, existing.x)` to merge data from multiple file sources without overwriting. **PocketID OIDC**: Optional passkey auth. Config is read from the admin user's DB record first, falling back to env vars. The OAuth callback redirects to `/?token=` and `useAuth.js` extracts the token from the URL at module load time. @@ -85,8 +86,10 @@ docker compose -f docker-compose.deploy.yml up -d - `App.jsx` — React Router v6, `RequireAuth` wrapper, all routes defined here - `hooks/useAuth.js` — Zustand store for auth state, reads JWT from `localStorage`, handles PocketID token-in-URL flow - `utils/api.js` — Axios instance with JWT interceptor and 401→redirect handler -- `pages/` — one file per route +- `utils/format.js` — shared formatting helpers: `formatDuration`, `formatPace`, `formatDistance`, `formatCadence`, `hrZoneColor`, `sportIcon`, `sportColor`, etc. +- `pages/` — one file per route; includes `SegmentsPage` for route segment management - `components/activity/` — `ActivityMap` (Leaflet), `MetricTimeline` (Recharts), `HRZoneBar`, `LapTable` +- `components/ui/RouteMiniMap` — small Leaflet map used in route/segment cards The Vite dev server proxies `/api` to `http://backend:8000` (for use inside the Docker Compose network). The production build bakes `VITE_API_URL` at build time. @@ -104,6 +107,10 @@ Required in `.env` (or passed to Docker Compose): | `VITE_MAPBOX_TOKEN` | Optional — enables satellite tile layer | | `POCKETID_ISSUER` / `POCKETID_CLIENT_ID` / `POCKETID_CLIENT_SECRET` | Optional OIDC | +## milevault_export/ + +`milevault_export/` is a sanitised snapshot of the project used for public distribution (stripped of dev-only configs). It mirrors the main project structure. When making changes that affect deployment files (`docker-compose.yml`, `nginx.conf`, `scripts/manage.sh`, `docker/init.sql`, etc.), keep this directory in sync manually. + ## Rules - The current build will always be running in docker at ~/milevault_docker with the following container names: `milevault_backend` diff --git a/backend/app/services/garmin_connect_sync.py b/backend/app/services/garmin_connect_sync.py index 1e02c73..1c83eff 100644 --- a/backend/app/services/garmin_connect_sync.py +++ b/backend/app/services/garmin_connect_sync.py @@ -72,7 +72,8 @@ def authenticate_garmin(email: str, password_enc: str, token_store: Optional[str # ── Activity sync ───────────────────────────────────────────────────────────── def sync_activities(garmin, user_id: int, since: Optional[datetime], - db, file_store_path: str, lookback_days: int = 30) -> int: + db, file_store_path: str, lookback_days: int = 30, + status_callback=None) -> int: """ List activities from Garmin Connect, skip any already in the DB, download FIT ZIPs for new ones, and queue them for processing. @@ -108,6 +109,10 @@ def sync_activities(garmin, user_id: int, since: Optional[datetime], logger.error("Failed to list Garmin activities: %s", exc) return 0 + total = len(activities) + if status_callback and total: + status_callback(f"Syncing activities: 0/{total} queued") + queued = 0 for act in activities: garmin_id = str(act.get("activityId", "")).strip() @@ -173,6 +178,9 @@ def sync_activities(garmin, user_id: int, since: Optional[datetime], process_activity_file.delay(str(dest), user_id, "fit", garmin_id) queued += 1 + if status_callback and (queued % 5 == 0 or queued == total): + status_callback(f"Syncing activities: {queued}/{total} queued") + # Brief pause to avoid hammering the Garmin API time.sleep(0.5) @@ -182,7 +190,7 @@ def sync_activities(garmin, user_id: int, since: Optional[datetime], # ── Wellness sync ───────────────────────────────────────────────────────────── def sync_wellness(garmin, user_id: int, since: Optional[datetime], db, - lookback_days: int = 90) -> int: + lookback_days: int = 90, status_callback=None) -> int: """ Fetch daily stats / sleep / HRV from the Garmin Connect JSON API for each day in the window and upsert into health_metrics. @@ -209,8 +217,13 @@ def sync_wellness(garmin, user_id: int, since: Optional[datetime], db, import time as _time import json as _json - for i in range(max(days, 1)): + total_days = max(days, 1) + if status_callback: + status_callback(f"Syncing wellness: 0/{total_days} days") + for i in range(total_days): day = start_date + timedelta(days=i) + if status_callback and (i % 5 == 0 or i == total_days - 1): + status_callback(f"Syncing wellness: {i + 1}/{total_days} days") day_str = day.isoformat() stats = _safe(garmin.get_stats, day_str) diff --git a/backend/app/workers/tasks.py b/backend/app/workers/tasks.py index a087b17..6c50fed 100644 --- a/backend/app/workers/tasks.py +++ b/backend/app/workers/tasks.py @@ -511,24 +511,28 @@ def sync_garmin_connect_user(user_id: int): wellness_days = 0 errors = [] - if sync_acts: - cfg.last_sync_status = "Syncing activities..." + def _set_status(text): + cfg.last_sync_status = text db.commit() + + if sync_acts: + _set_status("Syncing activities...") try: activities_queued = sync_activities( garmin, user_id, last_sync_at, db, settings.file_store_path, lookback_days=lookback, + status_callback=_set_status, ) except Exception as exc: errors.append(f"activities: {exc}") if sync_well: - cfg.last_sync_status = "Syncing wellness data..." - db.commit() + _set_status("Syncing wellness...") try: wellness_days = sync_wellness( garmin, user_id, last_sync_at, db, lookback_days=lookback, + status_callback=_set_status, ) except Exception as exc: errors.append(f"wellness: {exc}") diff --git a/frontend/src/pages/HealthPage.jsx b/frontend/src/pages/HealthPage.jsx index 7facf58..a682092 100644 --- a/frontend/src/pages/HealthPage.jsx +++ b/frontend/src/pages/HealthPage.jsx @@ -169,7 +169,7 @@ function SleepTimeline({ sleepStart, sleepEnd, deep, light, rem, awake }) { { key: 'deep', secs: deep || 0, color: '#6366f1', label: 'Deep' }, { key: 'rem', secs: rem || 0, color: '#8b5cf6', label: 'REM' }, { key: 'light', secs: light || 0, color: '#a78bfa', label: 'Light' }, - { key: 'awake', secs: awake || 0, color: '#374151', label: 'Awake' }, + { key: 'awake', secs: awake || 0, color: '#eab308', label: 'Awake' }, ].filter(s => s.secs > 0) // Generate hour tick marks within the sleep window @@ -185,27 +185,32 @@ function SleepTimeline({ sleepStart, sleepEnd, deep, light, rem, awake }) { } return ( -
- {/* Time bar */} -
-
- {stages.map((s, i) => ( -
- ))} -
- {/* Tick marks */} +
+ {/* Stage bars rising from the time axis */} +
+ {stages.map(s => ( +
+ ))} + {/* Tick lines overlaid on bars */} {ticks.map((t, i) => ( -
-
-
+
+ ))} +
+ {/* Axis line */} +
+ {ticks.map((t, i) => ( +
))}
{/* Time labels */} -
- +
+ {new Date(startMs).toLocaleTimeString('en-GB', { hour: '2-digit', minute: '2-digit' })} {ticks.map((t, i) => ( @@ -318,7 +323,7 @@ function DailySnapshot({ day, avg30, intradayHr, bodyBattery, bbHires, onOlder, ['Deep', day.sleep_deep_s, '#6366f1'], ['REM', day.sleep_rem_s, '#8b5cf6'], ['Light', day.sleep_light_s, '#a78bfa'], - ['Awake', day.sleep_awake_s, '#4b5563'], + ['Awake', day.sleep_awake_s, '#eab308'], ].map(([label, secs, color]) => secs ? (
@@ -565,7 +570,7 @@ function SleepChart({ data, selectedDate, onDayClick }) { - + ) @@ -704,7 +709,7 @@ export default function HealthPage() {
- {[['Deep','#6366f1'],['REM','#8b5cf6'],['Light','#a78bfa'],['Awake','#374151']].map(([l,c]) => ( + {[['Deep','#6366f1'],['REM','#8b5cf6'],['Light','#a78bfa'],['Awake','#eab308']].map(([l,c]) => (
{l} diff --git a/frontend/src/pages/ProfilePage.jsx b/frontend/src/pages/ProfilePage.jsx index ee76b4d..ab70688 100644 --- a/frontend/src/pages/ProfilePage.jsx +++ b/frontend/src/pages/ProfilePage.jsx @@ -115,7 +115,7 @@ export default function ProfilePage() { queryKey: ['garmin-config'], queryFn: () => api.get('/garmin-sync/config').then(r => r.data), }) - const [gcForm, setGcForm] = useState({ email: '', password: '', sync_enabled: true, sync_activities: true, sync_wellness: true, sync_lookback_days: 30 }) + const [gcForm, setGcForm] = useState({ email: '', password: '', sync_enabled: true, sync_activities: true, sync_wellness: true, sync_lookback_days: '30' }) const [gcSaved, setGcSaved] = useState(false) const [gcError, setGcError] = useState('') const [gcSyncing, setGcSyncing] = useState(false) @@ -129,7 +129,7 @@ export default function ProfilePage() { sync_enabled: garminConfig.sync_enabled, sync_activities: garminConfig.sync_activities, sync_wellness: garminConfig.sync_wellness, - sync_lookback_days: garminConfig.sync_lookback_days ?? 30, + sync_lookback_days: String(garminConfig.sync_lookback_days ?? 30), })) } }, [garminConfig]) @@ -148,14 +148,14 @@ export default function ProfilePage() { mutationFn: () => api.delete('/garmin-sync/config'), onSuccess: () => { refetchGarmin() - setGcForm({ email: '', password: '', sync_enabled: true, sync_activities: true, sync_wellness: true, sync_lookback_days: 30 }) + setGcForm({ email: '', password: '', sync_enabled: true, sync_activities: true, sync_wellness: true, sync_lookback_days: '30' }) }, }) const triggerGarminSync = async () => { setGcSyncing(true) try { await api.post('/garmin-sync/trigger') - // Poll every 2s: wait until we've seen an in-progress status, then wait for terminal + // Poll every 3s: wait until we've seen an in-progress status, then wait for terminal let seenInProgress = false syncPollRef.current = setInterval(async () => { const result = await refetchGarmin() @@ -167,23 +167,36 @@ export default function ProfilePage() { syncPollRef.current = null setGcSyncing(false) } - }, 2000) - // Safety: stop polling after 10 minutes regardless + }, 3000) + // Absolute safety: stop polling after 4 hours but keep bar visible — sync may still be running setTimeout(() => { if (syncPollRef.current) { clearInterval(syncPollRef.current); syncPollRef.current = null } - setGcSyncing(false) - }, 600000) + }, 4 * 60 * 60 * 1000) } catch { setGcSyncing(false) } } const syncProgressPct = status => { - if (!status) return 5 + if (!status) return 3 if (status.startsWith('Connecting')) return 10 - if (status.startsWith('Syncing activities')) return 35 - if (status.startsWith('Syncing wellness')) return 70 - return 5 + if (status.startsWith('Syncing activities')) { + const m = status.match(/(\d+)\/(\d+)/) + if (m) { + const done = parseInt(m[1], 10), total = parseInt(m[2], 10) + if (total > 0) return 15 + Math.round(done / total * 30) + } + return 20 + } + if (status.startsWith('Syncing wellness')) { + const m = status.match(/(\d+)\/(\d+)/) + if (m) { + const done = parseInt(m[1], 10), total = parseInt(m[2], 10) + if (total > 0) return 45 + Math.round(done / total * 45) + } + return 50 + } + return 3 } // PocketID config @@ -344,8 +357,8 @@ export default function ProfilePage() { setGcForm(f => ({ ...f, sync_lookback_days: parseInt(e.target.value, 10) || 30 }))} /> - {gcForm.sync_lookback_days > 365 && gcForm.sync_lookback_days !== -1 && ( + onChange={e => setGcForm(f => ({ ...f, sync_lookback_days: e.target.value }))} /> + {(() => { const n = parseInt(gcForm.sync_lookback_days, 10); return n > 365 && n !== -1 })() && (

Warning: syncing more than 365 days at once may take a long time and could trigger Garmin rate limits.

)} @@ -360,7 +373,10 @@ export default function ProfilePage() { setGcError('Password is required for first-time setup') return } - const payload = { ...gcForm } + const payload = { + ...gcForm, + sync_lookback_days: parseInt(gcForm.sync_lookback_days, 10) || 30, + } if (!payload.password) delete payload.password saveGarmin.mutate(payload) }} @@ -385,19 +401,35 @@ export default function ProfilePage() { )}
- {gcSyncing && ( -
-
-
+ {gcSyncing && (() => { + const status = garminConfig?.last_sync_status || '' + const pct = syncProgressPct(status) + const phase = status.startsWith('Connecting') ? 0 + : status.startsWith('Syncing activities') ? 1 + : status.startsWith('Syncing wellness') ? 2 + : status.startsWith('OK') || status.startsWith('Partial') ? 3 : -1 + return ( +
+
+ {[['Connect', 0], ['Activities', 1], ['Wellness', 2]].map(([label, idx]) => ( + = idx ? 'text-blue-400' : 'text-gray-600'}`}> + {idx > 0 && ›} + {label} + + ))} +
+
+
+
+

+ {status || 'Starting sync…'} +

-

- {garminConfig?.last_sync_status || 'Starting sync…'} -

-
- )} + ) + })()} {/* PocketID — admin only */}