Migration

.dockerignore

@@ -0,0 +1,10 @@
+.git
+.github
+r-archive/
+.pytest_cache
+.ruff_cache
+htmlcov
+.coverage
+profiling/*.prof
+data/
+secrets/

.env.example

@@ -0,0 +1,26 @@
+# Environment Configuration
+A4D_ENVIRONMENT=development
+
+# GCP Configuration
+A4D_PROJECT_ID=a4dphase2
+A4D_DATASET=tracker
+A4D_DOWNLOAD_BUCKET=a4dphase2_upload
+A4D_UPLOAD_BUCKET=a4dphase2_output
+
+# GCP Authentication
+# Use the a4d-pipeline service account key for local development — this gives identical
+# auth to Cloud Run and is required for Drive API access (user ADC blocks Drive scope).
+# Store the key in secrets/ (gitignored). In Cloud Run, the SA authenticates via metadata server.
+# GOOGLE_APPLICATION_CREDENTIALS=/path/to/a4d-pipeline-key.json
+
+# Paths
+A4D_DATA_ROOT=/path/to/tracker/files
+A4D_OUTPUT_DIR=output
+
+# Processing Settings
+A4D_MAX_WORKERS=4
+
+# Error Values (matching R pipeline)
+A4D_ERROR_VAL_NUMERIC=999999
+A4D_ERROR_VAL_CHARACTER=Undefined
+A4D_ERROR_VAL_DATE=9999-12-31

.github/workflows/python-ci.yml

@@ -0,0 +1,57 @@
+name: Python CI
+
+on:
+  push:
+    branches: [migration]
+    paths:
+      - 'src/**'
+      - 'tests/**'
+      - 'pyproject.toml'
+      - '.github/workflows/python-ci.yml'
+  pull_request:
+    branches: [main, develop, migration]
+    paths:
+      - 'src/**'
+      - 'tests/**'
+      - 'pyproject.toml'
+      - '.github/workflows/python-ci.yml'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v2
+      with:
+        enable-cache: true
+
+    - name: Set up Python
+      run: uv python install 3.14
+
+    - name: Install dependencies
+      run: uv sync --all-extras
+
+    - name: Run ruff linting
+      run: uv run ruff check .
+
+    - name: Run ruff formatting check
+      run: uv run ruff format --check .
+
+    - name: Run type checking with ty
+      run: uv run ty check src/
+
+    - name: Run tests
+      run: uv run pytest -m "not slow and not integration" --cov --cov-report=xml
+
+    - name: Enforce coverage floor for product pipeline code
+      run: |
+        uv run coverage report --include="src/a4d/extract/product.py,src/a4d/clean/product.py,src/a4d/clean/schema_product.py,src/a4d/extract/wide_format.py,src/a4d/pipeline/product.py,src/a4d/tables/product.py,src/a4d/validate/source_vs_output_product.py" --fail-under=85
+
+    - name: Upload coverage
+      uses: codecov/codecov-action@v3
+      with:
+        files: ./coverage.xml
+        flags: python

.gitignore

@@ -1,13 +1,80 @@
-.Rproj.user
-.Rhistory
-.RData
-.Ruserdata
-.Rhistory
-.Rdata
-.httr-oauth
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+env/
+
+# uv
+.uv/
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+
+# Type checking
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Environment
+.env
+.env.local
+
+# Logs
+*.log
+logs/
+
+# Data (sensitive)
+data/
+output/
+*.parquet
+*.xlsx
+!reference_data/
+compare_report*.html
+
+# OS
 .DS_Store
-.Renviron
-rsconnect
+Thumbs.db
 
-data/output
-data/mapping_table.csv
+# Serena (MCP server state)
+.serena/
+
+# Secrets (GCP service accounts, etc.)
+secrets/
+.opencode
+.Rproj.user
+CLAUDE.local.md
+.Rhistory
+r-archive/config.yml
+coverage.xml

.vscode/settings.json

@@ -0,0 +1,29 @@
+{
+    "python.testing.pytestEnabled": true,
+    "python.testing.unittestEnabled": false,
+    "python.testing.cwd": "${workspaceFolder}/a4d-python",
+    "python.testing.pytestArgs": [
+        "${workspaceFolder}/a4d-python/tests"
+    ],
+    "python.defaultInterpreterPath": "${workspaceFolder}/a4d-python/.venv/bin/python",
+    "workbench.colorCustomizations": {
+        "activityBar.activeBackground": "#ab307e",
+        "activityBar.background": "#ab307e",
+        "activityBar.foreground": "#e7e7e7",
+        "activityBar.inactiveForeground": "#e7e7e799",
+        "activityBarBadge.background": "#25320e",
+        "activityBarBadge.foreground": "#e7e7e7",
+        "commandCenter.border": "#e7e7e799",
+        "sash.hoverBorder": "#ab307e",
+        "statusBar.background": "#832561",
+        "statusBar.foreground": "#e7e7e7",
+        "statusBarItem.hoverBackground": "#ab307e",
+        "statusBarItem.remoteBackground": "#832561",
+        "statusBarItem.remoteForeground": "#e7e7e7",
+        "titleBar.activeBackground": "#832561",
+        "titleBar.activeForeground": "#e7e7e7",
+        "titleBar.inactiveBackground": "#83256199",
+        "titleBar.inactiveForeground": "#e7e7e799"
+    },
+    "peacock.color": "#832561"
+}

CLAUDE.md

@@ -0,0 +1,39 @@
+# CLAUDE.md
+
+## Python Pipeline
+
+**Location**: Repo root
+**Branch**: `migration`
+
+Python implementation of the A4D medical tracker data processing pipeline.
+
+**Key Directories**:
+- `src/` - Python package source
+- `tests/` - Test suite
+- `docs/` - Documentation (see [docs/CLAUDE.md](docs/CLAUDE.md) for detailed guidance)
+- `scripts/` - Utility scripts
+- `reference_data/` - Shared YAML configs (synonyms, validation rules, provinces)
+
+**Quick Start**:
+```bash
+uv sync
+uv run pytest
+```
+
+**Migration Guide**: [docs/migration/MIGRATION_GUIDE.md](docs/migration/MIGRATION_GUIDE.md)
+
+## R Archive
+
+**Location**: `r-archive/`
+
+Legacy R implementation, preserved for reference. Do not modify.
+
+## Shared Resources
+
+- `reference_data/synonyms/` - Column name mappings
+- `reference_data/data_cleaning.yaml` - Validation rules
+- `reference_data/provinces/` - Allowed provinces
+
+**Do not modify these** without testing the Python pipeline.
+- Always check your implementation against the original R pipeline and verify the logic is the same
+- Limit comments to explain why a design was made or give important context for the migration; do not use comments for obvious code

Dockerfile

@@ -0,0 +1,37 @@
+FROM python:3.14-slim
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gcc \
+    g++ \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install uv from the official image
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+# Use the system Python from the base image; do not let uv download its own
+ENV UV_PYTHON_DOWNLOADS=never
+
+WORKDIR /app
+
+# Install dependencies first (without the project) for better layer caching.
+# --no-install-project skips the editable install of a4d itself, which requires
+# src/ to be present. Dependencies rarely change so this layer stays cached.
+COPY pyproject.toml uv.lock README.md ./
+RUN uv sync --frozen --no-dev --no-install-project
+
+# Copy application code and reference data
+COPY src/ src/
+COPY reference_data/ reference_data/
+
+# Install the project itself now that src/ exists
+RUN uv sync --frozen --no-dev
+
+# Set environment
+ENV PYTHONPATH=/app/src
+ENV PYTHONUNBUFFERED=1
+ENV A4D_DATA_ROOT=/workspace/data
+ENV A4D_REFERENCE_DATA=/app/reference_data
+
+# Run the full pipeline: download → process → upload to GCS → ingest into BigQuery
+CMD ["uv", "run", "a4d", "run"]