feat: add hydrated fields to DataType

src/datatype/index.js

@@ -1,6 +1,6 @@
 import { validate } from '@comapeo/schema'
 import { getTableConfig } from 'drizzle-orm/sqlite-core'
-import { eq, inArray, sql } from 'drizzle-orm'
+import { eq, inArray, sql, Columns } from 'drizzle-orm'
 import { randomBytes } from 'node:crypto'
 import { noop, mutatingDeNullify } from '../utils.js'
 import { NotFoundError } from '../errors.js'
@@ -114,6 +114,8 @@
     db,
     getTranslations,
     getDeviceIdForVersionId,
+    // Which properties can be hydrated e.g. `observationTable.presetRef ->  presetTable`
+    hydratedRefs = [],
   }) {
     super()
     this.#dataStore = dataStore
@@ -135,6 +137,44 @@
         .prepare(),
       getManyWithDeleted: db.select().from(table).prepare(),
     }
+    if (hydratedRefs.length) {
+      // select all fields
+      // we will add more
+      const toSelect = { ...this.#table[Columns] }
+      for (const { field, table } of hydratedRefs) {
+        const { name } = field
+        if (!name.endsWith('Ref')) {
+          throw new Error('Only fields that end with `Ref` can be hydrated')
+        }
+        // Strip out Ref
+        const propertyName = name.slice(0, -3)
+        toSelect[propertyName] = table
+      }
+
+      let getManyHydrated = db
+        .select(toSelect)
+        .from(table)
+        .where(eq(table.deleted, false))
+
+      let getByDocIdHydrated = db
+        .select()
+        .from(table)
+        .where(eq(table.docId, sql.placeholder('docId')))
+
+      for (const { field, table } of hydratedRefs) {
+        getManyHydrated = getManyHydrated.fulljoin(
+          table,
+          eq(field, table.docId)
+        )
+        getByDocIdHydrated = getByDocIdHydrated.fulljoin(
+          table,
+          eq(field, table.docId)
+        )
+      }
+
+      this.#sql.getManyHydrated = getManyHydrated.prepare()
+      this.#sql.getByDocIdHydrated = getByDocIdHydrated.prepare()
+    }
     this.on('newListener', (eventName) => {
       if (eventName !== 'updated-docs') return
       if (this.listenerCount('updated-docs') > 1) return
@@ -228,9 +268,14 @@
    * @param {string} [options.lang]
    * @returns {Promise<null | (TDoc & DerivedDocFields)>}
    */
-  async getByDocId(docId, { mustBeFound = true, lang } = {}) {
+  async getByDocId(docId, { mustBeFound = true, lang, hydrate = false } = {}) {
     await this.#dataStore.indexer.idle()
-    const result = this.#sql.getByDocId.get({ docId })
+    let result = null
+    if (hydrate) {
+      result = this.#sql.getByDocIdHydrated.get({ docId })
+    } else {
+      result = this.#sql.getByDocId.get({ docId })
+    }
     if (result) {
       return this.#mutatingAddDerivedFields(result, { lang })
     } else if (mustBeFound) {

test/data-type.js

@@ -10,6 +10,7 @@ import RAM from 'random-access-memory'
 import crypto from 'hypercore-crypto'
 import {
   observationTable,
+  presetTable,
   trackTable,
   translationTable,
 } from '../src/schema/project.js'
@@ -353,6 +354,8 @@ test('translation', async () => {
   )
 })
 
+test.only('Hydration pulls data from other tables', async () => {})
+
 /**
  * @param {object} [opts={}]
  * @param {Buffer} [opts.projectKey]
@@ -425,9 +428,17 @@ async function testenv(opts = {}) {
 
   const translationApi = new TranslationApi({ dataType: translationDataType })
 
+  const hydratedRefs = [
+    {
+      field: observationTable.presetRef,
+      table: presetTable,
+    },
+  ]
+
   const dataType = new DataType({
     dataStore,
     table: observationTable,
+    hydratedRefs,
     db,
     getTranslations: translationApi.get.bind(translationApi),
     async getDeviceIdForVersionId() {