fix(bb.js): re-enable browser tests with a local CRS; restore typed C++ exceptions on threaded wasm

barretenberg/acir_tests/bootstrap.sh

@@ -164,11 +164,12 @@ function test_cmds {
   echo "$sol_prefix $scripts/bb_prove_bbjs_verify.sh a_1_mul"
   echo "$sol_prefix $scripts/bb_prove_bbjs_verify.sh assert_statement"
 
-  # bb.js browser tests. Isolate because server.
-  # Temporarily skipped due to flaky "Failed to fetch" errors in CI.
-  # local browser_prefix="$tests_hash:ISOLATE=1:NET=1:CPUS=8"
-  # echo "$browser_prefix $scripts/browser_prove.sh verify_honk_proof chrome"
-  # echo "$browser_prefix $scripts/browser_prove.sh a_1_mul chrome"
+  # bb.js browser tests. Isolate because server. The browser's CRS downloads are answered from the
+  # local CRS (headless-test/src/local_crs.ts), so they need no network.
+  # verify_honk_proof needs 2^20 points, past bb.js's default.
+  local browser_prefix="$tests_hash:ISOLATE=1:CPUS=8"
+  echo "$browser_prefix $scripts/browser_prove.sh verify_honk_proof chrome --srs-size $((2**20))"
+  echo "$browser_prefix $scripts/browser_prove.sh a_1_mul chrome"
 
   # bb.js tests.
   # ecdsa_secp256r1_3x through bb.js on node to check 256k support.

barretenberg/acir_tests/browser-test-app/src/index.ts

@@ -21,11 +21,12 @@ function installUltraHonkGlobals() {
     bytecode: string,
     witness: Uint8Array,
     threads?: number,
+    srsSize?: number,
   ): Promise<{ proofData: ProofData; verificationKey: Uint8Array }> {
     const { UltraHonkBackend } = await import('@aztec-foundation/bb.js');
 
     logger.debug('starting test...');
-    const bb = await Barretenberg.new({ threads, logger: bbLogger });
+    const bb = await Barretenberg.new({ threads, srsSize, logger: bbLogger });
     const backend = new UltraHonkBackend(bytecode, bb);
     const proofData = await backend.generateProof(witness);
 

barretenberg/acir_tests/headless-test/src/index.ts

@@ -7,6 +7,8 @@ import { createServer, type Server } from 'http';
 import { dirname, join } from 'path';
 import { fileURLToPath } from 'url';
 
+import { serveLocalCrs } from './local_crs.js';
+
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 
@@ -75,6 +77,7 @@ program
     const capturedLogs: string[] = [];
     const browser = await chromium.launch();
     const context = await browser.newContext();
+    await serveLocalCrs(context);
     const page = await context.newPage();
 
     // Capture console messages (this is what CI benchmark scripts rely on)
@@ -132,7 +135,8 @@ program
   .description('Generate a proof and verify it. Process exits with success or failure code.')
   .option('-b, --bytecode-path <path>', 'Specify the path to the ACIR artifact json file', './target/acir.json')
   .option('-w, --witness-path <path>', 'Specify the path to the gzip encoded ACIR witness', './target/witness.gz')
-  .action(async ({ bytecodePath, witnessPath }) => {
+  .option('-s, --srs-size <points>', 'CRS points to prove with, for a circuit larger than the bb.js default', Number)
+  .action(async ({ bytecodePath, witnessPath, srsSize }) => {
     const acir = readBytecodeFile(bytecodePath);
     const witness = readWitnessFile(witnessPath);
     const threads = Math.min(os.cpus().length, 16);
@@ -147,6 +151,7 @@ program
       const browser = await browserType.launch();
 
       const context = await browser.newContext();
+      await serveLocalCrs(context);
       const provingPage = await context.newPage();
 
       if (program.opts().verbose) {
@@ -175,19 +180,19 @@ program
       } = await provingPage.evaluate(
         async (args: any) => {
           // Convert the input data to Uint8Arrays within the browser context
-          const [acir, witnessData, threads] = args;
+          const [acir, witnessData, threads, srsSize] = args;
           const witnessUint8Array = new Uint8Array(witnessData);
 
           // Call the desired function and return the result
-          const { proofData, verificationKey } = await (window as any).prove(acir, witnessUint8Array, threads);
+          const { proofData, verificationKey } = await (window as any).prove(acir, witnessUint8Array, threads, srsSize);
 
           return {
             publicInputs: proofData.publicInputs,
             proof: Array.from(proofData.proof),
             verificationKey: Array.from(verificationKey),
           };
         },
-        [acir, Array.from(witness), threads],
+        [acir, Array.from(witness), threads, srsSize],
       );
       await provingPage.close();
 
@@ -254,6 +259,7 @@ program
       const browser = await browserType.launch();
 
       const context = await browser.newContext();
+      await serveLocalCrs(context);
       const provingPage = await context.newPage();
 
       if (program.opts().verbose) {
@@ -389,7 +395,7 @@ process.on('SIGTERM', () => {
     cleanup();
     process.exit(0);
   } catch (error) {
-    console.error('Failed to start test server:', error);
+    console.error('Test failed:', error);
     cleanup();
     process.exit(1);
   }

barretenberg/acir_tests/headless-test/src/local_crs.ts

@@ -0,0 +1,129 @@
+import fs from 'fs';
+import os from 'os';
+import { join } from 'path';
+import type { BrowserContext, Route } from 'playwright';
+
+/** The hosts bb.js downloads the CRS from in a browser: the CDN and its fallback. */
+const CRS_URL = /^https:\/\/crs\.aztec-(cdn\.foundation|labs\.com)\/([^?#]+)/;
+
+/** The CRS cache `barretenberg/crs/bootstrap.sh` fills, which bb.js on node reads too. */
+const crsPath = () => process.env.CRS_PATH ?? join(os.homedir(), '.bb-crs');
+
+const COMPRESSED_POINT_BYTES = 32;
+const UNCOMPRESSED_POINT_BYTES = 64;
+
+function fileSize(path: string): number {
+  try {
+    return fs.statSync(path).size;
+  } catch {
+    return 0;
+  }
+}
+
+function readRange(path: string, start: number, length: number): Buffer {
+  const out = Buffer.alloc(length);
+  const fd = fs.openSync(path, 'r');
+  try {
+    let read = 0;
+    while (read < length) {
+      const n = fs.readSync(fd, out, read, length - read, start + read);
+      if (n === 0) {
+        throw new Error(`${path} ends before byte ${start + length}`);
+      }
+      read += n;
+    }
+  } finally {
+    fs.closeSync(fd);
+  }
+  return out;
+}
+
+/**
+ * The compressed BN254 G1 points in bytes [start, end), from whichever form the cache holds: the compressed
+ * file, or the uncompressed one (CI's machine images keep only that), compressed here. A compressed point is
+ * x, big-endian, with bit 255 set to the low bit of y; points are whole, so the range is aligned to them.
+ */
+function bn254G1Compressed(dir: string, start: number, end: number): Buffer {
+  const compressed = join(dir, 'bn254_g1_compressed.dat');
+  if (fileSize(compressed) >= end) {
+    return readRange(compressed, start, end - start);
+  }
+  if (start % COMPRESSED_POINT_BYTES !== 0 || end % COMPRESSED_POINT_BYTES !== 0) {
+    throw new Error(`bytes ${start}-${end - 1} do not cover whole compressed points`);
+  }
+  const first = start / COMPRESSED_POINT_BYTES;
+  const count = (end - start) / COMPRESSED_POINT_BYTES;
+  const points = readRange(
+    join(dir, 'bn254_g1.dat'),
+    first * UNCOMPRESSED_POINT_BYTES,
+    count * UNCOMPRESSED_POINT_BYTES,
+  );
+  const out = Buffer.alloc(count * COMPRESSED_POINT_BYTES);
+  for (let i = 0; i < count; i++) {
+    const point = i * UNCOMPRESSED_POINT_BYTES;
+    points.copy(out, i * COMPRESSED_POINT_BYTES, point, point + COMPRESSED_POINT_BYTES);
+    if (points[point + UNCOMPRESSED_POINT_BYTES - 1] & 1) {
+      out[i * COMPRESSED_POINT_BYTES] |= 0x80;
+    }
+  }
+  return out;
+}
+
+/** The CDN file's bytes [start, end), or to its end when `end` is undefined. */
+function crsBytes(file: string, start: number, end: number | undefined): Buffer {
+  const dir = crsPath();
+  const whole = (path: string) => {
+    const stop = end ?? fileSize(path);
+    return readRange(path, start, stop - start);
+  };
+  switch (file) {
+    case 'g1_compressed.dat':
+      if (end === undefined) {
+        throw new Error('the whole compressed G1 CRS was asked for; bb.js always asks for a range');
+      }
+      return bn254G1Compressed(dir, start, end);
+    case 'g2.dat':
+      return whole(join(dir, 'bn254_g2.dat'));
+    case 'grumpkin_g1_v2.dat':
+      return whole(join(dir, 'grumpkin_g1_v2.flat.dat'));
+    default:
+      throw new Error(`no local copy of CRS file ${file}`);
+  }
+}
+
+async function answer(route: Route, file: string): Promise<void> {
+  const range = route.request().headers()['range'];
+  const match = range?.match(/^bytes=(\d+)-(\d+)$/);
+  if (range && !match) {
+    throw new Error(`unsupported Range header ${range}`);
+  }
+  const start = match ? Number(match[1]) : 0;
+  const body = crsBytes(file, start, match ? Number(match[2]) + 1 : undefined);
+  await route.fulfill({
+    status: match ? 206 : 200,
+    headers: {
+      'Content-Type': 'application/octet-stream',
+      'Access-Control-Allow-Origin': '*',
+      'Access-Control-Expose-Headers': 'Content-Length,Content-Range',
+      ...(match ? { 'Content-Range': `bytes ${start}-${start + body.length - 1}/*` } : {}),
+    },
+    body,
+  });
+}
+
+/**
+ * Answers the browser's CRS downloads from the local CRS cache, so a test needs no network and gets the same
+ * points bb on node uses. A file the cache can't answer fails the request, with the reason logged, rather than
+ * falling through to the network.
+ */
+export async function serveLocalCrs(context: BrowserContext): Promise<void> {
+  await context.route(CRS_URL, async route => {
+    const file = route.request().url().match(CRS_URL)![2];
+    try {
+      await answer(route, file);
+    } catch (err: any) {
+      console.error(`Local CRS can't answer ${route.request().url()}: ${err.message}`);
+      await route.fulfill({ status: 500, body: err.message, headers: { 'Access-Control-Allow-Origin': '*' } });
+    }
+  });
+}

barretenberg/acir_tests/scripts/browser_prove.sh

@@ -3,5 +3,5 @@ source $(git rev-parse --show-toplevel)/ci3/source
 cd ../acir_tests/$1
 export BROWSER=$2
 
-# The headless test will start its own server
-../../headless-test/bb.js.browser prove_and_verify -b target/program.json -v
+# The headless test will start its own server. Further arguments go to prove_and_verify.
+../../headless-test/bb.js.browser prove_and_verify -b target/program.json -v "${@:3}"

barretenberg/ts/bootstrap.sh

@@ -50,6 +50,14 @@ function generate_cdb_package {
 # The bb API as a generated client package: typed AsyncApi/SyncApi over bb spawned as a
 # process (uds, shm) or run in-process as the wasm module (node and browsers). bb.js is
 # a consumer of this package, keeping only its facades and CRS handling.
+#
+# Generation installs the wasm builds the package ships, so they must be built first: threads
+# (node, cross-origin isolated browsers) and single-thread (browsers without SharedArrayBuffer).
+# Shipped uncompressed. npm tarballs are gzipped either way, so this costs nothing on install,
+# and it is the form a host's own compression and the browser's compiled-code cache both want:
+# only a real application/wasm response can be streamed straight into WebAssembly.compileStreaming
+# and cached. A consumer serving from a host that does not compress can point bb.js's wasmPath (or
+# BB_WASM_PATH) at a compressed copy instead; the loader recognises gzip.
 function generate_bb_js_api_package {
   local bbapi="$ROOT/barretenberg/cpp/src/barretenberg/bbapi"
   # bb.js keeps its historical API surface (poseidon2Hash, Poseidon2Hash), so the Bb
@@ -66,8 +74,8 @@ function generate_bb_js_api_package {
     --strip-type-prefix \
     --package-transports uds,shm,wasm \
     --package-ipc-path-args 'msgpack,run,--input,{path}' \
-    --package-wasm-module barretenberg.wasm \
-    --package-wasm-threads-module barretenberg-threads.wasm
+    --package-wasm-module "barretenberg.wasm=$ROOT/barretenberg/cpp/build-wasm/bin/barretenberg.wasm" \
+    --package-wasm-threads-module "barretenberg-threads.wasm=$ROOT/barretenberg/cpp/build-wasm-threads/bin/barretenberg.wasm"
 }
 
 # bb-avm-sim, cdb and bb.js-api are gitignored workspaces declared in package.json, so
@@ -79,23 +87,6 @@ function generate_packages {
   generate_bb_js_api_package
 }
 
-# The wasm builds the bb.js-api package ships: threads (node, cross-origin isolated
-# browsers) and single-thread (browsers without SharedArrayBuffer).
-#
-# Shipped uncompressed. npm tarballs are gzipped either way, so this costs nothing on install,
-# and it is the form a host's own compression and the browser's compiled-code cache both want:
-# only a real application/wasm response can be streamed straight into WebAssembly.compileStreaming
-# and cached. A consumer serving from a host that does not compress can point bb.js's wasmPath (or
-# BB_WASM_PATH) at a compressed copy instead; the loader recognises gzip.
-function copy_bb_js_api_wasm {
-  # Replace rather than add to: the package publishes everything under wasm/, so a module left
-  # from an earlier build would ship alongside the current one.
-  rm -rf bb.js-api/wasm
-  mkdir -p bb.js-api/wasm
-  cp "$ROOT/barretenberg/cpp/build-wasm-threads/bin/barretenberg.wasm" bb.js-api/wasm/barretenberg-threads.wasm
-  cp "$ROOT/barretenberg/cpp/build-wasm/bin/barretenberg.wasm" bb.js-api/wasm/barretenberg.wasm
-}
-
 function copy_bb_js_api_native {
   local target_dir="bb.js-api/build/$(arch)-$(os)"
   mkdir -p "$target_dir"
@@ -121,8 +112,9 @@ function prepare_bb_js_api_arch_packages {
   yarn workspace "$BB_JS_API_PACKAGE" run prepare_arch_packages "$@"
 }
 
-# Generate + compile the package bb.js compiles against, without the wasm/binary artifacts
-# (enough for type-checking, formatting and lint). Not cached: it is a few seconds of tsc.
+# Generate + compile the package bb.js compiles against, without bb binaries (enough for
+# type-checking, formatting and lint). Generation installs the wasm modules, so they must be
+# built first. Not cached: it is a few seconds of tsc.
 function build_bb_js_api_ts {
   generate_packages
   npm_install_deps "$IPC_RUNTIME_PKG"
@@ -134,7 +126,6 @@ function build_bb_js_api_ts {
 function build_bb_js_api {
   echo_header "bb.js-api package build"
   build_bb_js_api_ts
-  copy_bb_js_api_wasm
   copy_bb_js_api_native
   prepare_bb_js_api_arch_packages "$(arch)-$(os)=build/$(arch)-$(os)/bb"
 }
@@ -310,7 +301,6 @@ function cross_copy_bb_js_api {
 # bb.js depends on bb.js-api, so it is published first, with its arch packages.
 function release_bb_js_api {
   generate_packages
-  copy_bb_js_api_wasm
   copy_bb_js_api_native
   copy_bb_js_api_cross
   npm_install_deps "$IPC_RUNTIME_PKG"
@@ -331,7 +321,7 @@ function release {
 }
 
 export -f generate_bb_avm_sim_package copy_bb_avm_sim_native copy_bb_avm_sim_cross generate_cdb_package generate_packages
-export -f generate_bb_js_api_package copy_bb_js_api_wasm copy_bb_js_api_native copy_bb_js_api_cross prepare_bb_js_api_arch_packages build_bb_js_api_ts build_bb_js_api
+export -f generate_bb_js_api_package copy_bb_js_api_native copy_bb_js_api_cross prepare_bb_js_api_arch_packages build_bb_js_api_ts build_bb_js_api
 export -f build_bb_js build_bb_avm_sim build_cdb build cross_copy_bb_js cross_copy_bb_avm_sim cross_copy_bb_js_api release release_cdb release_bb_js_api
 
 case "$cmd" in

bootstrap.sh

@@ -1034,12 +1034,14 @@ case "$cmd" in
     export AVM_TRANSPILER=0
     # bb.js formatting only runs yarn in the barretenberg/ts workspace: it needs that workspace installed and the
     # generated @aztec-foundation/bb.js-api compiled (build_bb_js_api_ts does both), and its lint resolves
-    # @aztec-foundation/ipc-runtime through ipc-runtime's built dest/.
+    # @aztec-foundation/ipc-runtime through ipc-runtime's built dest/. Generating that package reads and
+    # installs bb's wasm builds, so bb is built first.
+    barretenberg/cpp/bootstrap.sh build
     ipc-runtime/bootstrap.sh build
     barretenberg/ts/bootstrap.sh build_bb_js_api_ts
     barretenberg/ts/bb.js/bootstrap.sh formatting
     barretenberg/crs/bootstrap.sh
-    barretenberg/cpp/bootstrap.sh ci
+    barretenberg/cpp/bootstrap.sh test
     ;;
   "ci-barretenberg-full")
     export CI=1

ipc-codegen/README.md

@@ -110,8 +110,8 @@ node --experimental-strip-types --no-warnings \
 | `--uds` | Rust/Zig only. Copies the `Backend` trait template (and `error.rs` for Rust) into `<out>` so consumers can plug ipc-runtime — or any custom transport — behind the generated client. The flag name is historical: the trait is transport-agnostic. |
 | `--ffi` | In-process FFI, both directions of the contract in `SCHEMA_SPEC.md` ("FFI entry"). With `--client` (Rust/Zig/C++): generates the FFI backend (`ffi_backend.{rs,zig}`, `<service>_ffi_backend.hpp`), which calls a linked library's `<service>_ipc_ffi_entry`. With `--server` (Rust/Zig/C++): emits the entry itself — `<service>_ffi.rs` with an `export_<service>_ffi!` macro over a `Handler + Default` type, `<service>_ffi.zig` with `exportFfi(Handler)` over a default-initialised handler, or `<service>_ffi.{hpp,cpp}` defining `<service>_ipc_ffi_entry`/`_alloc`/`_free` over the generated dispatch (the service defines `ipc_ffi_dispatcher()`). Each example's FFI test runs the generated client over the FFI backend against its own entry, in-process. The service prefix on the symbols lets several services be linked into one binary. A wasm reactor built from these is what the TS `wasm` transport runs. |
 | `--package-transports <t>` | TS `--package` only. Comma-separated transports the package offers: `uds`, `shm` (spawned process) and `wasm` (the service's wasm module, run in-process through `@aztec-foundation/ipc-runtime/wasm`; adds a browser entry). |
-| `--package-wasm-module <file>`, `--package-wasm-threads-module <file>` | TS `wasm` transport. Basenames of the single-thread and threads builds the owning project copies into the package's `wasm/` directory; at least one is required. |
-| `--package-wasm-wide-module <file>`, `--package-wasm-wide-threads-module <file>` | TS `wasm` transport. Optional wide-arithmetic builds of the two, also copied into `wasm/` by the owning project; the package runs one in place of its counterpart where the engine validates the wide-arithmetic proposal. |
+| `--package-wasm-module <name>=<path>`, `--package-wasm-threads-module <name>=<path>` | TS `wasm` transport. The single-thread and threads builds at `<path>`, installed into the package's `wasm/` directory as `<name>`; at least one is required. Generation reads each module's memory import and records it in the package, since a host must create an imported memory matching it and JS cannot read it from a compiled module. |
+| `--package-wasm-wide-module <name>=<path>`, `--package-wasm-wide-threads-module <name>=<path>` | TS `wasm` transport. Optional wide-arithmetic builds of the two, installed the same way; the package runs one in place of its counterpart where the engine validates the wide-arithmetic proposal. |
 
 ### Naming flags
 

ipc-codegen/echo_example/ts_package/bootstrap.sh

@@ -6,29 +6,29 @@ CODEGEN="$(cd "$DIR/../.." && pwd)"
 REPO_ROOT="$(cd "$CODEGEN/.." && pwd)"
 NODE="node --experimental-strip-types --no-warnings"
 
+# Both the spawned binary and the wasm reactor come from the Rust crate, so the package
+# offers one implementation over three transports.
+(cd "$DIR/../rust" && ./bootstrap.sh)
+
 $NODE "$CODEGEN/src/generate.ts" \
   --schema "$DIR/../schema/schema.jsonc" \
   --lang ts \
   --package "$DIR" \
   --package-name "@aztec/echo-ipc" \
   --binary-name echo_server \
   --package-transports uds,shm,wasm \
-  --package-wasm-module echo.wasm \
+  --package-wasm-module "echo.wasm=$DIR/../rust/target/wasm32-wasip1/release/echo_wire_compat.wasm" \
   --ipc-runtime-dependency "file:../../../ipc-runtime/ts"
 
-# Both the spawned binary and the wasm reactor come from the Rust crate, so the package
-# offers one implementation over three transports.
-(cd "$DIR/../rust" && ./bootstrap.sh)
 # ipc-runtime is built by the Makefile (ipc-codegen depends on it) so its ts/dest
 # and NAPI addon are ready for the file: link below; don't reinstall the shared
 # ipc-runtime/ts here — concurrent build units doing so corrupt its node_modules.
 
 platform_dir="$(
   node -e "const arch = { x64: 'amd64', arm64: 'arm64' }[process.arch] ?? process.arch; const os = { linux: 'linux', darwin: 'macos' }[process.platform] ?? process.platform; console.log(arch + '-' + os);"
 )"
-mkdir -p "$DIR/build/$platform_dir" "$DIR/wasm"
+mkdir -p "$DIR/build/$platform_dir"
 cp "$DIR/../rust/target/debug/echo_server" "$DIR/build/$platform_dir/echo_server"
-cp "$DIR/../rust/target/wasm32-wasip1/release/echo_wire_compat.wasm" "$DIR/wasm/echo.wasm"
 
 rm -rf "$DIR/node_modules"
 (cd "$DIR" && npm install --omit=optional --no-package-lock --quiet)