diff --git a/README.md b/README.md index 1f7f074b..f578722a 100644 --- a/README.md +++ b/README.md @@ -764,6 +764,7 @@ See [`example-configs/`](./example-configs/) for annotated templates (`evm.base. | `rbeTlsClientCert` | string (optional) | TLS client certificate path, replaces the credential helper (`siso` only) | | `rbeTlsClientKey` | string (optional) | Private key path for `rbeTlsClientCert` (`siso` only) | | `rbePlatformProperties` | object (optional) | Remote platform properties, replaces the default backend config (`siso` only) | +| `rbeRemoteJobs` | integer (default: 200) | Maximum number of build steps to run remotely at once (`siso` only) | | `preserveSDK` | integer (default: 5) | Number of recent macOS SDKs to keep on disk | | `configValidationLevel` | `strict` \| `warn` \| `none` | How strictly to validate the config file (default: `strict`) | | `extends` | string (optional) | Name of a base config to inherit from; arrays concatenate and objects deep-merge | @@ -818,6 +819,8 @@ gen: - Some backends reject outputs outside the working directory. Add `clang_diagnostic_dir = "clang-crashreports"` to `gen.args` to keep the clang crash directory in the output directory. +- Set `rbeRemoteJobs` to match the capacity of the backend. For a single build, pass the flag + instead: `e build -remote_jobs 500`. ## Environment variables diff --git a/evm-config.schema.json b/evm-config.schema.json index e6397053..500cae24 100644 --- a/evm-config.schema.json +++ b/evm-config.schema.json @@ -68,6 +68,13 @@ }, "description": "Remote platform properties, replacing the default backend config. siso only" }, + "rbeRemoteJobs": { + "type": "integer", + "minimum": 1, + "maximum": 9007199254740991, + "description": "Maximum number of build steps to run remotely at once. siso only", + "default": 200 + }, "root": { "type": "string", "minLength": 1, diff --git a/src/e-build.ts b/src/e-build.ts index 6ef62c60..8a43f2ce 100644 --- a/src/e-build.ts +++ b/src/e-build.ts @@ -95,7 +95,7 @@ async function runNinja( } if (config.remoteBuild === 'siso') { - ninjaArgs.push(...siso.flags(config, hasExecute).map(String)); + ninjaArgs.push(...siso.flags(config, hasExecute, ninjaArgs).map(String)); } } else { console.info(`${color.info} Building ${target} with remote execution disabled`); diff --git a/src/types.ts b/src/types.ts index cbc825b9..4c0f0d4d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -79,6 +79,13 @@ const baseConfigShape = z.strictObject({ .record(z.string(), z.string()) .describe('Remote platform properties, replacing the default backend config. siso only') .optional(), + rbeRemoteJobs: z + .number() + .int() + .min(1) + .describe('Maximum number of build steps to run remotely at once. siso only') + .meta({ default: 200 }) + .optional(), root: z .string() .min(1) diff --git a/src/utils/siso.ts b/src/utils/siso.ts index 7af7e176..0ab991ed 100644 --- a/src/utils/siso.ts +++ b/src/utils/siso.ts @@ -10,6 +10,8 @@ import type { SanitizedConfig } from '../types.js'; // scoped to); everything else uses the shared CI/local namespace. const DEFAULT_REAPI_INSTANCE = 'projects/electron-rbe/instances/default_instance'; +const DEFAULT_REMOTE_JOBS = 200; + type ConfigLike = Pick< SanitizedConfig, | 'remoteBuild' @@ -19,6 +21,7 @@ type ConfigLike = Pick< | 'rbeTlsClientCert' | 'rbeTlsClientKey' | 'rbePlatformProperties' + | 'rbeRemoteJobs' | 'root' >; @@ -53,12 +56,22 @@ function getStarFile(envVar: string, filename: string): string { return path.resolve(import.meta.dirname, '../../tools', filename); } -export function flags(config: ConfigLike, hasExecute: boolean): (string | number)[] { +export function flags( + config: ConfigLike, + hasExecute: boolean, + userArgs: string[] = [], +): (string | number)[] { if (config.remoteBuild !== 'siso') return []; - const result: (string | number)[] = [ - '-remote_jobs', - 200, + const result: (string | number)[] = []; + + // These flags go after the user's and siso keeps the last value it sees, so + // leave -remote_jobs out when the user passed their own. + if (!userArgs.some((arg) => /^--?remote_jobs(=|$)/.test(arg.trim()))) { + result.push('-remote_jobs', config.rbeRemoteJobs ?? DEFAULT_REMOTE_JOBS); + } + + result.push( // Keep each value in its flag's token: the Windows shell join drops empty arguments. `-project=${project(config)}`, `-reapi_instance=${reapiInstance(config)}`, @@ -66,7 +79,7 @@ export function flags(config: ConfigLike, hasExecute: boolean): (string | number reclient.serviceAddress(config), '-load', getStarFile('ELECTRON_BUILD_TOOLS_MAIN_STAR', 'main.star'), - ]; + ); if (reclient.usesTlsClientAuth(config)) { result.push( diff --git a/tests/siso.spec.mjs b/tests/siso.spec.mjs index dee8d3df..c9728af2 100644 --- a/tests/siso.spec.mjs +++ b/tests/siso.spec.mjs @@ -68,6 +68,33 @@ describe('siso alternative RBE backends', () => { }); }); +describe('siso remote jobs', () => { + const remoteJobs = (result) => result[result.indexOf('-remote_jobs') + 1]; + + it('defaults to 200', () => { + expect(remoteJobs(flags({ remoteBuild: 'siso' }, true))).toBe(200); + }); + + it('uses rbeRemoteJobs when configured', () => { + expect(remoteJobs(flags({ remoteBuild: 'siso', rbeRemoteJobs: 500 }, true))).toBe(500); + }); + + it.each([ + [['-remote_jobs', '50']], + [['-remote_jobs=50']], + [['--remote_jobs', '50']], + [['--remote_jobs=50']], + ])('leaves the flag out when the user passes %j', (userArgs) => { + const result = flags({ remoteBuild: 'siso', rbeRemoteJobs: 500 }, true, userArgs); + expect(result).not.toContain('-remote_jobs'); + }); + + it('is not fooled by other flags', () => { + const result = flags({ remoteBuild: 'siso' }, true, ['-local_jobs', '8', '-remote_jobs_foo=1']); + expect(remoteJobs(result)).toBe(200); + }); +}); + describe('ensureBackendStarlark', () => { let root; const backendStar = () => path.join(root, 'src/build/config/siso/backend_config/backend.star');