Skip to content

Commit 6363b74

Browse files
lvaroquiclaude
andcommitted
feat(upload): check multipart uploads against CRC64NVME checksums
Replace the md5 of each part and of the archive with CRC64NVME checksums, and describe the profile archive with a single `profileArchive` field in the upload metadata (version 12): its encoding, size and CRC64NVME, and the size and CRC64NVME of its parts. It replaces `profileEncoding`, `profileMd5` and `profileMultipart`. Each part is hashed once, and the part CRCs are combined into the archive's. S3 can check a CRC64NVME on the whole archive of a multipart upload, which md5 does not support, so a completion that leaves out a part or assembles another archive is rejected, and not only a corrupted part. It is also much cheaper to compute: about 0.5s for a 6 GiB archive on a GitHub-hosted runner, against 24s for the part and archive md5s. The upload endpoint now answers with `multipartUpload`, replacing `multipartUploadUrls`: a presigned request per part and one to complete the upload, each with the headers to send as is, as they are part of the signature. The runner sends them without knowing which ones S3 checks, so the endpoint can change them without a runner release. `crc-fast` is pinned to 1.9, the last release supporting Rust 1.88, which requires `crc` 3.3. Refs COD-3700 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 4e4c21f commit 6363b74

10 files changed

Lines changed: 253 additions & 191 deletions

‎Cargo.lock‎

Lines changed: 21 additions & 9 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ sha256 = "1.6"
2929
tokio = { version = "1", features = ["macros", "rt"] }
3030
tokio-tar = { package = "astral-tokio-tar", version = "0.6.2" }
3131
tokio-util = "0.7.18"
32-
md5 = "0.8"
32+
crc-fast = { version = "1.9.0", default-features = false, features = ["std"] }
3333
base64 = "0.22.1"
3434
async-compression = { version = "0.4.42", features = ["tokio", "gzip"] }
3535
schemars = "1.2.1"

‎src/run_environment/provider.rs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -136,9 +136,7 @@ pub trait RunEnvironmentProvider {
136136
tokenless: api_client.token().is_none(),
137137
repository_provider: self.get_repository_provider(),
138138
run_environment_metadata,
139-
profile_md5: profile_archive.md5.clone(),
140-
profile_encoding: profile_archive.content.encoding(),
141-
profile_multipart: profile_archive.multipart.clone(),
139+
profile_archive: profile_archive.metadata.clone(),
142140
commit_hash,
143141
allow_empty: config.allow_empty,
144142
runner: Runner {

‎src/upload/interfaces.rs‎

Lines changed: 48 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
use std::collections::BTreeMap;
2+
13
use serde::{Deserialize, Serialize};
24

35
use crate::executor::ExecutorName;
@@ -13,9 +15,7 @@ pub struct UploadMetadata {
1315
pub repository_provider: RepositoryProvider,
1416
pub version: Option<u32>,
1517
pub tokenless: bool,
16-
pub profile_md5: String,
17-
pub profile_encoding: Option<String>,
18-
pub profile_multipart: ProfileMultipart,
18+
pub profile_archive: ProfileMetadata,
1919
pub runner: Runner,
2020
pub run_environment: RunEnvironment,
2121
pub run_part: Option<RunPart>,
@@ -25,16 +25,20 @@ pub struct UploadMetadata {
2525
pub run_environment_metadata: RunEnvironmentMetadata,
2626
}
2727

28-
/// Layout of a profile archive uploaded as an S3 multipart upload, in consecutive
29-
/// `part_size` chunks, the last one holding the remainder. S3 requires parts of 5 MiB
30-
/// to 5 GiB (the last one excepted from the minimum), and at most 10,000 of them.
28+
/// Profile archive, uploaded as an S3 multipart upload in consecutive `part_size`
29+
/// chunks, the last one holding the remainder. S3 requires parts of 5 MiB to 5 GiB
30+
/// (the last one excepted from the minimum), and at most 10,000 of them.
3131
#[derive(Serialize, Debug, Clone, PartialEq)]
3232
#[serde(rename_all = "camelCase")]
33-
pub struct ProfileMultipart {
33+
pub struct ProfileMetadata {
34+
/// `Content-Encoding` of the archive, such as `gzip`
35+
pub encoding: Option<String>,
3436
pub size: u64,
37+
/// Base64 big-endian CRC64NVME of the whole archive
38+
pub crc64nvme: String,
3539
pub part_size: u64,
36-
/// Base64 md5 of each part, in upload order
37-
pub part_md5s: Vec<String>,
40+
/// Base64 big-endian CRC64NVME of each part, in upload order
41+
pub part_crc64nvmes: Vec<String>,
3842
}
3943

4044
#[derive(Serialize, Debug)]
@@ -59,17 +63,25 @@ pub struct Runner {
5963
#[serde(rename_all = "camelCase")]
6064
pub struct UploadData {
6165
pub status: String,
62-
pub multipart_upload_urls: MultipartUploadUrls,
66+
pub multipart_upload: MultipartUpload,
6367
pub run_id: String,
6468
}
6569

6670
#[derive(Deserialize, Serialize, Debug)]
6771
#[serde(rename_all = "camelCase")]
68-
pub struct MultipartUploadUrls {
69-
/// Presigned S3 `UploadPart` URLs, one per part, in upload order
70-
pub part_urls: Vec<String>,
71-
/// Presigned S3 `CompleteMultipartUpload` URL
72-
pub complete_url: String,
72+
pub struct MultipartUpload {
73+
/// Presigned S3 `UploadPart` requests, one per part, in upload order
74+
pub parts: Vec<PresignedRequest>,
75+
/// Presigned S3 `CompleteMultipartUpload` request
76+
pub complete: PresignedRequest,
77+
}
78+
79+
/// Request presigned by the API, to send to `url` with `headers` as is: they are
80+
/// part of the signature
81+
#[derive(Deserialize, Serialize, Debug)]
82+
pub struct PresignedRequest {
83+
pub url: String,
84+
pub headers: BTreeMap<String, String>,
7385
}
7486

7587
#[derive(Deserialize, Debug)]
@@ -88,17 +100,32 @@ mod tests {
88100
r#"{
89101
"status": "success",
90102
"runId": "run-id",
91-
"multipartUploadUrls": {
92-
"partUrls": ["https://part/1", "https://part/2"],
93-
"completeUrl": "https://complete"
103+
"multipartUpload": {
104+
"parts": [
105+
{ "url": "https://part/1", "headers": { "x-amz-checksum-crc64nvme": "nq48tdaL2no=" } },
106+
{ "url": "https://part/2", "headers": { "x-amz-checksum-crc64nvme": "XMXoclwBfLo=" } }
107+
],
108+
"complete": {
109+
"url": "https://complete",
110+
"headers": { "x-amz-checksum-type": "FULL_OBJECT" }
111+
}
94112
}
95113
}"#,
96114
)
97115
.unwrap();
98116

99117
assert_eq!(upload_data.run_id, "run-id");
100-
let urls = upload_data.multipart_upload_urls;
101-
assert_eq!(urls.part_urls, ["https://part/1", "https://part/2"]);
102-
assert_eq!(urls.complete_url, "https://complete");
118+
let upload = upload_data.multipart_upload;
119+
let part_urls: Vec<_> = upload.parts.iter().map(|part| part.url.as_str()).collect();
120+
assert_eq!(part_urls, ["https://part/1", "https://part/2"]);
121+
assert_eq!(
122+
upload.parts[1].headers,
123+
BTreeMap::from([("x-amz-checksum-crc64nvme".into(), "XMXoclwBfLo=".into())])
124+
);
125+
assert_eq!(upload.complete.url, "https://complete");
126+
assert_eq!(
127+
upload.complete.headers,
128+
BTreeMap::from([("x-amz-checksum-type".into(), "FULL_OBJECT".into())])
129+
);
103130
}
104131
}

0 commit comments

Comments
 (0)