Render thumbnails with Vello (#4805)

Author: TrueDoctor

Cargo.lock

@@ -15,7 +15,7 @@ workspace = true
 
 [features]
 default = ["gpu"]
-wasm = ["graphene-std/wasm", "interpreted-executor/wasm", "dep:wasm-bindgen"]
+wasm = ["graphene-std/wasm", "interpreted-executor/wasm", "dep:wasm-bindgen", "dep:wasm-bindgen-futures"]
 gpu = ["interpreted-executor/gpu", "dep:wgpu-executor"]
 
 [dependencies]
@@ -64,6 +64,7 @@ wgpu-executor = { workspace = true, optional = true }
 
 # Optional workspace dependencies
 wasm-bindgen = { workspace = true, optional = true }
+wasm-bindgen-futures = { workspace = true, optional = true }
 
 [target.'cfg(not(target_family = "wasm"))'.dependencies]
 tokio = { workspace = true }

editor/Cargo.toml

@@ -332,6 +332,19 @@ pub enum FrontendMessage {
 	ClearNodeThumbnails {
 		ids: Vec<NodeId>,
 	},
+	UpdateNodeThumbnailPixels {
+		id: NodeId,
+		width: u32,
+		height: u32,
+		pixels: serde_bytes::ByteBuf,
+	},
+	UpdateNodeThumbnailCanvases {
+		id: NodeId,
+		#[serde(rename = "layersPanelCanvasId")]
+		layers_panel_canvas_id: u64,
+		#[serde(rename = "nodeGraphCanvasId")]
+		node_graph_canvas_id: u64,
+	},
 	UpdateOpenDocumentsList {
 		#[serde(rename = "openDocuments")]
 		open_documents: Vec<DocumentInfo>,

editor/src/messages/frontend/frontend_message.rs

@@ -10,8 +10,8 @@ pub struct LayersInView {
 	layers_panel: HashSet<NodeId>,
 	/// The nodes in view in the node graph while it's open.
 	node_graph: HashSet<NodeId>,
-	/// The latest thumbnail of each layer, as rendered by the graph runtime.
-	thumbnails: HashMap<NodeId, String>,
+	/// The message with the latest thumbnail of each layer, as rendered by the graph runtime.
+	thumbnails: HashMap<NodeId, FrontendMessage>,
 	/// The layers whose latest thumbnail the frontend has.
 	thumbnails_sent: HashSet<NodeId>,
 }
@@ -40,10 +40,17 @@ impl LayersInView {
 		responses.add(PortfolioMessage::SubmitThumbnailsInView);
 	}
 
-	/// Keeps a layer's newly rendered thumbnail, sending it to the frontend only if the layer is in view.
-	pub fn receive_thumbnail(&mut self, layer: NodeId, thumbnail: String, responses: &mut VecDeque<Message>) {
+	pub fn thumbnail_layer(message: &FrontendMessage) -> Option<NodeId> {
+		match message {
+			FrontendMessage::UpdateNodeThumbnail { id, .. } | FrontendMessage::UpdateNodeThumbnailCanvases { id, .. } | FrontendMessage::UpdateNodeThumbnailPixels { id, .. } => Some(*id),
+			_ => None,
+		}
+	}
+
+	/// Keeps the message with a layer's newly rendered thumbnail, sending it to the frontend only if the layer is in view.
+	pub fn receive_thumbnail(&mut self, layer: NodeId, thumbnail: FrontendMessage, responses: &mut VecDeque<Message>) {
 		if self.in_view(&layer) {
-			responses.add(FrontendMessage::UpdateNodeThumbnail { id: layer, value: thumbnail.clone() });
+			responses.add(thumbnail.clone());
 			self.thumbnails_sent.insert(layer);
 		} else {
 			// Out of view, the frontend's copy is now stale until the layer comes back into view
@@ -84,7 +91,7 @@ impl LayersInView {
 			}
 			let Some(thumbnail) = self.thumbnails.get(&layer) else { continue };
 
-			responses.add(FrontendMessage::UpdateNodeThumbnail { id: layer, value: thumbnail.clone() });
+			responses.add(thumbnail.clone());
 			self.thumbnails_sent.insert(layer);
 		}
 	}
@@ -94,6 +101,10 @@ impl LayersInView {
 mod tests {
 	use super::*;
 
+	fn svg_thumbnail(layer: NodeId, svg: &str) -> FrontendMessage {
+		FrontendMessage::UpdateNodeThumbnail { id: layer, value: svg.to_string() }
+	}
+
 	fn thumbnails_sent(responses: VecDeque<Message>) -> Vec<(NodeId, String)> {
 		responses
 			.into_iter()
@@ -110,8 +121,8 @@ mod tests {
 		let mut layers_in_view = LayersInView::default();
 
 		let mut responses = VecDeque::new();
-		layers_in_view.receive_thumbnail(layer, "first".to_string(), &mut responses);
-		layers_in_view.receive_thumbnail(layer, "latest".to_string(), &mut responses);
+		layers_in_view.receive_thumbnail(layer, svg_thumbnail(layer, "first"), &mut responses);
+		layers_in_view.receive_thumbnail(layer, svg_thumbnail(layer, "latest"), &mut responses);
 		assert!(thumbnails_sent(responses).is_empty(), "out of view, a thumbnail isn't sent");
 
 		let mut responses = VecDeque::new();
@@ -130,12 +141,12 @@ mod tests {
 
 		let mut responses = VecDeque::new();
 		layers_in_view.set_layers_panel(HashSet::from([layer]), &mut responses);
-		layers_in_view.receive_thumbnail(layer, "in view".to_string(), &mut responses);
+		layers_in_view.receive_thumbnail(layer, svg_thumbnail(layer, "in view"), &mut responses);
 		assert_eq!(thumbnails_sent(responses), vec![(layer, "in view".to_string())], "in view, a thumbnail is sent as it's received");
 
 		let mut responses = VecDeque::new();
 		layers_in_view.set_layers_panel(HashSet::new(), &mut responses);
-		layers_in_view.receive_thumbnail(layer, "changed".to_string(), &mut responses);
+		layers_in_view.receive_thumbnail(layer, svg_thumbnail(layer, "changed"), &mut responses);
 		layers_in_view.set_layers_panel(HashSet::from([layer]), &mut responses);
 		assert_eq!(thumbnails_sent(responses), vec![(layer, "changed".to_string())]);
 	}
@@ -147,7 +158,7 @@ mod tests {
 
 		let mut responses = VecDeque::new();
 		layers_in_view.set_layers_panel(HashSet::from([layer]), &mut responses);
-		layers_in_view.receive_thumbnail(layer, "before".to_string(), &mut responses);
+		layers_in_view.receive_thumbnail(layer, svg_thumbnail(layer, "before"), &mut responses);
 
 		let mut responses = VecDeque::new();
 		layers_in_view.forget_thumbnails(&[layer, other_documents_layer], &mut responses);
@@ -163,7 +174,7 @@ mod tests {
 		assert!(thumbnails_sent(responses).is_empty(), "a forgotten thumbnail isn't sent again");
 
 		let mut responses = VecDeque::new();
-		layers_in_view.receive_thumbnail(layer, "after".to_string(), &mut responses);
+		layers_in_view.receive_thumbnail(layer, svg_thumbnail(layer, "after"), &mut responses);
 		assert_eq!(thumbnails_sent(responses), vec![(layer, "after".to_string())], "rendered anew, it's sent as before");
 	}
 }

editor/src/messages/portfolio/document/utility_types/layers_in_view.rs

@@ -1,5 +1,6 @@
 use crate::messages::frontend::utility_types::{ExportBounds, FileType, RasterizedImage, RenderStatus};
 use crate::messages::portfolio::document::node_graph::utility_types::NodeEvaluationError;
+use crate::messages::portfolio::document::utility_types::layers_in_view::LayersInView;
 use crate::messages::portfolio::document::utility_types::network_interface::InputConnector;
 use crate::messages::prelude::*;
 use glam::{DAffine2, DVec2, UVec2};
@@ -497,14 +498,14 @@ impl NodeGraphExecutor {
 
 					// Thumbnails reach the frontend only for the layers it shows in view
 					for response in existing_responses {
-						match response {
-							FrontendMessage::UpdateNodeThumbnail { id, value } => {
+						match LayersInView::thumbnail_layer(&response) {
+							Some(layer) => {
 								// A render finishing after a switch to another document has thumbnails of the previous one's layers
 								if execution_context.document_id == document_id {
-									document.layers_in_view.receive_thumbnail(id, value, responses);
+									document.layers_in_view.receive_thumbnail(layer, response, responses);
 								}
 							}
-							response => responses.add(response),
+							None => responses.add(response),
 						}
 					}
 					document.network_interface.update_vector_modify(vector_modify);
@@ -616,8 +617,9 @@ impl NodeGraphExecutor {
 
 	fn receive_thumbnails(document: &mut DocumentMessageHandler, thumbnails: VecDeque<FrontendMessage>, responses: &mut VecDeque<Message>) {
 		for thumbnail in thumbnails {
-			if let FrontendMessage::UpdateNodeThumbnail { id, value } = thumbnail {
-				document.layers_in_view.receive_thumbnail(id, value, responses);
+			match LayersInView::thumbnail_layer(&thumbnail) {
+				Some(layer) => document.layers_in_view.receive_thumbnail(layer, thumbnail, responses),
+				None => responses.add(thumbnail),
 			}
 		}
 	}