docs/bevy/examples/remote/integration_test.rs 8.1 K · 229 lines · raw · history

1 //! An integration test that connects to a running Bevy app via the BRP,
2 //! finds a button's position, and sends a mouse click to press it.
3 //!
4 //! Run with the `bevy_remote` and `bevy_feathers` features enabled:
5 //! ```bash
6 //! cargo run --example integration_test --features="bevy_remote bevy_feathers"
7 //! ```
8 //! This example assumes that the `app_under_test` example is running on the same machine.
9
10 use std::{any::type_name, io::BufRead};
11
12 use anyhow::Result as AnyhowResult;
13 use bevy::{
14 feathers::controls::FeathersButton,
15 platform::collections::HashMap,
16 remote::{
17 builtin_methods::{
18 BrpObserveParams, BrpQuery, BrpQueryFilter, BrpQueryParams, BrpSpawnEntityParams,
19 BrpWriteMessageParams, ComponentSelector, BRP_OBSERVE_METHOD, BRP_QUERY_METHOD,
20 BRP_SPAWN_ENTITY_METHOD, BRP_WRITE_MESSAGE_METHOD,
21 },
22 http::{DEFAULT_ADDR, DEFAULT_PORT},
23 BrpRequest,
24 },
25 render::view::screenshot::{Screenshot, ScreenshotCaptured},
26 ui::UiGlobalTransform,
27 window::{Window, WindowEvent},
28 };
29
30 fn main() -> AnyhowResult<()> {
31 let url = format!("http://{DEFAULT_ADDR}:{DEFAULT_PORT}/");
32
33 // Step 1: Take a screenshot via BRP
34 // The window must be visible (not fully occluded) for the GPU to render content
35 // If the window is hidden, the screenshot will be black
36 println!("Spawning Screenshot entity...");
37 let spawn_response = brp_request(
38 &url,
39 BRP_SPAWN_ENTITY_METHOD,
40 1,
41 &BrpSpawnEntityParams {
42 components: HashMap::from([(
43 type_name::<Screenshot>().to_string(),
44 serde_json::json!({"Window": "Primary"}),
45 )]),
46 },
47 )?;
48 let screenshot_entity = &spawn_response["result"]["entity"];
49
50 println!("Observing ScreenshotCaptured on entity {screenshot_entity}...");
51 let observe_response = ureq::post(&url).send_json(BrpRequest {
52 method: BRP_OBSERVE_METHOD.to_string(),
53 id: Some(serde_json::to_value(2)?),
54 params: Some(serde_json::to_value(BrpObserveParams {
55 event: type_name::<ScreenshotCaptured>().to_string(),
56 entity: Some(serde_json::from_value(screenshot_entity.clone())?),
57 })?),
58 })?;
59
60 println!("Waiting for screenshot capture...");
61 let reader = std::io::BufReader::new(observe_response.into_body().into_reader());
62 for line in reader.lines() {
63 let line = line?;
64 if let Some(json_str) = line.strip_prefix("data: ") {
65 let response: serde_json::Value = serde_json::from_str(json_str)?;
66 if let Some(error) = response.get("error") {
67 anyhow::bail!("Observe error: {error}");
68 }
69 if let Some(result) = response.get("result") {
70 let events = result.as_array().expect("Expected events array");
71 let event = &events[0];
72
73 let image_data = &event["image"];
74 let width = image_data["texture_descriptor"]["size"]["width"]
75 .as_u64()
76 .unwrap();
77 let height = image_data["texture_descriptor"]["size"]["height"]
78 .as_u64()
79 .unwrap();
80 println!("Screenshot captured! Image size: {width}x{height}");
81
82 let image: bevy::image::Image = serde_json::from_value(image_data.clone())?;
83 let dyn_img = image
84 .try_into_dynamic()
85 .expect("Failed to convert screenshot to dynamic image");
86 let path = "screenshot.png";
87 dyn_img.to_rgb8().save(path)?;
88 println!("Screenshot saved to {path}");
89 break;
90 }
91 }
92 }
93
94 // Step 2: Find the button entity, and its global transform
95 println!("Querying for button entity...");
96 let button_query = brp_request(
97 &url,
98 BRP_QUERY_METHOD,
99 3,
100 &BrpQueryParams {
101 data: BrpQuery {
102 components: vec![type_name::<UiGlobalTransform>().to_string()],
103 option: ComponentSelector::default(),
104 has: Vec::default(),
105 },
106 strict: false,
107 filter: BrpQueryFilter {
108 with: vec![type_name::<FeathersButton>().to_string()],
109 without: Vec::default(),
110 },
111 },
112 )?;
113
114 let button_result = button_query["result"]
115 .as_array()
116 .expect("Expected result array");
117 let button = &button_result[0];
118
119 // UiGlobalTransform wraps an Affine2, serialized as a flat array:
120 // [_, _, _, _, translation_x, translation_y]
121 // The translation gives the node's center in physical pixels.
122 let transform = &button["components"][type_name::<UiGlobalTransform>()];
123 let transform_arr = transform.as_array().expect("Expected transform array");
124 let phys_x = transform_arr[4].as_f64().unwrap();
125 let phys_y = transform_arr[5].as_f64().unwrap();
126 println!("Found button at physical ({phys_x}, {phys_y})");
127
128 // Step 3: Find the window entity and scale factor
129 println!("Querying for window entity...");
130 let window_query = brp_request(
131 &url,
132 BRP_QUERY_METHOD,
133 4,
134 &BrpQueryParams {
135 data: BrpQuery {
136 components: vec![type_name::<Window>().to_string()],
137 option: ComponentSelector::default(),
138 has: Vec::default(),
139 },
140 strict: false,
141 filter: BrpQueryFilter::default(),
142 },
143 )?;
144
145 let window_result = window_query["result"]
146 .as_array()
147 .expect("Expected result array");
148 let window = &window_result[0];
149 let window_entity = &window["entity"];
150 let window_data = &window["components"][type_name::<Window>()];
151 let scale_factor = window_data["resolution"]["scale_factor"].as_f64().unwrap();
152 println!("Found window entity: {window_entity}, scale_factor: {scale_factor}");
153
154 // Step 4: Convert button center from physical to logical pixels
155 let logical_x = phys_x / scale_factor;
156 let logical_y = phys_y / scale_factor;
157 println!("Clicking at logical position: ({logical_x}, {logical_y})");
158
159 // Step 5: Send CursorMoved via WindowEvent message
160 // This lets the picking system know where the pointer is.
161 println!("Sending CursorMoved message...");
162 brp_request(
163 &url,
164 BRP_WRITE_MESSAGE_METHOD,
165 5,
166 &BrpWriteMessageParams {
167 message: type_name::<WindowEvent>().to_string(),
168 value: Some(serde_json::json!({
169 "CursorMoved": {
170 "window": window_entity,
171 "position": [logical_x, logical_y],
172 "delta": null
173 }
174 })),
175 },
176 )?;
177
178 // Step 6: Send MouseButtonInput Pressed + Released via WindowEvent messages.
179 // The picking system needs both press and release to generate a PointerClick.
180 println!("Sending mouse press...");
181 brp_request(
182 &url,
183 BRP_WRITE_MESSAGE_METHOD,
184 6,
185 &BrpWriteMessageParams {
186 message: type_name::<WindowEvent>().to_string(),
187 value: Some(serde_json::json!({
188 "MouseButtonInput": {
189 "button": "Left",
190 "state": "Pressed",
191 "window": window_entity,
192 }
193 })),
194 },
195 )?;
196
197 println!("Sending mouse release...");
198 brp_request(
199 &url,
200 BRP_WRITE_MESSAGE_METHOD,
201 7,
202 &BrpWriteMessageParams {
203 message: type_name::<WindowEvent>().to_string(),
204 value: Some(serde_json::json!({
205 "MouseButtonInput": {
206 "button": "Left",
207 "state": "Released",
208 "window": window_entity,
209 }
210 })),
211 },
212 )?;
213
214 Ok(())
215 }
216
217 fn brp_request(
218 url: &str,
219 method: &str,
220 id: u32,
221 params: &impl serde::Serialize,
222 ) -> AnyhowResult<serde_json::Value> {
223 let req = BrpRequest {
224 method: method.to_string(),
225 id: Some(serde_json::to_value(id)?),
226 params: Some(serde_json::to_value(params)?),
227 };
228 Ok(ureq::post(url).send_json(req)?.body_mut().read_json()?)
229 }