From 216a7ef2df32bea97c903cf2d301aa3dd1274588 Mon Sep 17 00:00:00 2001 From: Daniel McNab <36049421+DJMcNab@users.noreply.github.com> Date: Thu, 3 Oct 2024 08:23:01 +0100 Subject: [PATCH] More experimentation --- examples/with_winit/Cargo.toml | 6 +- examples/with_winit/src/lib.rs | 117 ++++++++++++++++----- vello_pacing/src/choreographed.rs | 167 +++++++++++++++++++++++++++--- 3 files changed, 248 insertions(+), 42 deletions(-) diff --git a/examples/with_winit/Cargo.toml b/examples/with_winit/Cargo.toml index f85c31fe0..7c26c491c 100644 --- a/examples/with_winit/Cargo.toml +++ b/examples/with_winit/Cargo.toml @@ -63,7 +63,7 @@ tracing-subscriber = { version = "0.3.18", default-features = false, features = "registry", ] } profiling = { version = "1.0.15", features = ["profile-with-tracing"] } -ndk = { version = "0.9", features = ["api-level-33"] } +ndk = { version = "0.9", features = ["api-level-33", "nativewindow"] } [target.'cfg(target_arch = "wasm32")'.dependencies] console_error_panic_hook = "0.1.7" @@ -75,3 +75,7 @@ getrandom = { version = "0.2.15", features = ["js"] } [package.metadata.android.application] debuggable = true + +[package.metadata.android.sdk] +target_sdk_version = 33 +min_sdk_version = 33 diff --git a/examples/with_winit/src/lib.rs b/examples/with_winit/src/lib.rs index 54acd3150..fd093d918 100644 --- a/examples/with_winit/src/lib.rs +++ b/examples/with_winit/src/lib.rs @@ -6,6 +6,7 @@ use std::collections::HashSet; use std::num::NonZeroUsize; +use std::rc::Rc; use std::sync::Arc; #[cfg(not(target_arch = "wasm32"))] @@ -15,6 +16,8 @@ use web_time::Instant; use winit::application::ApplicationHandler; use winit::event::*; use winit::keyboard::*; +use winit::raw_window_handle::HasRawWindowHandle; +use winit::raw_window_handle::HasWindowHandle; use winit::window::WindowId; #[cfg(all(feature = "wgpu-profiler", not(target_arch = "wasm32")))] @@ -168,7 +171,7 @@ struct VelloApp<'s> { modifiers: ModifiersState, debug: vello::DebugLayers, - choreographer: Option, + choreographer: Option>, animation_in_flight: bool, proxy: winit::event_loop::EventLoopProxy, } @@ -405,13 +408,45 @@ impl<'s> ApplicationHandler for VelloApp<'s> { // in a touch context (i.e. Windows/Linux/MacOS with a touch screen could // also be using mouse/keyboard controls) // Note that winit's rendering is y-down - if let Some(RenderState { surface, .. }) = &self.state { + if let Some(RenderState { surface, window }) = &self.state { if touch.location.y > surface.config.height as f64 * 2. / 3. { self.navigation_fingers.insert(touch.id); // The left third of the navigation zone navigates backwards if touch.location.x < surface.config.width as f64 / 3. { + if let wgpu::rwh::RawWindowHandle::AndroidNdk( + android_ndk_window_handle, + ) = window.window_handle().unwrap().as_raw() + { + let window = unsafe { + ndk::native_window::NativeWindow::clone_from_ptr( + android_ndk_window_handle.a_native_window.cast(), + ) + }; + window + .set_frame_rate( + 60., + ndk::native_window::FrameRateCompatibility::Default, + ) + .unwrap(); + } self.scene_ix = self.scene_ix.saturating_sub(1); } else if touch.location.x > 2. * surface.config.width as f64 / 3. { + if let wgpu::rwh::RawWindowHandle::AndroidNdk( + android_ndk_window_handle, + ) = window.window_handle().unwrap().as_raw() + { + let window = unsafe { + ndk::native_window::NativeWindow::clone_from_ptr( + android_ndk_window_handle.a_native_window.cast(), + ) + }; + window + .set_frame_rate( + 90., + ndk::native_window::FrameRateCompatibility::Default, + ) + .unwrap(); + } self.scene_ix = self.scene_ix.saturating_add(1); } } @@ -609,9 +644,9 @@ impl<'s> ApplicationHandler for VelloApp<'s> { // let result = display_timing.get_refresh_cycle_duration(swc); // eprintln!("Refresh duration: {result:?}"); if present_id % 5 == 0 { - let result = display_timing.get_past_presentation_timing(swc); - eprintln!("Display timings: {result:?}"); - eprintln!("Most recent present id: {}", present_id); + // let result = display_timing.get_past_presentation_timing(swc); + // eprintln!("Display timings: {result:?}"); + // eprintln!("Most recent present id: {}", present_id); } } } @@ -664,27 +699,27 @@ impl<'s> ApplicationHandler for VelloApp<'s> { if let Some(choreographer) = self.choreographer.as_ref() { let proxy = self.proxy.clone(); - choreographer.post_vsync_callback(Box::new(move |frame| { - // eprintln!("New frame"); - // let frame_time = frame.frame_time(); - // let preferred_index = frame.preferred_frame_timeline_index(); - // for timeline in 0..frame.frame_timelines_length() { - // eprintln!( - // "{:?} {}", - // frame.frame_timeline_deadline(timeline) - frame_time, - // if timeline == preferred_index { - // "(Preferred)" - // } else { - // "" - // } - // ); - // } - // eprintln!("{frame:?}"); - proxy - .send_event(UserEvent::ChoreographerFrame(window_id)) - .unwrap(); - })); - self.animation_in_flight = true; + // choreographer.post_vsync_callback(Box::new(move |frame| { + // eprintln!("New frame"); + // let frame_time = frame.frame_time(); + // let preferred_index = frame.preferred_frame_timeline_index(); + // for timeline in 0..(frame.frame_timelines_length().min(3)) { + // eprintln!( + // "{:?} {}", + // frame.frame_timeline_deadline(timeline) - frame_time, + // if timeline == preferred_index { + // "(Preferred)" + // } else { + // "" + // } + // ); + // } + // eprintln!("{frame:?}"); + // // proxy + // // .send_event(UserEvent::ChoreographerFrame(window_id)) + // // .unwrap(); + // })); + window.request_redraw(); } else { window.request_redraw(); } @@ -854,10 +889,38 @@ fn run( modifiers: ModifiersState::default(), debug, // We know looper is active since we have the `EventLoop` - choreographer: ndk::choreographer::Choreographer::instance(), + choreographer: ndk::choreographer::Choreographer::instance().map(Rc::new), proxy: event_loop.create_proxy(), animation_in_flight: false, }; + if let Some(choreographer) = app.choreographer.as_ref() { + fn post_callback(choreographer: &Rc) { + let new_choreographer = Rc::clone(choreographer); + choreographer.post_vsync_callback(Box::new(move |frame| { + eprintln!("New frame"); + let frame_time = frame.frame_time(); + let preferred_index = frame.preferred_frame_timeline_index(); + for timeline in 0..(frame.frame_timelines_length().min(4)) { + eprintln!( + "{:?} {}", + frame.frame_timeline_deadline(timeline) - frame_time, + if timeline == preferred_index { + "(Preferred)" + } else { + "" + } + ); + } + eprintln!("{frame:?}"); + post_callback(&new_choreographer); + })); + } + // post_callback(choreographer); + choreographer.register_refresh_rate_callback(Box::new(|value| { + let span = tracing::info_span!("Getting a new refresh rate", ?value).entered(); + eprintln!("New refresh rate Testing: {value:?}; {}", value.as_nanos()); + })); + } event_loop.run_app(&mut app).expect("run to completion"); } diff --git a/vello_pacing/src/choreographed.rs b/vello_pacing/src/choreographed.rs index aea07faca..9f2617214 100644 --- a/vello_pacing/src/choreographed.rs +++ b/vello_pacing/src/choreographed.rs @@ -1,6 +1,21 @@ -use std::ops::Mul; - +#![allow(unused)] +#![warn(unused_variables)] + +use std::{ + cell::RefCell, + mem::ManuallyDrop, + ops::Mul, + rc::Rc, + sync::mpsc::{self, Receiver}, + time::Duration, +}; + +use ndk::{ + choreographer::Choreographer, + looper::{self, ForeignLooper}, +}; use nix::time::ClockId; +use vello::Scene; /// A slightly tweaked version of the thinking, now that we have an understanding of `AChoreographer`. /// @@ -34,6 +49,30 @@ use nix::time::ClockId; /// That doesn't actually change any behaviour here, because we render with `MailBox`. pub struct Thinking; +enum PacingCommand {} + +pub struct PacingChannel { + waker: ForeignLooper, + channel: ManuallyDrop>, +} + +impl PacingChannel { + fn send_command(&self, command: PacingCommand) { + self.channel.send(command); + // We add to the channel before waking, so that the event will be received by the right wake. + self.waker.wake(); + } +} + +impl Drop for PacingChannel { + fn drop(&mut self) { + // Safety: We don't use `self.channel` after this line. + // We drop the value before performing the wake, so that we instantly know that the drop has happened. + unsafe { ManuallyDrop::drop(&mut self.channel) }; + self.waker.wake(); + } +} + // We generally want to be thinking about two frames at a time, except for some statistical modelling. /// A timestamp in `CLOCK_MONOTONIC` @@ -42,7 +81,8 @@ pub struct Thinking; /// We'll validate that GPU performance counter timestamps meet this expectation as it becomes relevant. /// /// This might not actually be true - the timebase of the return values from [`ndk::choreographer::ChoreographerFrameCallbackData`] -/// aren't documented by +/// aren't documented by anything to be `CLOCK_MONOTONIC`, and I suspect we'll need to use [`ash::khr::calibrated_timestamps`] to get +/// the proper results. struct Timestamp(i64); impl Mul for Timestamp { @@ -67,9 +107,6 @@ impl Timestamp { } /// Get the current time in `CLOCK_MONOTONIC`. - /// - /// TODO: This assumed the returned value is not negative. - /// Hopefully that's fine? fn now() -> Self { let spec = nix::time::clock_gettime(ClockId::CLOCK_MONOTONIC).unwrap(); Self(spec.tv_sec() * 1_000_000_000 + spec.tv_nsec()) @@ -77,37 +114,139 @@ impl Timestamp { } /// A margin of latency which we *always* render against for safety. -const DEADLINE_MARGIN: Timestamp = Timestamp::from_millis(2); +const DEADLINE_MARGIN: Timestamp = Timestamp::from_millis(3); /// A margin of latency before the deadline, which if we aren't before, we assume that the /// frame probably missed the deadline. /// -/// In those cases, we bring future frames forward. +/// In those cases, we bring future frames forward to try and avoid a cascading stutter +/// (and instead maintain only a dropped frame). +/// +/// Note that this frame might still have technically counted as hitting the deadline. +/// However, we think a prolonged timing mismatch is worse than one dropped frame. const DEADLINE_ASSUME_FAILED: Timestamp = Timestamp::from_micros(500); +/// The time within which we expect our estimations to be correct. +const EXPECTED_CONSISTENCY: Timestamp = Timestamp::from_millis(3); + struct OngoingFrame { - /// The [present time][ndk::choreographer::ChoreographerFrameCallbackData::frame_timeline_expected_presentation_time]. + /// The [present time][ndk::choreographer::ChoreographerFrameCallbackData::frame_timeline_expected_presentation_time] we + /// expect this frame to be displayed at. target_present_time: Timestamp, /// The [vsync id][ndk::choreographer::ChoreographerFrameCallbackData::frame_timeline_vsync_id] we're aiming for. target_vsync_id: i64, - /// The deadline which this frame needs to meet to be rendered at `target_present_time`. + /// The [deadline][ndk::choreographer::ChoreographerFrameCallbackData::frame_timeline_deadline] which this frame needs to meet to be rendered at `target_present_time`. /// /// We aim for a time [`DEADLINE_MARGIN`] before the deadline. target_deadline: Timestamp, /// The time at which we wanted to start this frame. /// - /// `start_time` should try to be `requested_start_time - EPSILON`, + /// `cpu_start_time` should try to be `requested_cpu_start_time - EPSILON`, /// but if this is far off, we know early that we might drop this frame (and so should request /// the next frame super early). /// If this is significantly off, then we will likely drop this frame to avoid stuttering. - requested_start_time: Timestamp, + requested_cpu_start_time: Timestamp, /// The time at which `Scene` [rendering](`vello::Renderer::render_to_texture`) began. /// /// TODO: Does this include `Scene` construction time? - start_time: Timestamp, + cpu_start_time: Timestamp, /// The time at which [`wgpu::Queue::submit`] finished for this frame. - submit_time: Timestamp, + /// + /// If this is "much" later than + cpu_submit_time: Timestamp, + + /// The time at which work on the GPU started. + gpu_start_time: Timestamp, + /// The time at which work on the GPU finished. + /// + /// This should be before `target_deadline`. + /// `gpu_finish_time` - `cpu_start_time` is used to estimate how long a total frame takes + /// (and `gpu_finished_time` - `cpu_submit_time`) is used to estimate if a submission has + /// missed a deadline. + /// + /// There is some really interesting trickery we can do here; the *next* frame + /// on the GPU can definitely know this value, and can compare it against the deadline. + /// If we know that the submitted frame will miss the deadline, then we can. + gpu_finish_time: Timestamp, +} + +struct VelloPacingController { + choreographer: Choreographer, + command_rx: Receiver, + /// The duration of each frame, as reported by the system. + /// + /// For a short time, we don't have the refresh rate. + /// + /// This is used to detect the case where `AChoreographer` is giving us incorrect future vsyncs. + refresh_rate: Option, + looper: looper::ThreadLooper, +} + +/// We need to use a shared +type SharedPacing = Rc>; + +enum GpuCommand { + Render(Scene), + Resize(u32, u32, Scene), +} + +pub fn launch_pacing() -> PacingChannel { + let (channel_tx, channel_rx) = std::sync::mpsc::sync_channel(0); + // TODO: Give thread a name + std::thread::spawn(|| { + let looper = looper::ThreadLooper::prepare(); + let waker = looper.as_foreign().clone(); + let (command_tx, command_rx) = std::sync::mpsc::channel(); + channel_tx.send(PacingChannel { + waker, + channel: ManuallyDrop::new(command_tx), + }); + drop(channel_tx); + let choreographer = Choreographer::instance().expect("We just made the `Looper`"); + + let state = VelloPacingController { + choreographer, + command_rx, + refresh_rate: None, + looper: looper::ThreadLooper::for_thread().unwrap(), + }; + let state = Rc::new(RefCell::new(state)); + { + let callback_state = Rc::clone(&state); + let state = state.borrow(); + state + .choreographer + .register_refresh_rate_callback(Box::new(move |rate| { + let mut state = callback_state.borrow_mut(); + state.refresh_rate = Some(rate); + })); + } + let (gpu_tx, gpu_rx) = std::sync::mpsc::channel::(); + // TODO: Give thread a name + std::thread::spawn(|| { + // We perform all GPU work on this thread + // Since submitting work and polling the GPU are mutually exclusive, we + }); + loop { + let poll = looper.poll_once().expect("'Unrecoverable' error"); + assert!( + !matches!(poll, looper::Poll::Timeout | looper::Poll::Event { .. }), + "Impossible poll results from our use of Looper APIs." + ); + let state = state.borrow_mut(); + match state.command_rx.try_recv() { + Ok(command) => { + // Action `command` + } + Err(mpsc::TryRecvError::Disconnected) => {} + Err(mpsc::TryRecvError::Empty) => {} + } + } + }); + channel_rx + .recv_timeout(Duration::from_millis(100)) + .expect("Could not create pacing controller") }