From 6dc76f4befe0c148ca22fb261c5b47dd3c7a4599 Mon Sep 17 00:00:00 2001 From: Daniel Almeida Date: Fri, 29 Dec 2023 17:28:21 +0000 Subject: [PATCH] decoders: vaapi: av1: add film grain support Use the new VAAPI handle to add film grain support in the VAAPI AV1 decoder. One surface contains the grain and is meant to be displayed, while the other one does not. We need the decode surface to not have any grain, otherwise this would break the decoding process. The number of surfaces is doubled to accomodate for this, but only when film grain is active. --- src/decoder/stateless/av1.rs | 1 + src/decoder/stateless/av1/vaapi.rs | 99 +++++++++++++++++++++++++----- 2 files changed, 84 insertions(+), 16 deletions(-) diff --git a/src/decoder/stateless/av1.rs b/src/decoder/stateless/av1.rs index 5359f961..07c2323a 100644 --- a/src/decoder/stateless/av1.rs +++ b/src/decoder/stateless/av1.rs @@ -258,6 +258,7 @@ where header, backend_picture, }) => (self.backend.submit_picture(backend_picture)?, header), + Some(CurrentPicState::ShowExistingFrame { header, handle }) => (handle, header), None => return Err(anyhow!("Broken stream: no picture to submit")), }; diff --git a/src/decoder/stateless/av1/vaapi.rs b/src/decoder/stateless/av1/vaapi.rs index 6c7f412b..577ae82e 100644 --- a/src/decoder/stateless/av1/vaapi.rs +++ b/src/decoder/stateless/av1/vaapi.rs @@ -12,6 +12,7 @@ use libva::SurfaceMemoryDescriptor; use crate::backend::vaapi::DecodedHandle as VADecodedHandle; use crate::backend::vaapi::PoolCreationMode; +use crate::backend::vaapi::PooledSurface; use crate::backend::vaapi::VaStreamInfo; use crate::backend::vaapi::VaapiBackend; use crate::backend::vaapi::VaapiPicture; @@ -87,7 +88,13 @@ impl VaStreamInfo for &Rc { } fn min_num_surfaces(&self) -> usize { - NUM_SURFACES + if self.film_grain_params_present { + /* assume grain will be applied. We need twice the number of surfaces + * for that. */ + NUM_SURFACES * 2 + } else { + NUM_SURFACES + } } fn coded_size(&self) -> (u32, u32) { @@ -271,6 +278,7 @@ fn build_pic_param( hdr: &FrameHeaderObu, seq: &SequenceHeaderObu, current_frame: libva::VASurfaceID, + current_display_picture: libva::VASurfaceID, reference_frames: &[Option>; NUM_REF_FRAMES], ) -> anyhow::Result { let seq_info_fields = libva::AV1SeqFields::new( @@ -487,8 +495,8 @@ fn build_pic_param( .context("Invalid matrix_coefficients")?, &seq_info_fields, current_frame, - libva::constants::VA_INVALID_SURFACE, /* film grain is unsupported for now */ - vec![], /* anchor_frames_list */ + current_display_picture, + vec![], /* anchor_frames_list */ u16::try_from(hdr.upscaled_width - 1).context("Invalid frame width")?, u16::try_from(hdr.frame_height - 1).context("Invalid frame height")?, 0, /* output_frame_width_in_tiles_minus_1 */ @@ -564,8 +572,14 @@ fn build_slice_data_for_tg(tg: TileGroupObu) -> libva::BufferType { libva::BufferType::SliceData(Vec::from(obu.as_ref())) } +pub struct Picture { + va_picture: VaapiPicture, + /// Some if film grain is to be applied. + display_surface: Option>, +} + impl StatelessDecoderBackendPicture for VaapiBackend { - type Picture = VaapiPicture; + type Picture = Picture; } impl StatelessAV1DecoderBackend for VaapiBackend { @@ -598,7 +612,7 @@ impl StatelessAV1DecoderBackend for VaapiB reference_frames: &[Option; NUM_REF_FRAMES], highest_spatial_layer: Option, ) -> crate::decoder::stateless::StatelessBackendResult { - let surface = match highest_spatial_layer { + let (decode_surface, display_surface) = match highest_spatial_layer { Some(_) => { let layer = Resolution { width: hdr.upscaled_width, @@ -611,33 +625,77 @@ impl StatelessAV1DecoderBackend for VaapiB "No pool available for this layer" )))?; - pool.borrow_mut() + let decode_surface = pool + .borrow_mut() .get_surface(pool) - .ok_or(StatelessBackendError::OutOfResources)? + .ok_or(StatelessBackendError::OutOfResources)?; + + let display_surface = if hdr.film_grain_params.apply_grain { + Some( + pool.borrow_mut() + .get_surface(pool) + .ok_or(StatelessBackendError::OutOfResources)?, + ) + } else { + None + }; + + (decode_surface, display_surface) } None => { let highest_pool = self.highest_pool(); - highest_pool + + let decode_surface = highest_pool .borrow_mut() .get_surface(highest_pool) - .ok_or(StatelessBackendError::OutOfResources)? + .ok_or(StatelessBackendError::OutOfResources)?; + + let display_surface = if hdr.film_grain_params.apply_grain { + Some( + highest_pool + .borrow_mut() + .get_surface(highest_pool) + .ok_or(StatelessBackendError::OutOfResources)?, + ) + } else { + None + }; + + (decode_surface, display_surface) } }; let metadata = self.metadata_state.get_parsed()?; - let mut picture = VaPicture::new(timestamp, Rc::clone(&metadata.context), surface); + let mut picture = VaPicture::new(timestamp, Rc::clone(&metadata.context), decode_surface); let surface_id = picture.surface().id(); + let display_surface_id = match display_surface { + Some(ref pooled_surface) => { + use std::borrow::Borrow; + let display_surface: &libva::Surface = pooled_surface.borrow(); + display_surface.id() + } + None => libva::constants::VA_INVALID_SURFACE, + }; - let pic_param = build_pic_param(hdr, sequence, surface_id, reference_frames) - .context("Failed to build picture parameter")?; + let pic_param = build_pic_param( + hdr, + sequence, + surface_id, + display_surface_id, + reference_frames, + ) + .context("Failed to build picture parameter")?; let pic_param = metadata .context .create_buffer(pic_param) .context("Failed to create picture parameter buffer")?; picture.add_buffer(pic_param); - Ok(picture) + Ok(Picture { + va_picture: picture, + display_surface, + }) } fn decode_tile_group( @@ -655,13 +713,13 @@ impl StatelessAV1DecoderBackend for VaapiB .create_buffer(slice_params) .context("Failed to create slice parameter buffer")?; - picture.add_buffer(buffer); + picture.va_picture.add_buffer(buffer); let buffer = context .create_buffer(slice_data) .context("Failed to create slice data buffer")?; - picture.add_buffer(buffer); + picture.va_picture.add_buffer(buffer); Ok(()) } @@ -670,7 +728,16 @@ impl StatelessAV1DecoderBackend for VaapiB &mut self, picture: Self::Picture, ) -> crate::decoder::stateless::StatelessBackendResult { - self.process_picture::(picture) + let Picture { + va_picture, + display_surface, + } = picture; + + if let Some(display_surface) = display_surface { + self.process_av1_picture::(va_picture, display_surface) + } else { + self.process_picture::(va_picture) + } } }