From 36def38e72aea22c508c724771511152ff303069 Mon Sep 17 00:00:00 2001 From: Jakob Meier Date: Tue, 15 Oct 2024 17:01:57 +0200 Subject: [PATCH] Save poses in local collection We can now upload a video, stop it at a desired timestamp, extract a skeleton from the video frame, edit the name of the pose and adjust the weights for each limb, and finally, store it as a pose in a collection that's stored on the user's local storage. --- .../lib/components/editor/PoseEditForm.svelte | 5 + .../lib/components/editor/VideoToStep.svelte | 18 ++- .../src/lib/instructor/bouncy_instructor.d.ts | 131 +++++++++++------- .../instructor/bouncy_instructor_bg.wasm.d.ts | 7 + .../src/routes/LocalCollectionContext.svelte | 60 ++++++-- .../src/routes/editor/+page.svelte | 9 +- .../src/public/parsing/pose_file.rs | 9 +- bouncy_instructor/src/public/wrapper.rs | 1 + .../src/public/wrapper/pose_file_wrapper.rs | 96 +++++++++++++ 9 files changed, 267 insertions(+), 69 deletions(-) create mode 100644 bouncy_instructor/src/public/wrapper/pose_file_wrapper.rs diff --git a/bouncy_frontend/src/lib/components/editor/PoseEditForm.svelte b/bouncy_frontend/src/lib/components/editor/PoseEditForm.svelte index cf15e35..79b5938 100644 --- a/bouncy_frontend/src/lib/components/editor/PoseEditForm.svelte +++ b/bouncy_frontend/src/lib/components/editor/PoseEditForm.svelte @@ -32,6 +32,11 @@ onPoseUpdated(pose); } + /** @returns {PoseWrapper} newPose */ + export function getPose() { + return pose; + } + /** @param {PoseWrapper} newPose */ function onPoseUpdated(newPose) { skeleton = newPose.skeleton(); diff --git a/bouncy_frontend/src/lib/components/editor/VideoToStep.svelte b/bouncy_frontend/src/lib/components/editor/VideoToStep.svelte index fc8066a..3da5de9 100644 --- a/bouncy_frontend/src/lib/components/editor/VideoToStep.svelte +++ b/bouncy_frontend/src/lib/components/editor/VideoToStep.svelte @@ -10,8 +10,10 @@ import PoseEditForm from '$lib/components/editor/PoseEditForm.svelte'; import { fileToUrl, waitForVideoMetaLoaded } from '$lib/promise_util'; import { PoseDetection } from '$lib/pose'; + import Button from '../ui/Button.svelte'; const poseCtx = getContext('pose'); + const localCollectionCtx = getContext('localCollection'); let tracker = new Tracker(); registerTracker(tracker); @@ -23,6 +25,8 @@ let poseFromForm; /** @type {(skeleton: import("$lib/instructor/bouncy_instructor").PoseWrapper)=>void} */ let loadPose; + /** @type {()=>import("$lib/instructor/bouncy_instructor").PoseWrapper} */ + let getPose; /** @type {import("$lib/instructor/bouncy_instructor").SkeletonV2 | undefined} */ let liveSkeleton; @@ -99,6 +103,11 @@ loadPose(pose); } } + + function savePose() { + let pose = getPose(); + localCollectionCtx.addPose(pose); + }

@@ -138,4 +147,11 @@ - + + + diff --git a/bouncy_frontend/src/lib/instructor/bouncy_instructor.d.ts b/bouncy_frontend/src/lib/instructor/bouncy_instructor.d.ts index 94a7bd7..a9d9a15 100644 --- a/bouncy_frontend/src/lib/instructor/bouncy_instructor.d.ts +++ b/bouncy_frontend/src/lib/instructor/bouncy_instructor.d.ts @@ -75,6 +75,31 @@ export function dances(): (DanceWrapper)[]; export function danceBuilderFromDance(dance_id: string): DanceBuilder; /** */ +export enum DetectionState { +/** +* Neutral state, not detecting anything. +*/ + Init = 1, +/** +* Dance is positioning themselves, detecting the idle position. +*/ + Positioning = 2, +/** +* About to go over to live tracking, playing a countdown audio. +*/ + CountDown = 3, +/** +* Tracking current movements. +*/ + LiveTracking = 4, +/** +* No longer tracking but the results of the previous tracking are +* available. +*/ + TrackingDone = 5, +} +/** +*/ export enum DetectionFailureReason { /** * The last match was too recent to have another match. @@ -102,29 +127,13 @@ export enum DetectionFailureReason { NoNewData = 6, } /** +* Best guess for what the dancer needs to change to fit the pose. */ -export enum DetectionState { -/** -* Neutral state, not detecting anything. -*/ - Init = 1, -/** -* Dance is positioning themselves, detecting the idle position. -*/ - Positioning = 2, -/** -* About to go over to live tracking, playing a countdown audio. -*/ - CountDown = 3, -/** -* Tracking current movements. -*/ - LiveTracking = 4, -/** -* No longer tracking but the results of the previous tracking are -* available. -*/ - TrackingDone = 5, +export enum PoseHint { + DontKnow = 0, + LeftRight = 1, + ZOrder = 2, + WrongDirection = 3, } /** */ @@ -140,15 +149,6 @@ export enum SkeletonField { RightForearm = 8, RightFoot = 9, } -/** -* Best guess for what the dancer needs to change to fit the pose. -*/ -export enum PoseHint { - DontKnow = 0, - LeftRight = 1, - ZOrder = 2, - WrongDirection = 3, -} import type { Readable } from "svelte/store"; @@ -606,6 +606,35 @@ export class PoseApproximation { } /** */ +export class PoseFileWrapper { + free(): void; +/** +*/ + constructor(); +/** +* @param {string} text +* @returns {PoseFileWrapper} +*/ + static fromRon(text: string): PoseFileWrapper; +/** +* @returns {(PoseWrapper)[]} +*/ + poses(): (PoseWrapper)[]; +/** +* @param {PoseWrapper} new_pose +*/ + addPose(new_pose: PoseWrapper): void; +/** +* @param {string} id +*/ + removePose(id: string): void; +/** +* @returns {string} +*/ + buildRon(): string; +} +/** +*/ export class PoseWrapper { free(): void; /** @@ -712,6 +741,17 @@ export class Segment { export class Skeleton { free(): void; /** +* Compute 2d coordinates for the skeleton for rendering. +* +* The skeleton will be rendered assuming hard-coded values for body part +* proportional lengths, multiplied with the size parameter. The hip +* segment will have its center at the given position. +* @param {Cartesian2d} hip_center +* @param {number} size +* @returns {SkeletonV2} +*/ + render(hip_center: Cartesian2d, size: number): SkeletonV2; +/** * @param {boolean} sideway * @returns {Skeleton} */ @@ -725,17 +765,6 @@ export class Skeleton { */ debugString(): string; /** -* Compute 2d coordinates for the skeleton for rendering. -* -* The skeleton will be rendered assuming hard-coded values for body part -* proportional lengths, multiplied with the size parameter. The hip -* segment will have its center at the given position. -* @param {Cartesian2d} hip_center -* @param {number} size -* @returns {SkeletonV2} -*/ - render(hip_center: Cartesian2d, size: number): SkeletonV2; -/** * Does the dancer face away more than they face the camera? */ backwards: boolean; @@ -917,6 +946,15 @@ export class StepWrapper { export class Tracker { free(): void; /** +* @param {number} timestamp +* @returns {ExportedFrame} +*/ + exportFrame(timestamp: number): ExportedFrame; +/** +* @returns {string} +*/ + exportKeypoints(): string; +/** * Create a tracker for all known steps. */ constructor(); @@ -1075,15 +1113,6 @@ export class Tracker { */ renderedKeypointsAt(timestamp: number, width: number, height: number): SkeletonV2 | undefined; /** -* @param {number} timestamp -* @returns {ExportedFrame} -*/ - exportFrame(timestamp: number): ExportedFrame; -/** -* @returns {string} -*/ - exportKeypoints(): string; -/** */ readonly detectionState: ReadableDetectionState; /** diff --git a/bouncy_frontend/src/lib/instructor/bouncy_instructor_bg.wasm.d.ts b/bouncy_frontend/src/lib/instructor/bouncy_instructor_bg.wasm.d.ts index 94bf073..2c2b0ca 100644 --- a/bouncy_frontend/src/lib/instructor/bouncy_instructor_bg.wasm.d.ts +++ b/bouncy_frontend/src/lib/instructor/bouncy_instructor_bg.wasm.d.ts @@ -74,6 +74,12 @@ export function poseapproximation_name(a: number, b: number): void; export function poseapproximation_worstLimbs(a: number, b: number, c: number): void; export function poseapproximation_zErrors(a: number, b: number): void; export function poseapproximation_zOrderErrors(a: number, b: number): void; +export function posefilewrapper_addPose(a: number, b: number, c: number): void; +export function posefilewrapper_buildRon(a: number, b: number): void; +export function posefilewrapper_fromRon(a: number, b: number, c: number): void; +export function posefilewrapper_new_empty(): number; +export function posefilewrapper_poses(a: number, b: number): void; +export function posefilewrapper_removePose(a: number, b: number, c: number, d: number): void; export function poses(a: number): void; export function posewrapper_getAngle(a: number, b: number): number; export function posewrapper_getWeight(a: number, b: number): number; @@ -215,6 +221,7 @@ export function __wbg_lesson_free(a: number, b: number): void; export function __wbg_lessonpart_free(a: number, b: number): void; export function __wbg_limberror_free(a: number, b: number): void; export function __wbg_poseapproximation_free(a: number, b: number): void; +export function __wbg_posefilewrapper_free(a: number, b: number): void; export function __wbg_posewrapper_free(a: number, b: number): void; export function __wbg_renderablesegment_free(a: number, b: number): void; export function __wbg_segment_free(a: number, b: number): void; diff --git a/bouncy_frontend/src/routes/LocalCollectionContext.svelte b/bouncy_frontend/src/routes/LocalCollectionContext.svelte index d03a01c..6c4315c 100644 --- a/bouncy_frontend/src/routes/LocalCollectionContext.svelte +++ b/bouncy_frontend/src/routes/LocalCollectionContext.svelte @@ -6,29 +6,45 @@ import { DanceFileBuilder, DanceBuilder, + PoseFileWrapper, + PoseWrapper, } from '$lib/instructor/bouncy_instructor'; import { setContext } from 'svelte'; import { derived, writable } from 'svelte/store'; - const ron = browser ? localStorage.dances : null; - const fileBuilder = ron - ? DanceFileBuilder.fromRon(ron) + const danceRon = browser ? localStorage.dances : null; + const danceFileBuilder = danceRon + ? DanceFileBuilder.fromRon(danceRon) : new DanceFileBuilder(); + const danceBuilderStore = writable(danceFileBuilder); + + const poseRon = browser ? localStorage.poses : null; + const poseFileBuilder = poseRon + ? PoseFileWrapper.fromRon(poseRon) + : new PoseFileWrapper(); + const poseBuilderStore = writable(poseFileBuilder); - const builderStore = writable(fileBuilder); const ctx = { - builder: builderStore, - dances: derived(builderStore, ($b) => $b.dances()), + danceBuilder: danceBuilderStore, + dances: derived(danceBuilderStore, ($b) => $b.dances()), + poseBuilder: poseBuilderStore, + poses: derived(poseBuilderStore, ($b) => $b.poses()), addDanceBuilder, overwriteDanceBuilder, removeDance, + addPose, + removePose, }; if (browser) { - ctx.builder.subscribe( + ctx.danceBuilder.subscribe( (/** @type {DanceFileBuilder} */ builder) => (localStorage.dances = builder.buildRon()) ); + ctx.poseBuilder.subscribe( + (/** @type {PoseFileWrapper} */ builder) => + (localStorage.poses = builder.buildRon()) + ); } setContext('localCollection', ctx); @@ -37,27 +53,45 @@ * @param {DanceBuilder} danceBuilder */ function addDanceBuilder(danceBuilder) { - $builderStore.addDance(danceBuilder); + $danceBuilderStore.addDance(danceBuilder); // trigger update (can I do better?) - $builderStore = $builderStore; + $danceBuilderStore = $danceBuilderStore; } /** * @param {DanceBuilder} danceBuilder */ function overwriteDanceBuilder(danceBuilder) { - $builderStore.overwriteDance(danceBuilder); + $danceBuilderStore.overwriteDance(danceBuilder); // trigger update (can I do better?) - $builderStore = $builderStore; + $danceBuilderStore = $danceBuilderStore; } /** * @param {String} id */ function removeDance(id) { - $builderStore.removeDance(id); + $danceBuilderStore.removeDance(id); + // trigger update (can I do better?) + $danceBuilderStore = $danceBuilderStore; + } + + /** + * @param {PoseWrapper} pose + */ + function addPose(pose) { + $poseBuilderStore.addPose(pose); + // trigger update (can I do better?) + $poseBuilderStore = $poseBuilderStore; + } + + /** + * @param {String} id + */ + function removePose(id) { + $poseBuilderStore.removePose(id); // trigger update (can I do better?) - $builderStore = $builderStore; + $poseBuilderStore = $poseBuilderStore; } diff --git a/bouncy_frontend/src/routes/editor/+page.svelte b/bouncy_frontend/src/routes/editor/+page.svelte index 0ab804d..6e5bd61 100644 --- a/bouncy_frontend/src/routes/editor/+page.svelte +++ b/bouncy_frontend/src/routes/editor/+page.svelte @@ -3,14 +3,17 @@ import SvgAvatar from '$lib/components/avatar/SvgAvatar.svelte'; import VideoToStep from '$lib/components/editor/VideoToStep.svelte'; import { LEFT_RIGHT_COLORING_LIGHT } from '$lib/constants'; + import { getContext } from 'svelte'; /** @type {import('./$types').PageData} */ export let data; - const poses = data.lookupPoses(); + // const poses = data.lookupPoses(); + const localCollectionCtx = getContext('localCollection'); + const poses = localCollectionCtx.poses;

- {#each poses as pose} + {#each $poses as pose}

{pose.name('en')}

@@ -37,4 +40,4 @@ .pose { max-width: 200px; } - \ No newline at end of file + diff --git a/bouncy_instructor/src/public/parsing/pose_file.rs b/bouncy_instructor/src/public/parsing/pose_file.rs index fe3ecc2..93f1bfb 100644 --- a/bouncy_instructor/src/public/parsing/pose_file.rs +++ b/bouncy_instructor/src/public/parsing/pose_file.rs @@ -11,7 +11,7 @@ use super::{ParseFileError, VersionCheck}; const CURRENT_VERSION: u16 = 1; /// Format for pose definition files. -#[derive(Deserialize)] +#[derive(Debug, Deserialize, Serialize)] pub(crate) struct PoseFile { #[allow(dead_code)] pub version: u16, @@ -159,6 +159,13 @@ pub enum PoseDirection { } impl PoseFile { + pub(crate) fn new() -> Self { + Self { + version: CURRENT_VERSION, + poses: vec![], + } + } + pub(crate) fn from_str(text: &str) -> Result { let check: VersionCheck = ron::from_str(text)?; diff --git a/bouncy_instructor/src/public/wrapper.rs b/bouncy_instructor/src/public/wrapper.rs index 2325af4..b1908f1 100644 --- a/bouncy_instructor/src/public/wrapper.rs +++ b/bouncy_instructor/src/public/wrapper.rs @@ -6,6 +6,7 @@ //! architecture makes sense. pub(crate) mod dance_wrapper; +pub(crate) mod pose_file_wrapper; pub(crate) mod pose_wrapper; pub(crate) mod skeleton_wrapper; pub(crate) mod step_wrapper; diff --git a/bouncy_instructor/src/public/wrapper/pose_file_wrapper.rs b/bouncy_instructor/src/public/wrapper/pose_file_wrapper.rs new file mode 100644 index 0000000..6da1001 --- /dev/null +++ b/bouncy_instructor/src/public/wrapper/pose_file_wrapper.rs @@ -0,0 +1,96 @@ +use std::cell::RefCell; +use std::rc::Rc; +use wasm_bindgen::prelude::wasm_bindgen; +use wasm_bindgen::JsValue; + +use crate::editor::ExportError; +use crate::pose_file::PoseFile; +use crate::wrapper::pose_wrapper::PoseWrapper; + +#[derive(Debug, Clone)] +#[wasm_bindgen] +pub struct PoseFileWrapper { + pose_file: Rc>, + // cache must always be kept in sync + poses_cache: Rc>>, +} + +#[wasm_bindgen] +impl PoseFileWrapper { + #[wasm_bindgen(constructor)] + pub fn new_empty() -> Self { + Self::new(PoseFile::new()) + } + + #[wasm_bindgen(js_name = "fromRon")] + pub fn from_ron(text: &str) -> Result { + let file = PoseFile::from_str(text)?; + + Ok(Self::new(file)) + } + + pub fn poses(&self) -> Vec { + self.poses_cache.as_ref().borrow().clone() + } + + #[wasm_bindgen(js_name = "addPose")] + pub fn add_pose(&mut self, new_pose: PoseWrapper) -> Result<(), String> { + let file = self.pose_file.as_ref().borrow(); + if file + .poses + .iter() + .any(|pose| new_pose.definition().id == pose.id) + { + return Err("Pose ID already exists".to_owned()); + } + drop(file); + self.push_pose_internal(new_pose); + Ok(()) + } + + // #[wasm_bindgen(js_name = "overwritePose")] + // pub fn overwrite_pose(&mut self, pose: &PoseWrapper) -> Result<(), String> { + // } + + #[wasm_bindgen(js_name = "removePose")] + pub fn remove_pose(&mut self, id: String) -> Result<(), String> { + let file = self.pose_file.as_ref().borrow(); + if let Some(index) = file.poses.iter().position(|pose| pose.id == id) { + drop(file); + self.remove_pose_internal(index); + Ok(()) + } else { + Err("Pose ID does not exists".to_owned()) + } + } + + #[wasm_bindgen(js_name = "buildRon")] + pub fn build_ron(&self) -> Result { + let file_data = self.pose_file.as_ref().borrow(); + let string = ron::ser::to_string(&*file_data)?; + Ok(string) + } +} + +impl PoseFileWrapper { + fn new(file: PoseFile) -> Self { + let poses = file.poses.iter().cloned().map(PoseWrapper::new).collect(); + Self { + pose_file: Rc::new(RefCell::new(file)), + poses_cache: Rc::new(RefCell::new(poses)), + } + } + + fn push_pose_internal(&self, pose: PoseWrapper) { + self.pose_file + .borrow_mut() + .poses + .push(pose.definition().clone()); + self.poses_cache.borrow_mut().push(pose); + } + + fn remove_pose_internal(&self, index: usize) { + self.poses_cache.borrow_mut().remove(index); + self.pose_file.borrow_mut().poses.remove(index); + } +}