feat(model): add skinning and bone animations

package.json

@@ -36,7 +36,7 @@
   ],
   "dependencies": {
     "@tweenjs/tween.js": "^23.1.1",
-    "@wowserhq/format": "^0.21.0"
+    "@wowserhq/format": "^0.22.0"
   },
   "peerDependencies": {
     "three": "^0.160.0"

src/lib/model/Model.ts

@@ -16,16 +16,29 @@ class Model extends THREE.Object3D {
     geometry: THREE.BufferGeometry,
     materials: THREE.Material[],
     animator: ModelAnimator,
+    skinned: boolean,
   ) {
     super();
 
-    this.#mesh = new THREE.Mesh(geometry, materials);
+    // Avoid skinning overhead when model does not make use of bone animations
+    if (skinned) {
+      this.#mesh = new THREE.SkinnedMesh(geometry, materials);
+    } else {
+      this.#mesh = new THREE.Mesh(geometry, materials);
+    }
+
     this.#mesh.onBeforeRender = this.#onBeforeRender.bind(this);
     this.add(this.#mesh);
 
     // Every model instance gets a unique animation state managed by a single animator
     this.animation = animator.createAnimation(this);
 
+    // Every skinned model instance gets a unique skeleton
+    if (skinned) {
+      this.#mesh.add(...this.animation.rootBones);
+      (this.#mesh as THREE.SkinnedMesh).bind(this.animation.skeleton);
+    }
+
     this.diffuseColor = new THREE.Color(1.0, 1.0, 1.0);
     this.emissiveColor = new THREE.Color(0.0, 0.0, 0.0);
     this.alpha = 1.0;
@@ -39,6 +52,12 @@ class Model extends THREE.Object3D {
     material: ModelMaterial,
     group: THREE.Group,
   ) {
+    // Ensure bone matrices are updated (matrix world auto-updates are disabled)
+    if ((this.#mesh as THREE.SkinnedMesh).isSkinnedMesh) {
+      this.#mesh.updateMatrixWorld();
+    }
+
+    // Update material uniforms to match animation states
     material.prepareMaterial(this);
   }
 

src/lib/model/ModelAnimation.ts

@@ -2,23 +2,34 @@ import * as THREE from 'three';
 import { ModelMaterialColor, ModelTextureTransform } from './types.js';
 import Model from './Model.js';
 import ModelAnimator from './ModelAnimator.js';
+import { BoneSpec } from './loader/types.js';
 
 class ModelAnimation extends THREE.Object3D {
   // States
   textureWeights: number[] = [];
   textureTransforms: ModelTextureTransform[] = [];
   materialColors: ModelMaterialColor[] = [];
 
+  // Skeleton
+  skeleton: THREE.Skeleton;
+  rootBones: THREE.Bone[];
+
   #model: Model;
   #animator: ModelAnimator;
   #actions: Set<THREE.AnimationAction> = new Set();
 
-  constructor(model: Model, animator: ModelAnimator, stateCounts: Record<string, number>) {
+  constructor(
+    model: Model,
+    animator: ModelAnimator,
+    bones: BoneSpec[],
+    stateCounts: Record<string, number>,
+  ) {
     super();
 
     this.#model = model;
     this.#animator = animator;
     this.#createStates(stateCounts);
+    this.#createSkeleton(bones);
 
     this.#autoplay();
   }
@@ -29,6 +40,8 @@ class ModelAnimation extends THREE.Object3D {
     }
 
     this.#actions.clear();
+
+    this.skeleton.dispose();
   }
 
   #createStates(stateCounts: Record<string, number>) {
@@ -52,19 +65,42 @@ class ModelAnimation extends THREE.Object3D {
     }
   }
 
+  #createSkeleton(boneSpecs: BoneSpec[]) {
+    const bones: THREE.Bone[] = [];
+    const rootBones: THREE.Bone[] = [];
+
+    for (const boneSpec of boneSpecs) {
+      const bone = new THREE.Bone();
+      bone.visible = false;
+      bone.position.set(boneSpec.position[0], boneSpec.position[1], boneSpec.position[2]);
+      bones.push(bone);
+
+      if (boneSpec.parentIndex === -1) {
+        rootBones.push(bone);
+      } else {
+        bones[boneSpec.parentIndex].add(bone);
+      }
+    }
+
+    this.skeleton = new THREE.Skeleton(bones);
+    this.rootBones = rootBones;
+  }
+
   #autoplay() {
     // Automatically play all loops
     for (let i = 0; i < this.#animator.loops.length; i++) {
       const action = this.#animator.getLoop(this, i).play();
       this.#actions.add(action);
     }
 
-    // Automatically play flagged sequences
-    for (let i = 0; i < this.#animator.sequences.length; i++) {
-      const sequence = this.#animator.sequences[i];
+    // Automatically play sequence id 0
+    if (this.#animator.sequences.has(0)) {
+      const variations = this.#animator.sequences.get(0);
+      const sequence = variations[0];
 
       if (sequence.flags & 0x20) {
-        const action = this.#animator.getSequence(this, i).play();
+        const action = this.#animator.getSequence(this, sequence.id, sequence.variationIndex);
+        action.play();
         this.#actions.add(action);
       }
     }

src/lib/model/ModelAnimator.ts

@@ -1,6 +1,6 @@
 import * as THREE from 'three';
 import { M2Track } from '@wowserhq/format';
-import { SequenceSpec } from './loader/types.js';
+import { BoneSpec, SequenceSpec } from './loader/types.js';
 import ModelAnimation from './ModelAnimation.js';
 import Model from './Model.js';
 
@@ -20,12 +20,15 @@ class ModelAnimator {
   #loops: number[] = [];
   #loopClips: THREE.AnimationClip[] = [];
 
-  #sequences: SequenceSpec[] = [];
-  #sequenceClips: THREE.AnimationClip[] = [];
+  #sequencesByIndex: SequenceSpec[] = [];
+  #sequences: Map<number, SequenceSpec[]> = new Map();
+  #sequenceClips: Map<number, THREE.AnimationClip[]> = new Map();
+
+  #bones: BoneSpec[] = [];
 
   #stateCounts: Record<string, number> = {};
 
-  constructor(loops: Uint32Array, sequences: SequenceSpec[]) {
+  constructor(loops: Uint32Array, sequences: SequenceSpec[], bones: BoneSpec[]) {
     this.#mixer = new THREE.AnimationMixer(new THREE.Object3D());
     this.#mixer.timeScale = 1000;
 
@@ -36,10 +39,12 @@ class ModelAnimator {
     for (const sequence of sequences) {
       this.#registerSequence(sequence);
     }
+
+    this.#bones = bones;
   }
 
   createAnimation(model: Model) {
-    return new ModelAnimation(model, this, this.#stateCounts);
+    return new ModelAnimation(model, this, this.#bones, this.#stateCounts);
   }
 
   get loops() {
@@ -64,8 +69,8 @@ class ModelAnimator {
     return this.#mixer.clipAction(clip, root);
   }
 
-  getSequence(root: THREE.Object3D, index: number) {
-    const clip = this.#sequenceClips[index];
+  getSequence(root: THREE.Object3D, id: number, variationIndex: number) {
+    const clip = this.#sequenceClips.get(id)[variationIndex];
     return this.#mixer.clipAction(clip, root);
   }
 
@@ -138,7 +143,8 @@ class ModelAnimator {
     transform?: (value: any) => any,
   ) {
     for (let s = 0; s < track.sequenceTimes.length; s++) {
-      const clip = this.#sequenceClips[s];
+      const sequence = this.#sequencesByIndex[s];
+      const clip = this.#sequenceClips.get(sequence.id)[sequence.variationIndex];
 
       const times = track.sequenceTimes[s];
       const values = transform
@@ -161,9 +167,17 @@ class ModelAnimator {
   }
 
   #registerSequence(spec: SequenceSpec) {
-    const index = this.#sequences.length;
-    this.#sequences[index] = spec;
-    this.#sequenceClips[index] = new THREE.AnimationClip(`sequence-${index}`, spec.duration, []);
+    if (!this.#sequences.has(spec.id)) {
+      this.#sequences.set(spec.id, []);
+      this.#sequenceClips.set(spec.id, []);
+    }
+
+    this.#sequences.get(spec.id)[spec.variationIndex] = spec;
+    this.#sequencesByIndex.push(spec);
+
+    const clipName = `sequence-${spec.id}-${spec.variationIndex}`;
+    const clip = new THREE.AnimationClip(clipName, spec.duration, []);
+    this.#sequenceClips.get(spec.id)[spec.variationIndex] = clip;
   }
 }
 

src/lib/model/ModelManager.ts

@@ -16,6 +16,7 @@ type ModelResources = {
   geometry: THREE.BufferGeometry;
   materials: THREE.Material[];
   animator: ModelAnimator;
+  skinned: boolean;
 };
 
 type ModelManagerOptions = {
@@ -86,6 +87,7 @@ class ModelManager {
       geometry,
       materials,
       animator,
+      skinned: spec.skinned,
     };
 
     this.#loaded.set(refId, resources);
@@ -105,13 +107,13 @@ class ModelManager {
 
     const boneWeights = new THREE.InterleavedBuffer(new Uint8Array(vertexBuffer), 48);
     geometry.setAttribute(
-      'boneWeights',
-      new THREE.InterleavedBufferAttribute(boneWeights, 4, 12, false),
+      'skinWeight',
+      new THREE.InterleavedBufferAttribute(boneWeights, 4, 12, true),
     );
 
     const boneIndices = new THREE.InterleavedBuffer(new Uint8Array(vertexBuffer), 48);
     geometry.setAttribute(
-      'boneIndices',
+      'skinIndex',
       new THREE.InterleavedBufferAttribute(boneIndices, 4, 16, false),
     );
 
@@ -152,10 +154,12 @@ class ModelManager {
   }
 
   #createMaterials(spec: ModelSpec) {
-    return Promise.all(spec.materials.map((materialSpec) => this.#createMaterial(materialSpec)));
+    return Promise.all(
+      spec.materials.map((materialSpec) => this.#createMaterial(materialSpec, spec.skinned)),
+    );
   }
 
-  async #createMaterial(spec: MaterialSpec) {
+  async #createMaterial(spec: MaterialSpec, skinned: boolean) {
     const vertexShader = getVertexShader(spec.vertexShader);
     const fragmentShader = getFragmentShader(spec.fragmentShader);
     const textures = await Promise.all(
@@ -173,6 +177,7 @@ class ModelManager {
       textureWeightIndex,
       textureTransformIndices,
       materialColorIndex,
+      skinned,
       uniforms,
       spec.blend,
       spec.flags,
@@ -195,7 +200,12 @@ class ModelManager {
   }
 
   #createModel(resources: ModelResources) {
-    const model = new Model(resources.geometry, resources.materials, resources.animator);
+    const model = new Model(
+      resources.geometry,
+      resources.materials,
+      resources.animator,
+      resources.skinned,
+    );
 
     model.name = resources.name;
 
@@ -207,7 +217,7 @@ class ModelManager {
       return null;
     }
 
-    const animator = new ModelAnimator(spec.loops, spec.sequences);
+    const animator = new ModelAnimator(spec.loops, spec.sequences, spec.bones);
 
     for (const [index, textureWeight] of spec.textureWeights.entries()) {
       animator.registerTrack(
@@ -253,6 +263,27 @@ class ModelManager {
       );
     }
 
+    for (const [index, bone] of spec.bones.entries()) {
+      animator.registerTrack(
+        { state: 'bones', index, property: 'position' },
+        bone.positionTrack,
+        THREE.VectorKeyframeTrack,
+      );
+
+      animator.registerTrack(
+        { state: 'bones', index, property: 'quaternion' },
+        bone.rotationTrack,
+        THREE.QuaternionKeyframeTrack,
+        (value: number) => (value > 0 ? value - 0x7fff : value + 0x7fff) / 0x7fff,
+      );
+
+      animator.registerTrack(
+        { state: 'bones', index, property: 'scale' },
+        bone.scaleTrack,
+        THREE.VectorKeyframeTrack,
+      );
+    }
+
     return animator;
   }
 }

src/lib/model/ModelMaterial.ts

@@ -32,6 +32,7 @@ class ModelMaterial extends THREE.RawShaderMaterial {
     textureWeightIndex: number,
     textureTransformIndices: number[],
     colorIndex: number,
+    skinned: boolean = false,
     uniforms: Record<string, THREE.IUniform> = {},
     blend = DEFAULT_BLEND,
     flags = DEFAULT_FLAGS,
@@ -65,6 +66,12 @@ class ModelMaterial extends THREE.RawShaderMaterial {
     this.fogged = flags & M2_MATERIAL_FLAG.FLAG_DISABLE_FOG ? 0.0 : 1.0;
     this.alpha = DEFAULT_ALPHA;
 
+    if (skinned) {
+      this.defines = {
+        USE_SKINNING: 1,
+      };
+    }
+
     this.glslVersion = THREE.GLSL3;
     this.vertexShader = vertexShader;
     this.fragmentShader = fragmentShader;