From a79e6bee21267cda339e307cbc43342e749cd293 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 14 May 2024 10:12:01 +0200 Subject: [PATCH 01/41] refactor: move histogram shader code to cdf.ts --- src/sliceview/volume/renderlayer.ts | 43 +++++++-------------------- src/webgl/empirical_cdf.ts | 46 +++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 33 deletions(-) diff --git a/src/sliceview/volume/renderlayer.ts b/src/sliceview/volume/renderlayer.ts index 2a8fa2de0..aeb5d3b49 100644 --- a/src/sliceview/volume/renderlayer.ts +++ b/src/sliceview/volume/renderlayer.ts @@ -59,11 +59,11 @@ import { makeWatchableShaderError, parameterizedContextDependentShaderGetter, } from "#src/webgl/dynamic_shader.js"; -import type { HistogramChannelSpecification } from "#src/webgl/empirical_cdf.js"; import { - defineInvlerpShaderFunction, - enableLerpShaderFunction, -} from "#src/webgl/lerp.js"; + defineShaderCodeForHistograms, + type HistogramChannelSpecification, +} from "#src/webgl/empirical_cdf.js"; +import { enableLerpShaderFunction } from "#src/webgl/lerp.js"; import { defineLineShader, drawLines, @@ -416,35 +416,12 @@ void emit(vec4 color) { ); const numHistograms = dataHistogramChannelSpecifications.length; if (dataHistogramsEnabled && numHistograms > 0) { - let histogramCollectionCode = ""; - const { dataType } = chunkFormat; - for (let i = 0; i < numHistograms; ++i) { - const { channel } = dataHistogramChannelSpecifications[i]; - const outputName = `out_histogram${i}`; - builder.addOutputBuffer("vec4", outputName, 1 + i); - const getDataValueExpr = `getDataValue(${channel.join(",")})`; - const invlerpName = `invlerpForHistogram${i}`; - builder.addFragmentCode( - defineInvlerpShaderFunction( - builder, - invlerpName, - dataType, - /*clamp=*/ false, - ), - ); - builder.addFragmentCode(` -float getHistogramValue${i}() { - return invlerpForHistogram${i}(${getDataValueExpr}); -} -`); - histogramCollectionCode += `{ -float x = getHistogramValue${i}(); -if (x < 0.0) x = 0.0; -else if (x > 1.0) x = 1.0; -else x = (1.0 + x * 253.0) / 255.0; -${outputName} = vec4(x, x, x, 1.0); -}`; - } + const histogramCollectionCode = defineShaderCodeForHistograms( + builder, + numHistograms, + chunkFormat, + dataHistogramChannelSpecifications, + ); builder.addFragmentCode(`void userMain(); void main() { ${histogramCollectionCode} diff --git a/src/webgl/empirical_cdf.ts b/src/webgl/empirical_cdf.ts index 799fe9812..a6c0d07a8 100644 --- a/src/webgl/empirical_cdf.ts +++ b/src/webgl/empirical_cdf.ts @@ -25,11 +25,13 @@ */ import type { WatchableValueInterface } from "#src/trackable_value.js"; +import type { DataType } from "#src/util/data_type.js"; import { RefCounted } from "#src/util/disposable.js"; import type { DataTypeInterval } from "#src/util/lerp.js"; import { VisibilityPriorityAggregator } from "#src/visibility_priority/frontend.js"; import { getMemoizedBuffer } from "#src/webgl/buffer.js"; import type { GL } from "#src/webgl/context.js"; +import { defineInvlerpShaderFunction } from "#src/webgl/lerp.js"; import type { TextureBuffer } from "#src/webgl/offscreen.js"; import { FramebufferConfiguration, @@ -41,6 +43,10 @@ import { setRawTextureParameters } from "#src/webgl/texture.js"; const DEBUG_HISTOGRAMS = false; +interface ChunkFormatInterface { + dataType: DataType; +} + export interface HistogramChannelSpecification { // Channel coordinates. channel: Uint32Array; @@ -229,3 +235,43 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); gl.disable(WebGL2RenderingContext.BLEND); } } + +export function defineShaderCodeForHistograms( + builder: ShaderBuilder, + numHistograms: number, + chunkFormat: ChunkFormatInterface, + dataHistogramChannelSpecifications: HistogramChannelSpecification[], + start: number = 1, + nameAppend: string = "", +) { + let histogramCollectionCode = ""; + const { dataType } = chunkFormat; + for (let i = 0; i < numHistograms; ++i) { + const { channel } = dataHistogramChannelSpecifications[i]; + const outputName = `out_histogram${i}${nameAppend}`; + builder.addOutputBuffer("vec4", outputName, start + i); + const getDataValueExpr = `getDataValue(${channel.join(",")})`; + const invlerpName = `invlerpForHistogram${i}${nameAppend}`; + builder.addFragmentCode( + defineInvlerpShaderFunction( + builder, + invlerpName, + dataType, + /*clamp=*/ false, + ), + ); + builder.addFragmentCode(` +float getHistogramValue${i}() { + return invlerpForHistogram${i}(${getDataValueExpr}); +} + `); + histogramCollectionCode += `{ +float x = getHistogramValue${i}(); +if (x < 0.0) x = 0.0; +else if (x > 1.0) x = 1.0; +else x = (1.0 + x * 253.0) / 255.0; +${outputName} = vec4(x, x, x, 1.0); + }`; + } + return histogramCollectionCode; +} From c326b7bc98a678b174fc38538b31f5ce0be5b8a5 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 14 May 2024 11:14:42 +0200 Subject: [PATCH 02/41] Revert "refactor: move histogram shader code to cdf.ts" This reverts commit a79e6bee21267cda339e307cbc43342e749cd293. --- src/sliceview/volume/renderlayer.ts | 43 ++++++++++++++++++++------- src/webgl/empirical_cdf.ts | 46 ----------------------------- 2 files changed, 33 insertions(+), 56 deletions(-) diff --git a/src/sliceview/volume/renderlayer.ts b/src/sliceview/volume/renderlayer.ts index aeb5d3b49..2a8fa2de0 100644 --- a/src/sliceview/volume/renderlayer.ts +++ b/src/sliceview/volume/renderlayer.ts @@ -59,11 +59,11 @@ import { makeWatchableShaderError, parameterizedContextDependentShaderGetter, } from "#src/webgl/dynamic_shader.js"; +import type { HistogramChannelSpecification } from "#src/webgl/empirical_cdf.js"; import { - defineShaderCodeForHistograms, - type HistogramChannelSpecification, -} from "#src/webgl/empirical_cdf.js"; -import { enableLerpShaderFunction } from "#src/webgl/lerp.js"; + defineInvlerpShaderFunction, + enableLerpShaderFunction, +} from "#src/webgl/lerp.js"; import { defineLineShader, drawLines, @@ -416,12 +416,35 @@ void emit(vec4 color) { ); const numHistograms = dataHistogramChannelSpecifications.length; if (dataHistogramsEnabled && numHistograms > 0) { - const histogramCollectionCode = defineShaderCodeForHistograms( - builder, - numHistograms, - chunkFormat, - dataHistogramChannelSpecifications, - ); + let histogramCollectionCode = ""; + const { dataType } = chunkFormat; + for (let i = 0; i < numHistograms; ++i) { + const { channel } = dataHistogramChannelSpecifications[i]; + const outputName = `out_histogram${i}`; + builder.addOutputBuffer("vec4", outputName, 1 + i); + const getDataValueExpr = `getDataValue(${channel.join(",")})`; + const invlerpName = `invlerpForHistogram${i}`; + builder.addFragmentCode( + defineInvlerpShaderFunction( + builder, + invlerpName, + dataType, + /*clamp=*/ false, + ), + ); + builder.addFragmentCode(` +float getHistogramValue${i}() { + return invlerpForHistogram${i}(${getDataValueExpr}); +} +`); + histogramCollectionCode += `{ +float x = getHistogramValue${i}(); +if (x < 0.0) x = 0.0; +else if (x > 1.0) x = 1.0; +else x = (1.0 + x * 253.0) / 255.0; +${outputName} = vec4(x, x, x, 1.0); +}`; + } builder.addFragmentCode(`void userMain(); void main() { ${histogramCollectionCode} diff --git a/src/webgl/empirical_cdf.ts b/src/webgl/empirical_cdf.ts index a6c0d07a8..799fe9812 100644 --- a/src/webgl/empirical_cdf.ts +++ b/src/webgl/empirical_cdf.ts @@ -25,13 +25,11 @@ */ import type { WatchableValueInterface } from "#src/trackable_value.js"; -import type { DataType } from "#src/util/data_type.js"; import { RefCounted } from "#src/util/disposable.js"; import type { DataTypeInterval } from "#src/util/lerp.js"; import { VisibilityPriorityAggregator } from "#src/visibility_priority/frontend.js"; import { getMemoizedBuffer } from "#src/webgl/buffer.js"; import type { GL } from "#src/webgl/context.js"; -import { defineInvlerpShaderFunction } from "#src/webgl/lerp.js"; import type { TextureBuffer } from "#src/webgl/offscreen.js"; import { FramebufferConfiguration, @@ -43,10 +41,6 @@ import { setRawTextureParameters } from "#src/webgl/texture.js"; const DEBUG_HISTOGRAMS = false; -interface ChunkFormatInterface { - dataType: DataType; -} - export interface HistogramChannelSpecification { // Channel coordinates. channel: Uint32Array; @@ -235,43 +229,3 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); gl.disable(WebGL2RenderingContext.BLEND); } } - -export function defineShaderCodeForHistograms( - builder: ShaderBuilder, - numHistograms: number, - chunkFormat: ChunkFormatInterface, - dataHistogramChannelSpecifications: HistogramChannelSpecification[], - start: number = 1, - nameAppend: string = "", -) { - let histogramCollectionCode = ""; - const { dataType } = chunkFormat; - for (let i = 0; i < numHistograms; ++i) { - const { channel } = dataHistogramChannelSpecifications[i]; - const outputName = `out_histogram${i}${nameAppend}`; - builder.addOutputBuffer("vec4", outputName, start + i); - const getDataValueExpr = `getDataValue(${channel.join(",")})`; - const invlerpName = `invlerpForHistogram${i}${nameAppend}`; - builder.addFragmentCode( - defineInvlerpShaderFunction( - builder, - invlerpName, - dataType, - /*clamp=*/ false, - ), - ); - builder.addFragmentCode(` -float getHistogramValue${i}() { - return invlerpForHistogram${i}(${getDataValueExpr}); -} - `); - histogramCollectionCode += `{ -float x = getHistogramValue${i}(); -if (x < 0.0) x = 0.0; -else if (x > 1.0) x = 1.0; -else x = (1.0 + x * 253.0) / 255.0; -${outputName} = vec4(x, x, x, 1.0); - }`; - } - return histogramCollectionCode; -} From 1b1b73137c46dd9b4da044af680c31f85fddaf40 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 14 May 2024 14:56:31 +0200 Subject: [PATCH 03/41] feat: grab global histogram specs from VR layer --- src/volume_rendering/volume_render_layer.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 49d79b1b5..897851e6a 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -82,6 +82,7 @@ import { parameterizedContextDependentShaderGetter, shaderCodeWithLineDirective, } from "#src/webgl/dynamic_shader.js"; +import type { HistogramSpecifications } from "#src/webgl/empirical_cdf.js"; import type { ShaderModule, ShaderProgram } from "#src/webgl/shader.js"; import type { ShaderControlsBuilderState, @@ -171,6 +172,7 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { mode: TrackableVolumeRenderingModeValue; backend: ChunkRenderLayerFrontend; private vertexIdHelper: VertexIdHelper; + dataHistogramSpecifications: HistogramSpecifications; private shaderGetter: ParameterizedContextDependentShaderGetter< { emitter: ShaderModule; chunkFormat: ChunkFormat; wireFrame: boolean }, @@ -201,6 +203,8 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { this.depthSamplesTarget = options.depthSamplesTarget; this.chunkResolutionHistogram = options.chunkResolutionHistogram; this.mode = options.mode; + this.dataHistogramSpecifications = + this.shaderControlState.histogramSpecifications; this.registerDisposer( this.chunkResolutionHistogram.visibility.add(this.visibility), ); From dc621ee7177d00e1b6ae91423365afbccfdf3159 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 14 May 2024 15:06:55 +0200 Subject: [PATCH 04/41] temp: clear all histogram buffers as demo --- src/perspective_view/panel.ts | 2 ++ src/volume_rendering/volume_render_layer.ts | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/src/perspective_view/panel.ts b/src/perspective_view/panel.ts index 46d0dcc27..4e2dae37f 100644 --- a/src/perspective_view/panel.ts +++ b/src/perspective_view/panel.ts @@ -1065,6 +1065,8 @@ export class PerspectivePanel extends RenderedDataPanel { continue; } renderLayer.draw(renderContext, attachment); + renderContext.bindFramebuffer(); + gl.clearColor(0.0, 0.0, 0.0, 1.0); } // Copy transparent rendering result back to primary buffer. gl.disable(WebGL2RenderingContext.DEPTH_TEST); diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 897851e6a..5a7e73c28 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -192,6 +192,10 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { return true; } + getDataHistogramCount() { + return this.dataHistogramSpecifications.visibleHistograms; + } + constructor(options: VolumeRenderingRenderLayerOptions) { super(); this.gain = options.gain; @@ -812,6 +816,13 @@ void main() { gl.disable(WebGL2RenderingContext.CULL_FACE); endShader(); this.vertexIdHelper.disable(); + const outputBuffers = this.dataHistogramSpecifications.getFramebuffers(gl); + const count = this.getDataHistogramCount(); + for (let i = 0; i < count; ++i) { + outputBuffers[i].bind(256, 1); + gl.clearColor(1.0, 1.0, 1.0, 1.0); + gl.clear(WebGL2RenderingContext.COLOR_BUFFER_BIT); + } } isReady( From 42c0219f6af05753ee56d4687a19e229a4dcd1bf Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 14 May 2024 15:17:37 +0200 Subject: [PATCH 05/41] feat: use VR layer visibility to also control hist showing --- src/volume_rendering/volume_render_layer.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 5a7e73c28..5b2311498 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -212,6 +212,11 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { this.registerDisposer( this.chunkResolutionHistogram.visibility.add(this.visibility), ); + this.registerDisposer( + this.dataHistogramSpecifications.producerVisibility.add( + this.visibility, + ), + ); const extraParameters = this.registerDisposer( makeCachedDerivedWatchableValue( (space: CoordinateSpace, mode: VolumeRenderingModes) => ({ From 3e8e46fe6418159ca4cb1abbcc72ee56918f1a62 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 14 May 2024 15:49:11 +0200 Subject: [PATCH 06/41] feat: only do VR hist drawing if no slice views --- src/perspective_view/panel.ts | 1 + src/perspective_view/render_layer.ts | 5 +++++ src/volume_rendering/volume_render_layer.ts | 19 ++++++++++--------- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/perspective_view/panel.ts b/src/perspective_view/panel.ts index 4e2dae37f..1da8da2dd 100644 --- a/src/perspective_view/panel.ts +++ b/src/perspective_view/panel.ts @@ -877,6 +877,7 @@ export class PerspectivePanel extends RenderedDataPanel { alreadyEmittedPickID: false, bindFramebuffer, frameNumber: this.context.frameNumber, + sliceViewsPresent: this.sliceViews.size > 0, }; mat4.copy( diff --git a/src/perspective_view/render_layer.ts b/src/perspective_view/render_layer.ts index c758e79d4..71da0d757 100644 --- a/src/perspective_view/render_layer.ts +++ b/src/perspective_view/render_layer.ts @@ -55,6 +55,11 @@ export interface PerspectiveViewRenderContext * Specifies the ID of the depth frame buffer texture to query during rendering. */ depthBufferTexture?: WebGLTexture | null; + + /** + * Specifies if there are any slice views + */ + sliceViewsPresent: boolean; } // eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 5b2311498..0b1551332 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -213,9 +213,7 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { this.chunkResolutionHistogram.visibility.add(this.visibility), ); this.registerDisposer( - this.dataHistogramSpecifications.producerVisibility.add( - this.visibility, - ), + this.dataHistogramSpecifications.producerVisibility.add(this.visibility), ); const extraParameters = this.registerDisposer( makeCachedDerivedWatchableValue( @@ -821,12 +819,15 @@ void main() { gl.disable(WebGL2RenderingContext.CULL_FACE); endShader(); this.vertexIdHelper.disable(); - const outputBuffers = this.dataHistogramSpecifications.getFramebuffers(gl); - const count = this.getDataHistogramCount(); - for (let i = 0; i < count; ++i) { - outputBuffers[i].bind(256, 1); - gl.clearColor(1.0, 1.0, 1.0, 1.0); - gl.clear(WebGL2RenderingContext.COLOR_BUFFER_BIT); + if (!renderContext.wireFrame && !renderContext.sliceViewsPresent) { + const outputBuffers = + this.dataHistogramSpecifications.getFramebuffers(gl); + const count = this.getDataHistogramCount(); + for (let i = 0; i < count; ++i) { + outputBuffers[i].bind(256, 1); + gl.clearColor(1.0, 1.0, 1.0, 1.0); + gl.clear(WebGL2RenderingContext.COLOR_BUFFER_BIT); + } } } From ecf2cefd76291ac9ae0b71fe243915d288769242 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 14 May 2024 18:11:54 +0200 Subject: [PATCH 07/41] temp: shader for 3d invlerp --- src/sliceview/volume/frontend.ts | 14 ++- src/volume_rendering/volume_render_layer.ts | 99 +++++++++++++++++++++ 2 files changed, 111 insertions(+), 2 deletions(-) diff --git a/src/sliceview/volume/frontend.ts b/src/sliceview/volume/frontend.ts index d2427b905..fab36ec3e 100644 --- a/src/sliceview/volume/frontend.ts +++ b/src/sliceview/volume/frontend.ts @@ -89,8 +89,10 @@ export function defineChunkDataShaderAccess( chunkFormat: ChunkFormat, numChannelDimensions: number, getPositionWithinChunkExpr: string, + inVertexShader: boolean = false, ) { const { dataType } = chunkFormat; + // TODO (SKM) - something I can do about this? Vertex shader chunkFormat.defineShader(builder, numChannelDimensions); let dataAccessChannelParams = ""; let dataAccessChannelArgs = ""; @@ -104,7 +106,11 @@ export function defineChunkDataShaderAccess( } } - builder.addFragmentCode(glsl_mixLinear); + if (inVertexShader) { + builder.addVertexCode(glsl_mixLinear); + } else { + builder.addFragmentCode(glsl_mixLinear); + } const dataAccessCode = ` ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { highp ivec3 p = ivec3(max(vec3(0.0, 0.0, 0.0), min(floor(${getPositionWithinChunkExpr}), uChunkDataSize - 1.0))); @@ -134,7 +140,11 @@ ${getShaderType( return mixLinear(xvalues[0], xvalues[1], mixCoeff.x); } `; - builder.addFragmentCode(dataAccessCode); + if (inVertexShader) { + builder.addVertexCode(dataAccessCode); + } else { + builder.addFragmentCode(dataAccessCode); + } if (numChannelDimensions <= 1) { builder.addFragmentCode(` ${getShaderType(dataType)} getDataValue() { return getDataValue(0); } diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 0b1551332..566137b25 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -83,7 +83,9 @@ import { shaderCodeWithLineDirective, } from "#src/webgl/dynamic_shader.js"; import type { HistogramSpecifications } from "#src/webgl/empirical_cdf.js"; +import { defineInvlerpShaderFunction } from "#src/webgl/lerp.js"; import type { ShaderModule, ShaderProgram } from "#src/webgl/shader.js"; +import { glsl_simpleFloatHash } from "#src/webgl/shader_lib.js"; import type { ShaderControlsBuilderState, ShaderControlState, @@ -180,6 +182,12 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { VolumeRenderingShaderParameters >; + private histogramShaderGetter: ParameterizedContextDependentShaderGetter< + { chunkFormat: ChunkFormat }, + ShaderControlsBuilderState, + VolumeRenderingShaderParameters + >; + get gl() { return this.multiscaleSource.chunkManager.gl; } @@ -453,6 +461,84 @@ void main() { }, }, ); + // TODO (SKM) - see volume/renderlayer.ts histogram code and follow the ideas + this.histogramShaderGetter = parameterizedContextDependentShaderGetter( + this, + this.gl, + { + memoizeKey: "VolumeRenderingRenderLayerHistogram", + parameters: options.shaderControlState.builderState, + getContextKey: ({ chunkFormat }) => `${chunkFormat.shaderKey}`, + shaderError: options.shaderError, + extraParameters: extraParameters, + defineShader: ( + builder, + { chunkFormat }, + shaderBuilderState, + shaderParametersState, + ) => { + if (shaderBuilderState.parseResult.errors.length !== 0) { + throw new Error("Invalid UI control specification"); + } + defineVertexId(builder); + defineChunkDataShaderAccess( + builder, + chunkFormat, + shaderParametersState.numChannelDimensions, + "chunkSamplePosition", + true, + ); + // TODO (SKM) provide a way to specify the number of histograms + //const numHistograms = dataHistogramChannelSpecifications.length; + const numHistograms = 1; + const { dataType } = chunkFormat; + for (let i = 0; i < numHistograms; ++i) { + //const { channel } = dataHistogramChannelSpecifications[i]; + //const getDataValueExpr = `getDataValueAt(chunkSamplePosition, 0)`; + const invlerpName = `invlerpForHistogram${i}`; + builder.addVertexCode( + defineInvlerpShaderFunction( + builder, + invlerpName, + dataType, + false, + ), + ); + } + builder.addOutputBuffer("vec4", "outputValue", 0); + builder.addTextureSampler( + "sampler2D", + "uDepthSampler", + depthSamplerTextureUnit, + ); + builder.addVertexCode(glsl_simpleFloatHash); + builder.addVertexCode(` +vec3 chunkSamplePosition; + `); + builder.setVertexMain(` +vec3 p = vec3(simpleFloatHash(vec2(float(gl_VertexID), float(gl_InstanceID))), + simpleFloatHash(vec2(float(gl_VertexID) + 10.0, 5.0 + float(gl_InstanceID))), + simpleFloatHash(vec2(float(gl_VertexID) + 20.0, 15.0 + float(gl_InstanceID))) + ); +float x = invlerpForHistogram0(getDataValueAt(p)); +if (x < 0.0) x = 0.0; +else if (x > 1.0) x = 1.0; +else x = (1.0 + x * 253.0) / 255.0; +float stencilValue = texture(uDepthSampler, p).x; +if (stencilValue == 1.0) { + gl_Position = vec4(2.0, 2.0, 2.0, 1.0); +} else { + gl_Position = vec4(2.0 * (dataValue * 255.0 + 0.5) / 256.0 - 1.0, 0.0, 0.0, 1.0); +} +gl_PointSize = 1.0; +`); + builder.setFragmentMain(` +outputValue = vec4(1.0, 1.0, 1.0, 1.0); +`); + }, + }, + ); + this.vertexIdHelper = this.registerDisposer(VertexIdHelper.get(this.gl)); this.registerDisposer( @@ -559,11 +645,16 @@ void main() { activeIndex: 0, }; let shader: ShaderProgram | null = null; + let histogramShader: ShaderProgram | null = null; let prevChunkFormat: ChunkFormat | undefined | null; let shaderResult: ParameterizedShaderGetterResult< ShaderControlsBuilderState, VolumeRenderingShaderParameters >; + let histogramShaderResult: ParameterizedShaderGetterResult< + ShaderControlsBuilderState, + VolumeRenderingShaderParameters + >; // Size of chunk (in voxels) in the "display" subspace of the chunk coordinate space. const chunkDataDisplaySize = vec3.create(); @@ -670,6 +761,10 @@ void main() { wireFrame: renderContext.wireFrame, }); shader = shaderResult.shader; + histogramShaderResult = this.histogramShaderGetter({ + chunkFormat: chunkFormat!, + }); + histogramShader = histogramShaderResult.shader; if (shader !== null) { shader.bind(); if (chunkFormat !== null) { @@ -828,6 +923,10 @@ void main() { gl.clearColor(1.0, 1.0, 1.0, 1.0); gl.clear(WebGL2RenderingContext.COLOR_BUFFER_BIT); } + console.log(histogramShader); + // if (histogramShader !== null) { + // histogramShader.bind(); + // } } } From 448f277e9119eb1403f85b44a6c6c90715dd291c Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 14 May 2024 19:21:38 +0200 Subject: [PATCH 08/41] temp: more progress --- src/volume_rendering/volume_render_layer.ts | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 566137b25..a4804b78a 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -477,10 +477,14 @@ void main() { shaderBuilderState, shaderParametersState, ) => { - if (shaderBuilderState.parseResult.errors.length !== 0) { - throw new Error("Invalid UI control specification"); - } + shaderBuilderState; defineVertexId(builder); + builder.addVertexCode(` +vec3 chunkSamplePosition; +struct uint16_t { + highp uint value; +}; +`); defineChunkDataShaderAccess( builder, chunkFormat, @@ -512,9 +516,6 @@ void main() { depthSamplerTextureUnit, ); builder.addVertexCode(glsl_simpleFloatHash); - builder.addVertexCode(` -vec3 chunkSamplePosition; - `); builder.setVertexMain(` vec3 p = vec3(simpleFloatHash(vec2(float(gl_VertexID), float(gl_InstanceID))), simpleFloatHash(vec2(float(gl_VertexID) + 10.0, 5.0 + float(gl_InstanceID))), @@ -524,12 +525,7 @@ float x = invlerpForHistogram0(getDataValueAt(p)); if (x < 0.0) x = 0.0; else if (x > 1.0) x = 1.0; else x = (1.0 + x * 253.0) / 255.0; -float stencilValue = texture(uDepthSampler, p).x; -if (stencilValue == 1.0) { - gl_Position = vec4(2.0, 2.0, 2.0, 1.0); -} else { - gl_Position = vec4(2.0 * (dataValue * 255.0 + 0.5) / 256.0 - 1.0, 0.0, 0.0, 1.0); -} +gl_Position = vec4(2.0 * (x * 255.0 + 0.5) / 256.0 - 1.0, 0.0, 0.0, 1.0); gl_PointSize = 1.0; `); builder.setFragmentMain(` @@ -764,7 +760,11 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); histogramShaderResult = this.histogramShaderGetter({ chunkFormat: chunkFormat!, }); + console.log("shaderResult", shaderResult); + console.log("histogramShaderResult", histogramShaderResult); histogramShader = histogramShaderResult.shader; + console.log("shader", shader); + console.log("histogramShader", histogramShader); if (shader !== null) { shader.bind(); if (chunkFormat !== null) { From b5ccff641f5fb059356724fac00bb11cfa222baf Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 14 May 2024 19:42:02 +0200 Subject: [PATCH 09/41] temp: minimal shader to debug with --- src/volume_rendering/volume_render_layer.ts | 113 ++++++++++---------- 1 file changed, 59 insertions(+), 54 deletions(-) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index a4804b78a..0c82356b2 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -83,9 +83,9 @@ import { shaderCodeWithLineDirective, } from "#src/webgl/dynamic_shader.js"; import type { HistogramSpecifications } from "#src/webgl/empirical_cdf.js"; -import { defineInvlerpShaderFunction } from "#src/webgl/lerp.js"; +// import { defineInvlerpShaderFunction } from "#src/webgl/lerp.js"; import type { ShaderModule, ShaderProgram } from "#src/webgl/shader.js"; -import { glsl_simpleFloatHash } from "#src/webgl/shader_lib.js"; +// import { glsl_simpleFloatHash } from "#src/webgl/shader_lib.js"; import type { ShaderControlsBuilderState, ShaderControlState, @@ -478,59 +478,64 @@ void main() { shaderParametersState, ) => { shaderBuilderState; - defineVertexId(builder); - builder.addVertexCode(` -vec3 chunkSamplePosition; -struct uint16_t { - highp uint value; -}; -`); - defineChunkDataShaderAccess( - builder, - chunkFormat, - shaderParametersState.numChannelDimensions, - "chunkSamplePosition", - true, - ); - // TODO (SKM) provide a way to specify the number of histograms - //const numHistograms = dataHistogramChannelSpecifications.length; - const numHistograms = 1; - const { dataType } = chunkFormat; - for (let i = 0; i < numHistograms; ++i) { - //const { channel } = dataHistogramChannelSpecifications[i]; - //const getDataValueExpr = `getDataValueAt(chunkSamplePosition, 0)`; - const invlerpName = `invlerpForHistogram${i}`; - builder.addVertexCode( - defineInvlerpShaderFunction( - builder, - invlerpName, - dataType, - false, - ), - ); - } + chunkFormat; + shaderParametersState; + // defineVertexId(builder); + // builder.addVertexCode(` + // vec3 chunkSamplePosition; + // struct uint16_t { + // highp uint value; + // }; + // `); + // defineChunkDataShaderAccess( + // builder, + // chunkFormat, + // shaderParametersState.numChannelDimensions, + // "chunkSamplePosition", + // true, + // ); + // // TODO (SKM) provide a way to specify the number of histograms + // //const numHistograms = dataHistogramChannelSpecifications.length; + // const numHistograms = 1; + // const { dataType } = chunkFormat; + // for (let i = 0; i < numHistograms; ++i) { + // //const { channel } = dataHistogramChannelSpecifications[i]; + // //const getDataValueExpr = `getDataValueAt(chunkSamplePosition, 0)`; + // const invlerpName = `invlerpForHistogram${i}`; + // builder.addVertexCode( + // defineInvlerpShaderFunction( + // builder, + // invlerpName, + // dataType, + // false, + // ), + // ); + // } + // builder.addOutputBuffer("vec4", "outputValue", 0); + // builder.addTextureSampler( + // "sampler2D", + // "uDepthSampler", + // depthSamplerTextureUnit, + // ); + // builder.addVertexCode(glsl_simpleFloatHash); + // builder.setVertexMain(` + // vec3 p = vec3(simpleFloatHash(vec2(float(gl_VertexID), float(gl_InstanceID))), + // simpleFloatHash(vec2(float(gl_VertexID) + 10.0, 5.0 + float(gl_InstanceID))), + // simpleFloatHash(vec2(float(gl_VertexID) + 20.0, 15.0 + float(gl_InstanceID))) + // ); + // float x = invlerpForHistogram0(getDataValueAt(p)); + // if (x < 0.0) x = 0.0; + // else if (x > 1.0) x = 1.0; + // else x = (1.0 + x * 253.0) / 255.0; + // gl_Position = vec4(2.0 * (x * 255.0 + 0.5) / 256.0 - 1.0, 0.0, 0.0, 1.0); + // gl_PointSize = 1.0; + // `); + // builder.setFragmentMain(` + // outputValue = vec4(1.0, 1.0, 1.0, 1.0); + // `); builder.addOutputBuffer("vec4", "outputValue", 0); - builder.addTextureSampler( - "sampler2D", - "uDepthSampler", - depthSamplerTextureUnit, - ); - builder.addVertexCode(glsl_simpleFloatHash); - builder.setVertexMain(` -vec3 p = vec3(simpleFloatHash(vec2(float(gl_VertexID), float(gl_InstanceID))), - simpleFloatHash(vec2(float(gl_VertexID) + 10.0, 5.0 + float(gl_InstanceID))), - simpleFloatHash(vec2(float(gl_VertexID) + 20.0, 15.0 + float(gl_InstanceID))) - ); -float x = invlerpForHistogram0(getDataValueAt(p)); -if (x < 0.0) x = 0.0; -else if (x > 1.0) x = 1.0; -else x = (1.0 + x * 253.0) / 255.0; -gl_Position = vec4(2.0 * (x * 255.0 + 0.5) / 256.0 - 1.0, 0.0, 0.0, 1.0); -gl_PointSize = 1.0; -`); - builder.setFragmentMain(` -outputValue = vec4(1.0, 1.0, 1.0, 1.0); -`); + builder.setVertexMain(`gl_Position = vec4(0.0, 0.0, 0.0, 1.0);`); + builder.setFragmentMain(`outputValue = vec4(1.0, 1.0, 1.0, 1.0);`); }, }, ); From b62c8cb8c3297e6c3510724662d4dd036d4db921 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Wed, 15 May 2024 15:52:07 +0200 Subject: [PATCH 10/41] revert: remove in vertex --- src/sliceview/volume/frontend.ts | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/sliceview/volume/frontend.ts b/src/sliceview/volume/frontend.ts index fab36ec3e..dea825db8 100644 --- a/src/sliceview/volume/frontend.ts +++ b/src/sliceview/volume/frontend.ts @@ -52,7 +52,11 @@ export interface ChunkFormat { * * where value_type is `getShaderType(this.dataType)`. */ - defineShader: (builder: ShaderBuilder, numChannelDimensions: number) => void; + defineShader: ( + builder: ShaderBuilder, + numChannelDimensions: number, + inVertexShader?: boolean, + ) => void; /** * Called once per RenderLayer when starting to draw chunks, on the ChunkFormat of the first @@ -89,10 +93,8 @@ export function defineChunkDataShaderAccess( chunkFormat: ChunkFormat, numChannelDimensions: number, getPositionWithinChunkExpr: string, - inVertexShader: boolean = false, ) { const { dataType } = chunkFormat; - // TODO (SKM) - something I can do about this? Vertex shader chunkFormat.defineShader(builder, numChannelDimensions); let dataAccessChannelParams = ""; let dataAccessChannelArgs = ""; @@ -106,11 +108,7 @@ export function defineChunkDataShaderAccess( } } - if (inVertexShader) { - builder.addVertexCode(glsl_mixLinear); - } else { - builder.addFragmentCode(glsl_mixLinear); - } + builder.addFragmentCode(glsl_mixLinear); const dataAccessCode = ` ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { highp ivec3 p = ivec3(max(vec3(0.0, 0.0, 0.0), min(floor(${getPositionWithinChunkExpr}), uChunkDataSize - 1.0))); @@ -140,11 +138,7 @@ ${getShaderType( return mixLinear(xvalues[0], xvalues[1], mixCoeff.x); } `; - if (inVertexShader) { - builder.addVertexCode(dataAccessCode); - } else { - builder.addFragmentCode(dataAccessCode); - } + builder.addFragmentCode(dataAccessCode); if (numChannelDimensions <= 1) { builder.addFragmentCode(` ${getShaderType(dataType)} getDataValue() { return getDataValue(0); } From 304374091e80d1d5a7f8859e7acbe2ce210c1c37 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Wed, 15 May 2024 15:52:22 +0200 Subject: [PATCH 11/41] temp: compiling vertex shader for data access --- src/sliceview/uncompressed_chunk_format.ts | 36 +++++-- src/volume_rendering/volume_render_layer.ts | 108 +++++++++++--------- 2 files changed, 85 insertions(+), 59 deletions(-) diff --git a/src/sliceview/uncompressed_chunk_format.ts b/src/sliceview/uncompressed_chunk_format.ts index 836bd2201..9896b7d56 100644 --- a/src/sliceview/uncompressed_chunk_format.ts +++ b/src/sliceview/uncompressed_chunk_format.ts @@ -139,7 +139,11 @@ export class ChunkFormat ); } - defineShader(builder: ShaderBuilder, numChannelDimensions: number) { + defineShader( + builder: ShaderBuilder, + numChannelDimensions: number, + inVertexShader: boolean = false, + ) { super.defineShader(builder, numChannelDimensions); const { textureDims } = this; const textureVecType = `ivec${this.textureDims}`; @@ -153,13 +157,23 @@ export class ChunkFormat "uVolumeChunkStrides", 4 + numChannelDimensions, ); - builder.addFragmentCode( - textureAccessHelper.getAccessor( - "readVolumeData", - "uVolumeChunkSampler", - this.dataType, - ), - ); + if (inVertexShader) { + builder.addVertexCode( + textureAccessHelper.getAccessor( + "readVolumeData", + "uVolumeChunkSampler", + this.dataType, + ), + ); + } else { + builder.addFragmentCode( + textureAccessHelper.getAccessor( + "readVolumeData", + "uVolumeChunkSampler", + this.dataType, + ), + ); + } const shaderType = getShaderType(this.dataType); let code = ` ${shaderType} getDataValueAt(highp ivec3 p`; @@ -181,7 +195,11 @@ ${shaderType} getDataValueAt(highp ivec3 p`; return readVolumeData(offset); } `; - builder.addFragmentCode(code); + if (inVertexShader) { + builder.addVertexCode(code); + } else { + builder.addFragmentCode(code); + } } /** diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 0c82356b2..ef9ff3de2 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -83,8 +83,10 @@ import { shaderCodeWithLineDirective, } from "#src/webgl/dynamic_shader.js"; import type { HistogramSpecifications } from "#src/webgl/empirical_cdf.js"; +import { defineInvlerpShaderFunction } from "#src/webgl/lerp.js"; // import { defineInvlerpShaderFunction } from "#src/webgl/lerp.js"; import type { ShaderModule, ShaderProgram } from "#src/webgl/shader.js"; +import { getShaderType, glsl_simpleFloatHash } from "#src/webgl/shader_lib.js"; // import { glsl_simpleFloatHash } from "#src/webgl/shader_lib.js"; import type { ShaderControlsBuilderState, @@ -480,20 +482,36 @@ void main() { shaderBuilderState; chunkFormat; shaderParametersState; - // defineVertexId(builder); - // builder.addVertexCode(` - // vec3 chunkSamplePosition; - // struct uint16_t { - // highp uint value; - // }; - // `); - // defineChunkDataShaderAccess( - // builder, - // chunkFormat, - // shaderParametersState.numChannelDimensions, - // "chunkSamplePosition", - // true, - // ); + builder.addUniform("highp vec3", "uChunkDataSize"); + //defineVertexId(builder); + builder.addVertexCode(` +vec3 chunkSamplePosition; + `); + const numChannelDimensions = + shaderParametersState.numChannelDimensions; + chunkFormat.defineShader(builder, numChannelDimensions, true); + const { dataType } = chunkFormat; + let dataAccessChannelParams = ""; + let dataAccessChannelArgs = ""; + if (numChannelDimensions === 0) { + dataAccessChannelParams += "highp int ignoredChannelIndex"; + } else { + for ( + let channelDim = 0; + channelDim < numChannelDimensions; + ++channelDim + ) { + if (channelDim !== 0) dataAccessChannelParams += ", "; + dataAccessChannelParams += `highp int channelIndex${channelDim}`; + dataAccessChannelArgs += `, channelIndex${channelDim}`; + } + } + const dataAccessCode = ` +${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { + highp ivec3 p = ivec3(max(vec3(0.0, 0.0, 0.0), min(floor(chunkSamplePosition), uChunkDataSize - 1.0))); + return getDataValueAt(p${dataAccessChannelArgs}); +}`; + builder.addVertexCode(dataAccessCode); // // TODO (SKM) provide a way to specify the number of histograms // //const numHistograms = dataHistogramChannelSpecifications.length; // const numHistograms = 1; @@ -501,41 +519,32 @@ void main() { // for (let i = 0; i < numHistograms; ++i) { // //const { channel } = dataHistogramChannelSpecifications[i]; // //const getDataValueExpr = `getDataValueAt(chunkSamplePosition, 0)`; - // const invlerpName = `invlerpForHistogram${i}`; - // builder.addVertexCode( - // defineInvlerpShaderFunction( - // builder, - // invlerpName, - // dataType, - // false, - // ), - // ); - // } - // builder.addOutputBuffer("vec4", "outputValue", 0); + const invlerpName = `invlerpForHistogram0`; + builder.addVertexCode( + defineInvlerpShaderFunction(builder, invlerpName, dataType, false), + ); // builder.addTextureSampler( // "sampler2D", // "uDepthSampler", // depthSamplerTextureUnit, // ); - // builder.addVertexCode(glsl_simpleFloatHash); - // builder.setVertexMain(` - // vec3 p = vec3(simpleFloatHash(vec2(float(gl_VertexID), float(gl_InstanceID))), - // simpleFloatHash(vec2(float(gl_VertexID) + 10.0, 5.0 + float(gl_InstanceID))), - // simpleFloatHash(vec2(float(gl_VertexID) + 20.0, 15.0 + float(gl_InstanceID))) - // ); - // float x = invlerpForHistogram0(getDataValueAt(p)); - // if (x < 0.0) x = 0.0; - // else if (x > 1.0) x = 1.0; - // else x = (1.0 + x * 253.0) / 255.0; - // gl_Position = vec4(2.0 * (x * 255.0 + 0.5) / 256.0 - 1.0, 0.0, 0.0, 1.0); - // gl_PointSize = 1.0; - // `); - // builder.setFragmentMain(` - // outputValue = vec4(1.0, 1.0, 1.0, 1.0); - // `); + builder.addVertexCode(glsl_simpleFloatHash); + builder.setVertexMain(` + vec3 chunkSamplePosition = vec3(simpleFloatHash(vec2(float(gl_VertexID), float(gl_InstanceID))), + simpleFloatHash(vec2(float(gl_VertexID) + 10.0, 5.0 + float(gl_InstanceID))), + simpleFloatHash(vec2(float(gl_VertexID) + 20.0, 15.0 + float(gl_InstanceID))) + ); + float x = invlerpForHistogram0(getDataValue(0)); + if (x < 0.0) x = 0.0; + else if (x > 1.0) x = 1.0; + else x = (1.0 + x * 253.0) / 255.0; + gl_Position = vec4(2.0 * (x * 255.0 + 0.5) / 256.0 - 1.0, 0.0, 0.0, 1.0); + gl_PointSize = 1.0; + `); + builder.setFragmentMain(` + outputValue = vec4(1.0, 1.0, 1.0, 1.0); + `); builder.addOutputBuffer("vec4", "outputValue", 0); - builder.setVertexMain(`gl_Position = vec4(0.0, 0.0, 0.0, 1.0);`); - builder.setFragmentMain(`outputValue = vec4(1.0, 1.0, 1.0, 1.0);`); }, }, ); @@ -765,10 +774,7 @@ void main() { histogramShaderResult = this.histogramShaderGetter({ chunkFormat: chunkFormat!, }); - console.log("shaderResult", shaderResult); - console.log("histogramShaderResult", histogramShaderResult); histogramShader = histogramShaderResult.shader; - console.log("shader", shader); console.log("histogramShader", histogramShader); if (shader !== null) { shader.bind(); @@ -928,10 +934,12 @@ void main() { gl.clearColor(1.0, 1.0, 1.0, 1.0); gl.clear(WebGL2RenderingContext.COLOR_BUFFER_BIT); } - console.log(histogramShader); - // if (histogramShader !== null) { - // histogramShader.bind(); - // } + //console.log(histogramShader); + const histogramShaderAgain : ShaderProgram | null = histogramShader as ShaderProgram | null; + console.log(histogramShaderAgain); + if (histogramShaderAgain !== null) { + histogramShaderAgain.bind(); + } } } From aca897a55365dd3350b761c85631ea7b6b76d764 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Wed, 15 May 2024 16:36:04 +0200 Subject: [PATCH 12/41] temp: simple buffer output hist --- src/volume_rendering/volume_render_layer.ts | 136 ++++++++++++++++---- 1 file changed, 112 insertions(+), 24 deletions(-) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index ef9ff3de2..0133bb1b5 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -53,6 +53,7 @@ import { makeCachedDerivedWatchableValue, registerNested, } from "#src/trackable_value.js"; +import type { RefCountedValue } from "#src/util/disposable.js"; import { getFrustrumPlanes, mat4, vec3 } from "#src/util/geom.js"; import { clampToInterval } from "#src/util/lerp.js"; import { getObjectId } from "#src/util/object_id.js"; @@ -72,6 +73,8 @@ import { drawBoxes, glsl_getBoxFaceVertexPosition, } from "#src/webgl/bounding_box.js"; +import type { Buffer } from "#src/webgl/buffer.js"; +import { getMemoizedBuffer } from "#src/webgl/buffer.js"; import { glsl_COLORMAPS } from "#src/webgl/colormaps.js"; import type { ParameterizedContextDependentShaderGetter, @@ -84,10 +87,8 @@ import { } from "#src/webgl/dynamic_shader.js"; import type { HistogramSpecifications } from "#src/webgl/empirical_cdf.js"; import { defineInvlerpShaderFunction } from "#src/webgl/lerp.js"; -// import { defineInvlerpShaderFunction } from "#src/webgl/lerp.js"; import type { ShaderModule, ShaderProgram } from "#src/webgl/shader.js"; import { getShaderType, glsl_simpleFloatHash } from "#src/webgl/shader_lib.js"; -// import { glsl_simpleFloatHash } from "#src/webgl/shader_lib.js"; import type { ShaderControlsBuilderState, ShaderControlState, @@ -164,6 +165,12 @@ function clampAndRoundResolutionTargetValue(value: number) { return clampToInterval(depthSamplesBounds, Math.round(value)) as number; } +const histogramSamplesPerInstance = 4096; + +// Number of points to sample in computing the histogram. Increasing this increases the precision +// of the histogram but also slows down rendering. +const histogramSamples = 2 ** 14; + export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { gain: WatchableValueInterface; multiscaleSource: MultiscaleVolumeChunkSource; @@ -206,6 +213,8 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { return this.dataHistogramSpecifications.visibleHistograms; } + private inputIndexBuffer: RefCountedValue; + constructor(options: VolumeRenderingRenderLayerOptions) { super(); this.gain = options.gain; @@ -219,6 +228,13 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { this.mode = options.mode; this.dataHistogramSpecifications = this.shaderControlState.histogramSpecifications; + this.inputIndexBuffer = this.registerDisposer( + getMemoizedBuffer( + this.gl, + WebGL2RenderingContext.ARRAY_BUFFER, + () => new Uint8Array(histogramSamplesPerInstance), + ), + ); this.registerDisposer( this.chunkResolutionHistogram.visibility.add(this.visibility), ); @@ -480,9 +496,19 @@ void main() { shaderParametersState, ) => { shaderBuilderState; - chunkFormat; - shaderParametersState; + builder.addOutputBuffer("vec4", "outputValue", null); builder.addUniform("highp vec3", "uChunkDataSize"); + builder.addAttribute("float", "aInput1"); + const TEMP_SIMPLE = true; + if (TEMP_SIMPLE) { + builder.setVertexMain(` +gl_Position = vec4(0.1, aInput1, 0.0, 1.0); +gl_PointSize = 1.0; +`); + builder.setFragmentMain(`outputValue = vec4(1.0, 1.0, 1.0, 1.0);`); + return; + } + //defineVertexId(builder); builder.addVertexCode(` vec3 chunkSamplePosition; @@ -530,7 +556,7 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { // ); builder.addVertexCode(glsl_simpleFloatHash); builder.setVertexMain(` - vec3 chunkSamplePosition = vec3(simpleFloatHash(vec2(float(gl_VertexID), float(gl_InstanceID))), + vec3 chunkSamplePosition = vec3(simpleFloatHash(vec2(aInput1 + float(gl_VertexID), float(gl_InstanceID))), simpleFloatHash(vec2(float(gl_VertexID) + 10.0, 5.0 + float(gl_InstanceID))), simpleFloatHash(vec2(float(gl_VertexID) + 20.0, 15.0 + float(gl_InstanceID))) ); @@ -538,13 +564,13 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { if (x < 0.0) x = 0.0; else if (x > 1.0) x = 1.0; else x = (1.0 + x * 253.0) / 255.0; - gl_Position = vec4(2.0 * (x * 255.0 + 0.5) / 256.0 - 1.0, 0.0, 0.0, 1.0); + //gl_Position = vec4(2.0 * (x * 255.0 + 0.5) / 256.0 - 1.0, 0.0, 0.0, 1.0); + gl_Position = vec4(0.5, 0.0, 0.0, 1.0); gl_PointSize = 1.0; `); builder.setFragmentMain(` outputValue = vec4(1.0, 1.0, 1.0, 1.0); `); - builder.addOutputBuffer("vec4", "outputValue", 0); }, }, ); @@ -733,6 +759,19 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { gl.enable(WebGL2RenderingContext.CULL_FACE); gl.cullFace(WebGL2RenderingContext.FRONT); + if (!renderContext.wireFrame && !renderContext.sliceViewsPresent) { + const outputBuffers = + this.dataHistogramSpecifications.getFramebuffers(gl); + const count = this.getDataHistogramCount(); + for (let i = 0; i < count; ++i) { + outputBuffers[i].bind(256, 1); + gl.clearColor(0.0, 0.0, 0.0, 1.0); + gl.clear(WebGL2RenderingContext.COLOR_BUFFER_BIT); + } + // TODO (SKM) handle max projection + renderContext.bindFramebuffer(); + } + forEachVisibleVolumeRenderingChunk( renderContext.projectionParameters, this.localPosition.value, @@ -775,7 +814,6 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { chunkFormat: chunkFormat!, }); histogramShader = histogramShaderResult.shader; - console.log("histogramShader", histogramShader); if (shader !== null) { shader.bind(); if (chunkFormat !== null) { @@ -916,6 +954,55 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { newSource = false; gl.uniform3fv(shader.uniform("uTranslation"), chunkPosition); drawBoxes(gl, 1, 1); + console.log(histogramShader); + histogramSamples; + this.inputIndexBuffer; + if (histogramShader !== null) { + const outputBuffers = + this.dataHistogramSpecifications.getFramebuffers(gl); + // const count = this.getDataHistogramCount(); + // for (let i = 0; i < count; ++i) { + // outputBuffers[i].bind(256, 1); + // } + outputBuffers[0].bind(256, 1); + // TODO (SKM) handle max projection + histogramShader.bind(); + gl.disable(WebGL2RenderingContext.DEPTH_TEST); + gl.enable(WebGL2RenderingContext.BLEND); + this.inputIndexBuffer.value.bindToVertexAttrib( + histogramShader.attribute("aInput1"), + 1, + WebGL2RenderingContext.UNSIGNED_BYTE, + /*normalized=*/ true, + ); + gl.drawArraysInstanced( + WebGL2RenderingContext.POINTS, + 0, + histogramSamplesPerInstance, + histogramSamples / histogramSamplesPerInstance, + ); + gl.enable(WebGL2RenderingContext.DEPTH_TEST); + + const DEBUG_HISTOGRAMS = true; + if (DEBUG_HISTOGRAMS) { + const tempBuffer = new Float32Array(256 * 4); + gl.readPixels( + 0, + 0, + 256, + 1, + WebGL2RenderingContext.RGBA, + WebGL2RenderingContext.FLOAT, + tempBuffer, + ); + const tempBuffer2 = new Float32Array(256); + for (let j = 0; j < 256; ++j) { + tempBuffer2[j] = tempBuffer[j * 4]; + } + console.log("histogram", tempBuffer2.join(" ")); + } + renderContext.bindFramebuffer(); + } ++presentCount; } else { ++notPresentCount; @@ -925,22 +1012,23 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { gl.disable(WebGL2RenderingContext.CULL_FACE); endShader(); this.vertexIdHelper.disable(); - if (!renderContext.wireFrame && !renderContext.sliceViewsPresent) { - const outputBuffers = - this.dataHistogramSpecifications.getFramebuffers(gl); - const count = this.getDataHistogramCount(); - for (let i = 0; i < count; ++i) { - outputBuffers[i].bind(256, 1); - gl.clearColor(1.0, 1.0, 1.0, 1.0); - gl.clear(WebGL2RenderingContext.COLOR_BUFFER_BIT); - } - //console.log(histogramShader); - const histogramShaderAgain : ShaderProgram | null = histogramShader as ShaderProgram | null; - console.log(histogramShaderAgain); - if (histogramShaderAgain !== null) { - histogramShaderAgain.bind(); - } - } + // if (!renderContext.wireFrame && !renderContext.sliceViewsPresent) { + // const outputBuffers = + // this.dataHistogramSpecifications.getFramebuffers(gl); + // const count = this.getDataHistogramCount(); + // for (let i = 0; i < count; ++i) { + // outputBuffers[i].bind(256, 1); + // gl.clearColor(1.0, 1.0, 1.0, 1.0); + // gl.clear(WebGL2RenderingContext.COLOR_BUFFER_BIT); + // } + // //console.log(histogramShader); + // const histogramShaderAgain: ShaderProgram | null = + // histogramShader as ShaderProgram | null; + // console.log(histogramShaderAgain); + // if (histogramShaderAgain !== null) { + // histogramShaderAgain.bind(); + // } + // } } isReady( From f6b231c367142384d99692f06292e69e63081060 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Wed, 15 May 2024 16:50:02 +0200 Subject: [PATCH 13/41] fix: volume chunks work ok --- src/perspective_view/panel.ts | 2 -- src/volume_rendering/volume_render_layer.ts | 15 +++++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/perspective_view/panel.ts b/src/perspective_view/panel.ts index 1da8da2dd..2495c5b3e 100644 --- a/src/perspective_view/panel.ts +++ b/src/perspective_view/panel.ts @@ -1066,8 +1066,6 @@ export class PerspectivePanel extends RenderedDataPanel { continue; } renderLayer.draw(renderContext, attachment); - renderContext.bindFramebuffer(); - gl.clearColor(0.0, 0.0, 0.0, 1.0); } // Copy transparent rendering result back to primary buffer. gl.disable(WebGL2RenderingContext.DEPTH_TEST); diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 0133bb1b5..ad5bcbcee 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -499,7 +499,7 @@ void main() { builder.addOutputBuffer("vec4", "outputValue", null); builder.addUniform("highp vec3", "uChunkDataSize"); builder.addAttribute("float", "aInput1"); - const TEMP_SIMPLE = true; + const TEMP_SIMPLE = false; if (TEMP_SIMPLE) { builder.setVertexMain(` gl_Position = vec4(0.1, aInput1, 0.0, 1.0); @@ -564,8 +564,7 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { if (x < 0.0) x = 0.0; else if (x > 1.0) x = 1.0; else x = (1.0 + x * 253.0) / 255.0; - //gl_Position = vec4(2.0 * (x * 255.0 + 0.5) / 256.0 - 1.0, 0.0, 0.0, 1.0); - gl_Position = vec4(0.5, 0.0, 0.0, 1.0); + gl_Position = vec4(2.0 * (x * 255.0 + 0.5) / 256.0 - 1.0, 0.0, 0.0, 1.0); gl_PointSize = 1.0; `); builder.setFragmentMain(` @@ -957,7 +956,8 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { console.log(histogramShader); histogramSamples; this.inputIndexBuffer; - if (histogramShader !== null) { + const DO_DRAW = true; + if (histogramShader !== null && DO_DRAW) { const outputBuffers = this.dataHistogramSpecifications.getFramebuffers(gl); // const count = this.getDataHistogramCount(); @@ -965,7 +965,7 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { // outputBuffers[i].bind(256, 1); // } outputBuffers[0].bind(256, 1); - // TODO (SKM) handle max projection + //TODO (SKM) handle max projection histogramShader.bind(); gl.disable(WebGL2RenderingContext.DEPTH_TEST); gl.enable(WebGL2RenderingContext.BLEND); @@ -981,7 +981,6 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { histogramSamplesPerInstance, histogramSamples / histogramSamplesPerInstance, ); - gl.enable(WebGL2RenderingContext.DEPTH_TEST); const DEBUG_HISTOGRAMS = true; if (DEBUG_HISTOGRAMS) { @@ -1001,7 +1000,11 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { } console.log("histogram", tempBuffer2.join(" ")); } + // TODO (SKM) first bind - see picking in VR branch + gl.enable(WebGL2RenderingContext.DEPTH_TEST); renderContext.bindFramebuffer(); + shader.bind(); + this.vertexIdHelper.enable(); } ++presentCount; } else { From 01574db1492ca644b75d366cd326a3c79c4c00ea Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Wed, 15 May 2024 17:35:31 +0200 Subject: [PATCH 14/41] fix: working hist for one chunk broken rendering though --- src/volume_rendering/volume_render_layer.ts | 56 +++++++++++++++++---- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index ad5bcbcee..bfa0642a4 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -86,7 +86,10 @@ import { shaderCodeWithLineDirective, } from "#src/webgl/dynamic_shader.js"; import type { HistogramSpecifications } from "#src/webgl/empirical_cdf.js"; -import { defineInvlerpShaderFunction } from "#src/webgl/lerp.js"; +import { + defineInvlerpShaderFunction, + enableLerpShaderFunction, +} from "#src/webgl/lerp.js"; import type { ShaderModule, ShaderProgram } from "#src/webgl/shader.js"; import { getShaderType, glsl_simpleFloatHash } from "#src/webgl/shader_lib.js"; import type { @@ -501,8 +504,13 @@ void main() { builder.addAttribute("float", "aInput1"); const TEMP_SIMPLE = false; if (TEMP_SIMPLE) { + builder.addVertexCode(glsl_simpleFloatHash); builder.setVertexMain(` -gl_Position = vec4(0.1, aInput1, 0.0, 1.0); +float x = simpleFloatHash(vec2(aInput1 + float(gl_VertexID), float(gl_InstanceID))); +if (x < 0.0) x = 0.0; +else if (x > 1.0) x = 1.0; +else x = (1.0 + x * 253.0) / 255.0; +gl_Position = vec4(2.0 * (x * 255.0 + 0.5) / 256.0 - 1.0, 0.0, 0.0, 1.0); gl_PointSize = 1.0; `); builder.setFragmentMain(`outputValue = vec4(1.0, 1.0, 1.0, 1.0);`); @@ -556,15 +564,14 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { // ); builder.addVertexCode(glsl_simpleFloatHash); builder.setVertexMain(` - vec3 chunkSamplePosition = vec3(simpleFloatHash(vec2(aInput1 + float(gl_VertexID), float(gl_InstanceID))), - simpleFloatHash(vec2(float(gl_VertexID) + 10.0, 5.0 + float(gl_InstanceID))), - simpleFloatHash(vec2(float(gl_VertexID) + 20.0, 15.0 + float(gl_InstanceID))) + vec3 rand3 = vec3(simpleFloatHash(vec2(aInput1 + float(gl_VertexID), float(gl_InstanceID))), + simpleFloatHash(vec2(aInput1 + float(gl_VertexID) + 10.0, 5.0 + float(gl_InstanceID))), + simpleFloatHash(vec2(aInput1 + float(gl_VertexID) + 20.0, 15.0 + float(gl_InstanceID))) ); + chunkSamplePosition = rand3 * uChunkDataSize; float x = invlerpForHistogram0(getDataValue(0)); - if (x < 0.0) x = 0.0; - else if (x > 1.0) x = 1.0; - else x = (1.0 + x * 253.0) / 255.0; - gl_Position = vec4(2.0 * (x * 255.0 + 0.5) / 256.0 - 1.0, 0.0, 0.0, 1.0); + x = clamp(x, 0.0, 1.0); + gl_Position = vec4(x * 2.0 - 1.0, 0.0, 0.0, 1.0); gl_PointSize = 1.0; `); builder.setFragmentMain(` @@ -967,6 +974,24 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { outputBuffers[0].bind(256, 1); //TODO (SKM) handle max projection histogramShader.bind(); + const chunkFormat = transformedSource.source.chunkFormat; + chunkFormat.beginDrawing(gl, histogramShader); + chunkFormat.beginSource(gl, histogramShader); + chunkFormat.bindChunk( + gl, + histogramShader!, + chunk, + fixedPositionWithinChunk, + chunkDisplayDimensionIndices, + channelToChunkDimensionIndices, + true, + ); + // TODO (SKM) need uniform again? + gl.uniform3fv( + shader.uniform("uChunkDataSize"), + chunkDataDisplaySize, + ); + console.log(chunkDataDisplaySize) gl.disable(WebGL2RenderingContext.DEPTH_TEST); gl.enable(WebGL2RenderingContext.BLEND); this.inputIndexBuffer.value.bindToVertexAttrib( @@ -975,6 +1000,15 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { WebGL2RenderingContext.UNSIGNED_BYTE, /*normalized=*/ true, ); + const dataType = this.dataType; + const histogramSpecifications = this.dataHistogramSpecifications; + enableLerpShaderFunction( + histogramShader, + "invlerpForHistogram0", + dataType, + histogramSpecifications.bounds.value[0], + ); + console.log(histogramSpecifications.bounds.value[0]); gl.drawArraysInstanced( WebGL2RenderingContext.POINTS, 0, @@ -1004,6 +1038,10 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { gl.enable(WebGL2RenderingContext.DEPTH_TEST); renderContext.bindFramebuffer(); shader.bind(); + gl.uniform3fv( + shader.uniform("uChunkDataSize"), + chunkDataDisplaySize, + ); this.vertexIdHelper.enable(); } ++presentCount; From 034be6d3ad037324fcb455231fa2a97a1f8b5a89 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Wed, 15 May 2024 17:51:40 +0200 Subject: [PATCH 15/41] fix: drawing works but very slow. Multi-pass probably better but then that's another pass --- src/volume_rendering/volume_render_layer.ts | 37 ++++++++++++--------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index bfa0642a4..3b08afba6 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -762,10 +762,15 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { const chunkRank = this.multiscaleSource.rank; const chunkPosition = vec3.create(); + const needToDrawHistogram = + this.getDataHistogramCount() > 0 && + !renderContext.wireFrame && + !renderContext.sliceViewsPresent; + gl.enable(WebGL2RenderingContext.CULL_FACE); gl.cullFace(WebGL2RenderingContext.FRONT); - if (!renderContext.wireFrame && !renderContext.sliceViewsPresent) { + if (needToDrawHistogram) { const outputBuffers = this.dataHistogramSpecifications.getFramebuffers(gl); const count = this.getDataHistogramCount(); @@ -957,14 +962,11 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { newSource, ); } - newSource = false; gl.uniform3fv(shader.uniform("uTranslation"), chunkPosition); drawBoxes(gl, 1, 1); console.log(histogramShader); - histogramSamples; - this.inputIndexBuffer; const DO_DRAW = true; - if (histogramShader !== null && DO_DRAW) { + if (histogramShader !== null && DO_DRAW && needToDrawHistogram) { const outputBuffers = this.dataHistogramSpecifications.getFramebuffers(gl); // const count = this.getDataHistogramCount(); @@ -977,21 +979,23 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { const chunkFormat = transformedSource.source.chunkFormat; chunkFormat.beginDrawing(gl, histogramShader); chunkFormat.beginSource(gl, histogramShader); - chunkFormat.bindChunk( - gl, - histogramShader!, - chunk, - fixedPositionWithinChunk, - chunkDisplayDimensionIndices, - channelToChunkDimensionIndices, - true, - ); + if (prevChunkFormat != null) { + prevChunkFormat.bindChunk( + gl, + histogramShader!, + chunk, + fixedPositionWithinChunk, + chunkDisplayDimensionIndices, + channelToChunkDimensionIndices, + newSource, + ); + } // TODO (SKM) need uniform again? gl.uniform3fv( shader.uniform("uChunkDataSize"), chunkDataDisplaySize, ); - console.log(chunkDataDisplaySize) + console.log(chunkDataDisplaySize); gl.disable(WebGL2RenderingContext.DEPTH_TEST); gl.enable(WebGL2RenderingContext.BLEND); this.inputIndexBuffer.value.bindToVertexAttrib( @@ -1043,7 +1047,10 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { chunkDataDisplaySize, ); this.vertexIdHelper.enable(); + chunkFormat.beginDrawing(gl, shader); + chunkFormat.beginSource(gl, shader); } + newSource = false; ++presentCount; } else { ++notPresentCount; From ce05ebf464079eedb3b5ca210849d920d43c9297 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Wed, 15 May 2024 18:11:45 +0200 Subject: [PATCH 16/41] fix: working histogram 3d --- src/volume_rendering/volume_render_layer.ts | 108 +++++++------------- 1 file changed, 36 insertions(+), 72 deletions(-) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 3b08afba6..f26df0255 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -168,11 +168,11 @@ function clampAndRoundResolutionTargetValue(value: number) { return clampToInterval(depthSamplesBounds, Math.round(value)) as number; } -const histogramSamplesPerInstance = 4096; +const histogramSamplesPerInstance = 1024; // Number of points to sample in computing the histogram. Increasing this increases the precision // of the histogram but also slows down rendering. -const histogramSamples = 2 ** 14; +const histogramSamples = 4096; export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { gain: WatchableValueInterface; @@ -502,22 +502,6 @@ void main() { builder.addOutputBuffer("vec4", "outputValue", null); builder.addUniform("highp vec3", "uChunkDataSize"); builder.addAttribute("float", "aInput1"); - const TEMP_SIMPLE = false; - if (TEMP_SIMPLE) { - builder.addVertexCode(glsl_simpleFloatHash); - builder.setVertexMain(` -float x = simpleFloatHash(vec2(aInput1 + float(gl_VertexID), float(gl_InstanceID))); -if (x < 0.0) x = 0.0; -else if (x > 1.0) x = 1.0; -else x = (1.0 + x * 253.0) / 255.0; -gl_Position = vec4(2.0 * (x * 255.0 + 0.5) / 256.0 - 1.0, 0.0, 0.0, 1.0); -gl_PointSize = 1.0; -`); - builder.setFragmentMain(`outputValue = vec4(1.0, 1.0, 1.0, 1.0);`); - return; - } - - //defineVertexId(builder); builder.addVertexCode(` vec3 chunkSamplePosition; `); @@ -557,25 +541,21 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { builder.addVertexCode( defineInvlerpShaderFunction(builder, invlerpName, dataType, false), ); - // builder.addTextureSampler( - // "sampler2D", - // "uDepthSampler", - // depthSamplerTextureUnit, - // ); builder.addVertexCode(glsl_simpleFloatHash); builder.setVertexMain(` - vec3 rand3 = vec3(simpleFloatHash(vec2(aInput1 + float(gl_VertexID), float(gl_InstanceID))), - simpleFloatHash(vec2(aInput1 + float(gl_VertexID) + 10.0, 5.0 + float(gl_InstanceID))), - simpleFloatHash(vec2(aInput1 + float(gl_VertexID) + 20.0, 15.0 + float(gl_InstanceID))) - ); - chunkSamplePosition = rand3 * uChunkDataSize; - float x = invlerpForHistogram0(getDataValue(0)); - x = clamp(x, 0.0, 1.0); - gl_Position = vec4(x * 2.0 - 1.0, 0.0, 0.0, 1.0); - gl_PointSize = 1.0; +vec3 rand3 = vec3(simpleFloatHash(vec2(aInput1 + float(gl_VertexID), float(gl_InstanceID))), + simpleFloatHash(vec2(aInput1 + float(gl_VertexID) + 10.0, 5.0 + float(gl_InstanceID))), + simpleFloatHash(vec2(aInput1 + float(gl_VertexID) + 20.0, 15.0 + float(gl_InstanceID))) + ); +chunkSamplePosition = rand3 * (uChunkDataSize - 1.0); +//chunkSamplePosition = rand3; +float x = invlerpForHistogram0(getDataValue(0)); +x = clamp(x, 0.0, 1.0); +gl_Position = vec4(x * 2.0 - 1.0, 0.0, 0.0, 1.0); +gl_PointSize = 1.0; `); builder.setFragmentMain(` - outputValue = vec4(1.0, 1.0, 1.0, 1.0); +outputValue = vec4(1.0, 1.0, 1.0, 1.0); `); }, }, @@ -964,7 +944,6 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { } gl.uniform3fv(shader.uniform("uTranslation"), chunkPosition); drawBoxes(gl, 1, 1); - console.log(histogramShader); const DO_DRAW = true; if (histogramShader !== null && DO_DRAW && needToDrawHistogram) { const outputBuffers = @@ -992,10 +971,9 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { } // TODO (SKM) need uniform again? gl.uniform3fv( - shader.uniform("uChunkDataSize"), + histogramShader.uniform("uChunkDataSize"), chunkDataDisplaySize, ); - console.log(chunkDataDisplaySize); gl.disable(WebGL2RenderingContext.DEPTH_TEST); gl.enable(WebGL2RenderingContext.BLEND); this.inputIndexBuffer.value.bindToVertexAttrib( @@ -1012,7 +990,6 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { dataType, histogramSpecifications.bounds.value[0], ); - console.log(histogramSpecifications.bounds.value[0]); gl.drawArraysInstanced( WebGL2RenderingContext.POINTS, 0, @@ -1020,24 +997,6 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { histogramSamples / histogramSamplesPerInstance, ); - const DEBUG_HISTOGRAMS = true; - if (DEBUG_HISTOGRAMS) { - const tempBuffer = new Float32Array(256 * 4); - gl.readPixels( - 0, - 0, - 256, - 1, - WebGL2RenderingContext.RGBA, - WebGL2RenderingContext.FLOAT, - tempBuffer, - ); - const tempBuffer2 = new Float32Array(256); - for (let j = 0; j < 256; ++j) { - tempBuffer2[j] = tempBuffer[j * 4]; - } - console.log("histogram", tempBuffer2.join(" ")); - } // TODO (SKM) first bind - see picking in VR branch gl.enable(WebGL2RenderingContext.DEPTH_TEST); renderContext.bindFramebuffer(); @@ -1060,23 +1019,28 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { gl.disable(WebGL2RenderingContext.CULL_FACE); endShader(); this.vertexIdHelper.disable(); - // if (!renderContext.wireFrame && !renderContext.sliceViewsPresent) { - // const outputBuffers = - // this.dataHistogramSpecifications.getFramebuffers(gl); - // const count = this.getDataHistogramCount(); - // for (let i = 0; i < count; ++i) { - // outputBuffers[i].bind(256, 1); - // gl.clearColor(1.0, 1.0, 1.0, 1.0); - // gl.clear(WebGL2RenderingContext.COLOR_BUFFER_BIT); - // } - // //console.log(histogramShader); - // const histogramShaderAgain: ShaderProgram | null = - // histogramShader as ShaderProgram | null; - // console.log(histogramShaderAgain); - // if (histogramShaderAgain !== null) { - // histogramShaderAgain.bind(); - // } - // } + const DEBUG_HISTOGRAMS = true; + if (needToDrawHistogram && DEBUG_HISTOGRAMS) { + const outputBuffers = + this.dataHistogramSpecifications.getFramebuffers(gl); + outputBuffers[0].bind(256, 1); + const tempBuffer = new Float32Array(256 * 4); + gl.readPixels( + 0, + 0, + 256, + 1, + WebGL2RenderingContext.RGBA, + WebGL2RenderingContext.FLOAT, + tempBuffer, + ); + const tempBuffer2 = new Float32Array(256); + for (let j = 0; j < 256; ++j) { + tempBuffer2[j] = tempBuffer[j * 4]; + } + console.log("histogram", tempBuffer2.join(" ")); + renderContext.bindFramebuffer(); + } } isReady( From 6d1db25b20a52e60d7c5cc4d13d9abd5f7d53c64 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Wed, 15 May 2024 18:28:05 +0200 Subject: [PATCH 17/41] fix: histogram data display of off invlerp range --- src/volume_rendering/volume_render_layer.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index f26df0255..f4c2c69ee 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -548,10 +548,16 @@ vec3 rand3 = vec3(simpleFloatHash(vec2(aInput1 + float(gl_VertexID), float(gl_In simpleFloatHash(vec2(aInput1 + float(gl_VertexID) + 20.0, 15.0 + float(gl_InstanceID))) ); chunkSamplePosition = rand3 * (uChunkDataSize - 1.0); -//chunkSamplePosition = rand3; float x = invlerpForHistogram0(getDataValue(0)); -x = clamp(x, 0.0, 1.0); -gl_Position = vec4(x * 2.0 - 1.0, 0.0, 0.0, 1.0); +if (x == 0.0) { + gl_Position = vec4(2.0, 2.0, 2.0, 1.0); +} +else { + if (x < 0.0) x = 0.0; + else if (x > 1.0) x = 1.0; + else x = (1.0 + x * 253.0) / 255.0; + gl_Position = vec4(2.0 * (x * 255.0 + 0.5) / 256.0 - 1.0, 0.0, 0.0, 1.0); +} gl_PointSize = 1.0; `); builder.setFragmentMain(` From 04828d8eb97b538b7ee1391e94e033cb346d9dae Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Thu, 16 May 2024 11:28:57 +0200 Subject: [PATCH 18/41] feat: work towards multi-channel multi-invlerp support --- src/volume_rendering/volume_render_layer.ts | 91 +++++++++++++-------- 1 file changed, 59 insertions(+), 32 deletions(-) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index f4c2c69ee..abdcadaf2 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -85,7 +85,10 @@ import { parameterizedContextDependentShaderGetter, shaderCodeWithLineDirective, } from "#src/webgl/dynamic_shader.js"; -import type { HistogramSpecifications } from "#src/webgl/empirical_cdf.js"; +import type { + HistogramChannelSpecification, + HistogramSpecifications, +} from "#src/webgl/empirical_cdf.js"; import { defineInvlerpShaderFunction, enableLerpShaderFunction, @@ -246,11 +249,20 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { ); const extraParameters = this.registerDisposer( makeCachedDerivedWatchableValue( - (space: CoordinateSpace, mode: VolumeRenderingModes) => ({ + ( + space: CoordinateSpace, + mode: VolumeRenderingModes, + dataHistogramChannelSpecifications: HistogramChannelSpecification[], + ) => ({ numChannelDimensions: space.rank, mode, + dataHistogramChannelSpecifications, }), - [this.channelCoordinateSpace, this.mode], + [ + this.channelCoordinateSpace, + this.mode, + this.dataHistogramSpecifications.channels, + ], ), ); this.shaderGetter = parameterizedContextDependentShaderGetter( @@ -501,6 +513,7 @@ void main() { shaderBuilderState; builder.addOutputBuffer("vec4", "outputValue", null); builder.addUniform("highp vec3", "uChunkDataSize"); + builder.addUniform("highp int", "uHistogramIndex"); builder.addAttribute("float", "aInput1"); builder.addVertexCode(` vec3 chunkSamplePosition; @@ -530,17 +543,32 @@ ${getShaderType(dataType)} getDataValue(${dataAccessChannelParams}) { return getDataValueAt(p${dataAccessChannelArgs}); }`; builder.addVertexCode(dataAccessCode); - // // TODO (SKM) provide a way to specify the number of histograms - // //const numHistograms = dataHistogramChannelSpecifications.length; - // const numHistograms = 1; - // const { dataType } = chunkFormat; - // for (let i = 0; i < numHistograms; ++i) { - // //const { channel } = dataHistogramChannelSpecifications[i]; - // //const getDataValueExpr = `getDataValueAt(chunkSamplePosition, 0)`; - const invlerpName = `invlerpForHistogram0`; - builder.addVertexCode( - defineInvlerpShaderFunction(builder, invlerpName, dataType, false), - ); + if (numChannelDimensions <= 1) { + builder.addVertexCode(` +${getShaderType(dataType)} getDataValue() { return getDataValue(0); } +`); + } + const dataHistogramChannelSpecifications = + shaderParametersState.dataHistogramChannelSpecifications; + const numHistograms = dataHistogramChannelSpecifications.length; + for (let i = 0; i < numHistograms; ++i) { + const { channel } = dataHistogramChannelSpecifications[i]; + const getDataValueExpr = `getDataValue(${channel.join(",")})`; + const invlerpName = `invlerpForHistogram${i}`; + builder.addVertexCode( + defineInvlerpShaderFunction( + builder, + invlerpName, + dataType, + false, + ), + ); + builder.addVertexCode(` + float getHistogramValue${i}() { + return invlerpForHistogram${i}(${getDataValueExpr}); + } + `); + } builder.addVertexCode(glsl_simpleFloatHash); builder.setVertexMain(` vec3 rand3 = vec3(simpleFloatHash(vec2(aInput1 + float(gl_VertexID), float(gl_InstanceID))), @@ -548,7 +576,7 @@ vec3 rand3 = vec3(simpleFloatHash(vec2(aInput1 + float(gl_VertexID), float(gl_In simpleFloatHash(vec2(aInput1 + float(gl_VertexID) + 20.0, 15.0 + float(gl_InstanceID))) ); chunkSamplePosition = rand3 * (uChunkDataSize - 1.0); -float x = invlerpForHistogram0(getDataValue(0)); +float x = getHistogramValue0(); if (x == 0.0) { gl_Position = vec4(2.0, 2.0, 2.0, 1.0); } @@ -950,15 +978,8 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); } gl.uniform3fv(shader.uniform("uTranslation"), chunkPosition); drawBoxes(gl, 1, 1); - const DO_DRAW = true; - if (histogramShader !== null && DO_DRAW && needToDrawHistogram) { - const outputBuffers = - this.dataHistogramSpecifications.getFramebuffers(gl); - // const count = this.getDataHistogramCount(); - // for (let i = 0; i < count; ++i) { - // outputBuffers[i].bind(256, 1); - // } - outputBuffers[0].bind(256, 1); + console.log(histogramShader); + if (histogramShader !== null && needToDrawHistogram) { //TODO (SKM) handle max projection histogramShader.bind(); const chunkFormat = transformedSource.source.chunkFormat; @@ -988,14 +1009,20 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); WebGL2RenderingContext.UNSIGNED_BYTE, /*normalized=*/ true, ); - const dataType = this.dataType; - const histogramSpecifications = this.dataHistogramSpecifications; - enableLerpShaderFunction( - histogramShader, - "invlerpForHistogram0", - dataType, - histogramSpecifications.bounds.value[0], - ); + const { dataType, dataHistogramSpecifications } = this; + const count = this.getDataHistogramCount(); + const outputFramebuffers = + dataHistogramSpecifications.getFramebuffers(gl); + const bounds = this.dataHistogramSpecifications.bounds.value; + for (let i = 0; i < count; ++i) { + outputFramebuffers[i].bind(256, 1); + enableLerpShaderFunction( + histogramShader, + `invlerpForHistogram${i}`, + dataType, + bounds[i], + ); + } gl.drawArraysInstanced( WebGL2RenderingContext.POINTS, 0, From 17eab670de18a05d603d425ad378db0c649541bd Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Thu, 16 May 2024 11:45:43 +0200 Subject: [PATCH 19/41] fix: VR tool from Python --- python/neuroglancer/__init__.py | 2 +- python/neuroglancer/viewer_state.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/neuroglancer/__init__.py b/python/neuroglancer/__init__.py index 367a4431f..836bbebd1 100644 --- a/python/neuroglancer/__init__.py +++ b/python/neuroglancer/__init__.py @@ -49,7 +49,7 @@ PlaceEllipsoidTool, # noqa: F401 BlendTool, # noqa: F401 OpacityTool, # noqa: F401 - VolumeRenderingModeTool, # noqa: F401 + VolumeRenderingTool, # noqa: F401 VolumeRenderingGainTool, # noqa: F401 VolumeRenderingDepthSamplesTool, # noqa: F401 CrossSectionRenderScaleTool, # noqa: F401 diff --git a/python/neuroglancer/viewer_state.py b/python/neuroglancer/viewer_state.py index e23e51514..301cc65f8 100644 --- a/python/neuroglancer/viewer_state.py +++ b/python/neuroglancer/viewer_state.py @@ -165,9 +165,9 @@ class OpacityTool(Tool): @export_tool -class VolumeRenderingModeTool(Tool): +class VolumeRenderingTool(Tool): __slots__ = () - TOOL_TYPE = "volumeRenderingMode" + TOOL_TYPE = "volumeRendering" @export_tool From 4b1e6542cc545c6ddcd20f1fa00c23cdb9f198ca Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Thu, 16 May 2024 11:58:23 +0200 Subject: [PATCH 20/41] fix: VR sample works for multi-invlerp --- src/volume_rendering/volume_render_layer.ts | 36 ++++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index abdcadaf2..429a0ab50 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -171,7 +171,7 @@ function clampAndRoundResolutionTargetValue(value: number) { return clampToInterval(depthSamplesBounds, Math.round(value)) as number; } -const histogramSamplesPerInstance = 1024; +const histogramSamplesPerInstance = 512; // Number of points to sample in computing the histogram. Increasing this increases the precision // of the histogram but also slows down rendering. @@ -551,6 +551,9 @@ ${getShaderType(dataType)} getDataValue() { return getDataValue(0); } const dataHistogramChannelSpecifications = shaderParametersState.dataHistogramChannelSpecifications; const numHistograms = dataHistogramChannelSpecifications.length; + let histogramFetchCode = ` +float x; +switch (uHistogramIndex) {`; for (let i = 0; i < numHistograms; ++i) { const { channel } = dataHistogramChannelSpecifications[i]; const getDataValueExpr = `getDataValue(${channel.join(",")})`; @@ -564,11 +567,19 @@ ${getShaderType(dataType)} getDataValue() { return getDataValue(0); } ), ); builder.addVertexCode(` - float getHistogramValue${i}() { - return invlerpForHistogram${i}(${getDataValueExpr}); - } - `); +float getHistogramValue${i}() { + return invlerpForHistogram${i}(${getDataValueExpr}); +} +`); + histogramFetchCode += ` +case ${i}: + x = getHistogramValue${i}(); + break; +`; } + histogramFetchCode += ` +} +`; builder.addVertexCode(glsl_simpleFloatHash); builder.setVertexMain(` vec3 rand3 = vec3(simpleFloatHash(vec2(aInput1 + float(gl_VertexID), float(gl_InstanceID))), @@ -576,7 +587,7 @@ vec3 rand3 = vec3(simpleFloatHash(vec2(aInput1 + float(gl_VertexID), float(gl_In simpleFloatHash(vec2(aInput1 + float(gl_VertexID) + 20.0, 15.0 + float(gl_InstanceID))) ); chunkSamplePosition = rand3 * (uChunkDataSize - 1.0); -float x = getHistogramValue0(); +${histogramFetchCode} if (x == 0.0) { gl_Position = vec4(2.0, 2.0, 2.0, 1.0); } @@ -1022,13 +1033,14 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); dataType, bounds[i], ); + gl.uniform1i(histogramShader.uniform("uHistogramIndex"), i); + gl.drawArraysInstanced( + WebGL2RenderingContext.POINTS, + 0, + histogramSamplesPerInstance, + histogramSamples / histogramSamplesPerInstance, + ); } - gl.drawArraysInstanced( - WebGL2RenderingContext.POINTS, - 0, - histogramSamplesPerInstance, - histogramSamples / histogramSamplesPerInstance, - ); // TODO (SKM) first bind - see picking in VR branch gl.enable(WebGL2RenderingContext.DEPTH_TEST); From ae7bf33423f42c0835b7d14e7e32958f98e5563c Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Thu, 16 May 2024 12:04:37 +0200 Subject: [PATCH 21/41] fix: VR max projection works with data sampling --- src/perspective_view/panel.ts | 1 + src/perspective_view/render_layer.ts | 5 +++++ src/volume_rendering/volume_render_layer.ts | 13 +++++++------ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/perspective_view/panel.ts b/src/perspective_view/panel.ts index 2495c5b3e..7595b74d5 100644 --- a/src/perspective_view/panel.ts +++ b/src/perspective_view/panel.ts @@ -963,6 +963,7 @@ export class PerspectivePanel extends RenderedDataPanel { }; gl.depthMask(true); bindMaxProjectionBuffer(); + renderContext.bindMaxProjectionBuffer = bindMaxProjectionBuffer; gl.clearColor(0.0, 0.0, 0.0, 0.0); gl.clearDepth(0.0); gl.clear( diff --git a/src/perspective_view/render_layer.ts b/src/perspective_view/render_layer.ts index 71da0d757..fa5e3e348 100644 --- a/src/perspective_view/render_layer.ts +++ b/src/perspective_view/render_layer.ts @@ -60,6 +60,11 @@ export interface PerspectiveViewRenderContext * Specifies if there are any slice views */ sliceViewsPresent: boolean; + + /** + * Specifices how to bind the max projection buffer + */ + bindMaxProjectionBuffer?: () => void | undefined; } // eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 429a0ab50..89ec04317 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -1007,7 +1007,6 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); newSource, ); } - // TODO (SKM) need uniform again? gl.uniform3fv( histogramShader.uniform("uChunkDataSize"), chunkDataDisplaySize, @@ -1044,12 +1043,14 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); // TODO (SKM) first bind - see picking in VR branch gl.enable(WebGL2RenderingContext.DEPTH_TEST); - renderContext.bindFramebuffer(); + if (isProjectionMode(this.mode.value)) { + gl.disable(WebGL2RenderingContext.BLEND); + renderContext.bindMaxProjectionBuffer!(); + } + else { + renderContext.bindFramebuffer(); + } shader.bind(); - gl.uniform3fv( - shader.uniform("uChunkDataSize"), - chunkDataDisplaySize, - ); this.vertexIdHelper.enable(); chunkFormat.beginDrawing(gl, shader); chunkFormat.beginSource(gl, shader); From 3370b7882e465936f8d34d56996f3342201253da Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Thu, 16 May 2024 18:29:59 +0200 Subject: [PATCH 22/41] feat: add ability to only activate chunk textures --- src/sliceview/single_texture_chunk_format.ts | 6 ++++-- src/sliceview/volume/frontend.ts | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/sliceview/single_texture_chunk_format.ts b/src/sliceview/single_texture_chunk_format.ts index 4fe03f273..71180f04d 100644 --- a/src/sliceview/single_texture_chunk_format.ts +++ b/src/sliceview/single_texture_chunk_format.ts @@ -56,10 +56,12 @@ export abstract class SingleTextureChunkFormat abstract get shaderSamplerType(): ShaderSamplerType; - beginDrawing(gl: GL, shader: ShaderProgram) { + beginDrawing(gl: GL, shader: ShaderProgram, onlyActivateTexture: boolean = false) { const textureUnit = shader.textureUnit(textureUnitSymbol); gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + textureUnit); - (shader)[textureLayoutSymbol] = null; + if (!onlyActivateTexture) { + (shader)[textureLayoutSymbol] = null; + } } endDrawing(gl: GL, shader: ShaderProgram) { diff --git a/src/sliceview/volume/frontend.ts b/src/sliceview/volume/frontend.ts index dea825db8..f3a11f289 100644 --- a/src/sliceview/volume/frontend.ts +++ b/src/sliceview/volume/frontend.ts @@ -62,7 +62,7 @@ export interface ChunkFormat { * Called once per RenderLayer when starting to draw chunks, on the ChunkFormat of the first * source. This is not called before each source is drawn. */ - beginDrawing: (gl: GL, shader: ShaderProgram) => void; + beginDrawing: (gl: GL, shader: ShaderProgram, reset?: boolean) => void; /** * Called once after all chunks have been drawn, on the ChunkFormat of the first source. From 99c939c76de2d909e0c5cb49fcd7d1d3db400e94 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Thu, 16 May 2024 18:30:20 +0200 Subject: [PATCH 23/41] fix: more error checking in VR layer for histograms --- src/volume_rendering/volume_render_layer.ts | 41 +++++++++++++++------ 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 89ec04317..5d8e5a5a6 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -737,6 +737,9 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); if (shader === null) return; if (prevChunkFormat !== null) { prevChunkFormat!.endDrawing(gl, shader); + if (histogramShader !== null) { + prevChunkFormat!.endDrawing(gl, histogramShader); + } } const depthTextureUnit = shader.textureUnit(depthSamplerTextureUnit); gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + depthTextureUnit); @@ -804,8 +807,17 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); gl.clearColor(0.0, 0.0, 0.0, 1.0); gl.clear(WebGL2RenderingContext.COLOR_BUFFER_BIT); } - // TODO (SKM) handle max projection - renderContext.bindFramebuffer(); + if (isProjectionMode(this.mode.value)) { + if (renderContext.bindMaxProjectionBuffer !== undefined) { + renderContext.bindMaxProjectionBuffer(); + } else { + throw new Error( + "bindMaxProjectionBuffer is undefined in VolumeRenderingRenderLayer", + ); + } + } else { + renderContext.bindFramebuffer(); + } } forEachVisibleVolumeRenderingChunk( @@ -989,17 +1001,17 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); } gl.uniform3fv(shader.uniform("uTranslation"), chunkPosition); drawBoxes(gl, 1, 1); - console.log(histogramShader); if (histogramShader !== null && needToDrawHistogram) { - //TODO (SKM) handle max projection + // Setup the state for drawing histograms histogramShader.bind(); const chunkFormat = transformedSource.source.chunkFormat; - chunkFormat.beginDrawing(gl, histogramShader); + const onlyActivateTexture = !newSource; + chunkFormat.beginDrawing(gl, histogramShader, onlyActivateTexture); chunkFormat.beginSource(gl, histogramShader); if (prevChunkFormat != null) { prevChunkFormat.bindChunk( gl, - histogramShader!, + histogramShader, chunk, fixedPositionWithinChunk, chunkDisplayDimensionIndices, @@ -1024,6 +1036,8 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); const outputFramebuffers = dataHistogramSpecifications.getFramebuffers(gl); const bounds = this.dataHistogramSpecifications.bounds.value; + + // Draw each histogram for (let i = 0; i < count; ++i) { outputFramebuffers[i].bind(256, 1); enableLerpShaderFunction( @@ -1041,18 +1055,23 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); ); } - // TODO (SKM) first bind - see picking in VR branch + // Reset the state back to regular drawing mode gl.enable(WebGL2RenderingContext.DEPTH_TEST); if (isProjectionMode(this.mode.value)) { gl.disable(WebGL2RenderingContext.BLEND); - renderContext.bindMaxProjectionBuffer!(); - } - else { + if (renderContext.bindMaxProjectionBuffer !== undefined) { + renderContext.bindMaxProjectionBuffer(); + } else { + throw new Error( + "bindMaxProjectionBuffer is undefined in VolumeRenderingRenderLayer", + ); + } + } else { renderContext.bindFramebuffer(); } shader.bind(); this.vertexIdHelper.enable(); - chunkFormat.beginDrawing(gl, shader); + chunkFormat.beginDrawing(gl, shader, true); chunkFormat.beginSource(gl, shader); } newSource = false; From 93c592e8cd39cdf4983484fc7d582cfc83bb525d Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Thu, 16 May 2024 18:31:18 +0200 Subject: [PATCH 24/41] chore: add comment --- src/volume_rendering/volume_render_layer.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 5d8e5a5a6..e4ef41d20 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -1001,6 +1001,8 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); } gl.uniform3fv(shader.uniform("uTranslation"), chunkPosition); drawBoxes(gl, 1, 1); + + // Draw histograms if needed if (histogramShader !== null && needToDrawHistogram) { // Setup the state for drawing histograms histogramShader.bind(); From d8e414cefe2f89bb947e04e8fd6572ca01f98a00 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 20 May 2024 14:42:40 +0200 Subject: [PATCH 25/41] feat: detect camera move in perspective panel --- src/perspective_view/panel.ts | 22 ++++++++++++++++++++++ src/rendered_data_panel.ts | 3 +++ 2 files changed, 25 insertions(+) diff --git a/src/perspective_view/panel.ts b/src/perspective_view/panel.ts index a38bd43e2..949c5f582 100644 --- a/src/perspective_view/panel.ts +++ b/src/perspective_view/panel.ts @@ -81,6 +81,8 @@ import { MultipleScaleBarTextures } from "#src/widget/scale_bar.js"; import type { RPC } from "#src/worker_rpc.js"; import { SharedObject } from "#src/worker_rpc.js"; +const REDRAW_DELAY_AFTER_CAMERA_MOVE = 400; + export interface PerspectiveViewerState extends RenderedDataViewerState { wireFrame: WatchableValueInterface; orthographicProjection: TrackableBoolean; @@ -251,6 +253,7 @@ export class PerspectivePanel extends RenderedDataPanel { protected visibleLayerTracker: Owned< VisibleRenderLayerTracker >; + private redrawAfterMoveTimeOutId: number = -1; get rpc() { return this.sharedObject.rpc!; @@ -261,6 +264,9 @@ export class PerspectivePanel extends RenderedDataPanel { get displayDimensionRenderInfo() { return this.navigationState.displayDimensionRenderInfo; } + get isCameraMoving() { + return this.redrawAfterMoveTimeOutId !== -1; + } /** * If boolean value is true, sliceView is shown unconditionally, regardless of the value of @@ -417,6 +423,22 @@ export class PerspectivePanel extends RenderedDataPanel { this, ); + this.registerDisposer( + this.viewer.navigationState.changed.add(() => { + // Don't mark camera moving on picking requests + if (this.isMovingToMousePosition) { + return; + } + if (this.redrawAfterMoveTimeOutId !== -1) { + window.clearTimeout(this.redrawAfterMoveTimeOutId); + } + this.redrawAfterMoveTimeOutId = window.setTimeout(() => { + this.redrawAfterMoveTimeOutId = -1; + this.context.scheduleRedraw(); + }, REDRAW_DELAY_AFTER_CAMERA_MOVE); + }), + ); + registerActionListener( element, "rotate-via-mouse-drag", diff --git a/src/rendered_data_panel.ts b/src/rendered_data_panel.ts index c62b6ec6f..9c2b95449 100644 --- a/src/rendered_data_panel.ts +++ b/src/rendered_data_panel.ts @@ -165,6 +165,7 @@ export abstract class RenderedDataPanel extends RenderedPanel { pickRequestPending = false; private mouseStateForcer = () => this.blockOnPickRequest(); + protected isMovingToMousePosition: boolean = false; inputEventMap: EventActionMap; @@ -616,7 +617,9 @@ export abstract class RenderedDataPanel extends RenderedPanel { registerActionListener(element, "move-to-mouse-position", () => { const { mouseState } = this.viewer; if (mouseState.updateUnconditionally()) { + this.isMovingToMousePosition = true; this.navigationState.position.value = mouseState.position; + this.isMovingToMousePosition = false; } }); From c3022337853cd1388e287ba7f1b9dd754e2b496d Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 20 May 2024 15:51:54 +0200 Subject: [PATCH 26/41] refactor: rename code snippets in chunk --- src/sliceview/uncompressed_chunk_format.ts | 38 ++++++++-------------- 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/src/sliceview/uncompressed_chunk_format.ts b/src/sliceview/uncompressed_chunk_format.ts index 9896b7d56..f9fb310a9 100644 --- a/src/sliceview/uncompressed_chunk_format.ts +++ b/src/sliceview/uncompressed_chunk_format.ts @@ -157,48 +157,38 @@ export class ChunkFormat "uVolumeChunkStrides", 4 + numChannelDimensions, ); - if (inVertexShader) { - builder.addVertexCode( - textureAccessHelper.getAccessor( - "readVolumeData", - "uVolumeChunkSampler", - this.dataType, - ), - ); - } else { - builder.addFragmentCode( - textureAccessHelper.getAccessor( - "readVolumeData", - "uVolumeChunkSampler", - this.dataType, - ), - ); - } + const textureSamplerCode = textureAccessHelper.getAccessor( + "readVolumeData", + "uVolumeChunkSampler", + this.dataType, + ); const shaderType = getShaderType(this.dataType); - let code = ` + let dataAccessCode = ` ${shaderType} getDataValueAt(highp ivec3 p`; for (let channelDim = 0; channelDim < numChannelDimensions; ++channelDim) { - code += `, highp int channelIndex${channelDim}`; + dataAccessCode += `, highp int channelIndex${channelDim}`; } - code += `) { + dataAccessCode += `) { highp ${textureVecType} offset = uVolumeChunkStrides[0] + p.x * uVolumeChunkStrides[1] + p.y * uVolumeChunkStrides[2] + p.z * uVolumeChunkStrides[3]; `; for (let channelDim = 0; channelDim < numChannelDimensions; ++channelDim) { - code += ` + dataAccessCode += ` offset += channelIndex${channelDim} * uVolumeChunkStrides[${4 + channelDim}]; `; } - code += ` + dataAccessCode += ` return readVolumeData(offset); } `; if (inVertexShader) { - builder.addVertexCode(code); + builder.addVertexCode(textureSamplerCode); + builder.addVertexCode(dataAccessCode); } else { - builder.addFragmentCode(code); + builder.addVertexCode(textureSamplerCode); + builder.addFragmentCode(dataAccessCode); } } From b29b50106f7a977b04e09d90fa4bafc312db3dab Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 20 May 2024 15:55:22 +0200 Subject: [PATCH 27/41] refactor: rename variable --- src/sliceview/volume/frontend.ts | 2 +- src/volume_rendering/volume_render_layer.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sliceview/volume/frontend.ts b/src/sliceview/volume/frontend.ts index f3a11f289..cb32e06ba 100644 --- a/src/sliceview/volume/frontend.ts +++ b/src/sliceview/volume/frontend.ts @@ -62,7 +62,7 @@ export interface ChunkFormat { * Called once per RenderLayer when starting to draw chunks, on the ChunkFormat of the first * source. This is not called before each source is drawn. */ - beginDrawing: (gl: GL, shader: ShaderProgram, reset?: boolean) => void; + beginDrawing: (gl: GL, shader: ShaderProgram, onlyActivateTexture?: boolean) => void; /** * Called once after all chunks have been drawn, on the ChunkFormat of the first source. diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index e4ef41d20..0a82c449e 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -1073,7 +1073,7 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); } shader.bind(); this.vertexIdHelper.enable(); - chunkFormat.beginDrawing(gl, shader, true); + chunkFormat.beginDrawing(gl, shader, true /* onlyActivateTexture */); chunkFormat.beginSource(gl, shader); } newSource = false; From d3b274b58606dba2cdb49c270bc7f026f5923e52 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 20 May 2024 16:02:16 +0200 Subject: [PATCH 28/41] refactor: clarify volume rendering code with histograms --- src/volume_rendering/volume_render_layer.ts | 86 ++++++++++----------- 1 file changed, 42 insertions(+), 44 deletions(-) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 0a82c449e..b272e783d 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -108,6 +108,12 @@ import { defineVertexId, VertexIdHelper } from "#src/webgl/vertex_id.js"; export const VOLUME_RENDERING_DEPTH_SAMPLES_DEFAULT_VALUE = 64; const VOLUME_RENDERING_DEPTH_SAMPLES_LOG_SCALE_ORIGIN = 1; const VOLUME_RENDERING_RESOLUTION_INDICATOR_BAR_HEIGHT = 10; +const HISTOGRAM_SAMPLES_PER_INSTANCE = 512; + +// Number of points to sample in computing the histogram. Increasing this increases the precision +// of the histogram but also slows down rendering. +const NUM_HISTOGRAM_SAMPLES = 4096; +const DEBUG_HISTOGRAMS = true; const depthSamplerTextureUnit = Symbol("depthSamplerTextureUnit"); @@ -171,12 +177,6 @@ function clampAndRoundResolutionTargetValue(value: number) { return clampToInterval(depthSamplesBounds, Math.round(value)) as number; } -const histogramSamplesPerInstance = 512; - -// Number of points to sample in computing the histogram. Increasing this increases the precision -// of the histogram but also slows down rendering. -const histogramSamples = 4096; - export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { gain: WatchableValueInterface; multiscaleSource: MultiscaleVolumeChunkSource; @@ -189,7 +189,7 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { mode: TrackableVolumeRenderingModeValue; backend: ChunkRenderLayerFrontend; private vertexIdHelper: VertexIdHelper; - dataHistogramSpecifications: HistogramSpecifications; + private dataHistogramSpecifications: HistogramSpecifications; private shaderGetter: ParameterizedContextDependentShaderGetter< { emitter: ShaderModule; chunkFormat: ChunkFormat; wireFrame: boolean }, @@ -219,7 +219,7 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { return this.dataHistogramSpecifications.visibleHistograms; } - private inputIndexBuffer: RefCountedValue; + private histogramIndexBuffer: RefCountedValue; constructor(options: VolumeRenderingRenderLayerOptions) { super(); @@ -234,11 +234,11 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { this.mode = options.mode; this.dataHistogramSpecifications = this.shaderControlState.histogramSpecifications; - this.inputIndexBuffer = this.registerDisposer( + this.histogramIndexBuffer = this.registerDisposer( getMemoizedBuffer( this.gl, WebGL2RenderingContext.ARRAY_BUFFER, - () => new Uint8Array(histogramSamplesPerInstance), + () => new Uint8Array(HISTOGRAM_SAMPLES_PER_INSTANCE), ), ); this.registerDisposer( @@ -494,7 +494,6 @@ void main() { }, }, ); - // TODO (SKM) - see volume/renderlayer.ts histogram code and follow the ideas this.histogramShaderGetter = parameterizedContextDependentShaderGetter( this, this.gl, @@ -733,6 +732,21 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); this.chunkManager.chunkQueueManager.frameNumberCounter.frameNumber, ); + const restoreFrameBuffer = () => { + if (isProjectionMode(this.mode.value)) { + gl.disable(WebGL2RenderingContext.BLEND); + if (renderContext.bindMaxProjectionBuffer !== undefined) { + renderContext.bindMaxProjectionBuffer(); + } else { + throw new Error( + "bindMaxProjectionBuffer is undefined in VolumeRenderingRenderLayer", + ); + } + } else { + renderContext.bindFramebuffer(); + } + }; + const endShader = () => { if (shader === null) return; if (prevChunkFormat !== null) { @@ -807,17 +821,7 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); gl.clearColor(0.0, 0.0, 0.0, 1.0); gl.clear(WebGL2RenderingContext.COLOR_BUFFER_BIT); } - if (isProjectionMode(this.mode.value)) { - if (renderContext.bindMaxProjectionBuffer !== undefined) { - renderContext.bindMaxProjectionBuffer(); - } else { - throw new Error( - "bindMaxProjectionBuffer is undefined in VolumeRenderingRenderLayer", - ); - } - } else { - renderContext.bindFramebuffer(); - } + restoreFrameBuffer(); } forEachVisibleVolumeRenderingChunk( @@ -858,10 +862,12 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); wireFrame: renderContext.wireFrame, }); shader = shaderResult.shader; - histogramShaderResult = this.histogramShaderGetter({ - chunkFormat: chunkFormat!, - }); - histogramShader = histogramShaderResult.shader; + if (needToDrawHistogram) { + histogramShaderResult = this.histogramShaderGetter({ + chunkFormat: chunkFormat!, + }); + histogramShader = histogramShaderResult.shader; + } if (shader !== null) { shader.bind(); if (chunkFormat !== null) { @@ -1027,7 +1033,7 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); ); gl.disable(WebGL2RenderingContext.DEPTH_TEST); gl.enable(WebGL2RenderingContext.BLEND); - this.inputIndexBuffer.value.bindToVertexAttrib( + this.histogramIndexBuffer.value.bindToVertexAttrib( histogramShader.attribute("aInput1"), 1, WebGL2RenderingContext.UNSIGNED_BYTE, @@ -1052,28 +1058,21 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); gl.drawArraysInstanced( WebGL2RenderingContext.POINTS, 0, - histogramSamplesPerInstance, - histogramSamples / histogramSamplesPerInstance, + HISTOGRAM_SAMPLES_PER_INSTANCE, + NUM_HISTOGRAM_SAMPLES / HISTOGRAM_SAMPLES_PER_INSTANCE, ); } // Reset the state back to regular drawing mode gl.enable(WebGL2RenderingContext.DEPTH_TEST); - if (isProjectionMode(this.mode.value)) { - gl.disable(WebGL2RenderingContext.BLEND); - if (renderContext.bindMaxProjectionBuffer !== undefined) { - renderContext.bindMaxProjectionBuffer(); - } else { - throw new Error( - "bindMaxProjectionBuffer is undefined in VolumeRenderingRenderLayer", - ); - } - } else { - renderContext.bindFramebuffer(); - } + restoreFrameBuffer(); shader.bind(); this.vertexIdHelper.enable(); - chunkFormat.beginDrawing(gl, shader, true /* onlyActivateTexture */); + chunkFormat.beginDrawing( + gl, + shader, + true /* onlyActivateTexture */, + ); chunkFormat.beginSource(gl, shader); } newSource = false; @@ -1086,7 +1085,6 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); gl.disable(WebGL2RenderingContext.CULL_FACE); endShader(); this.vertexIdHelper.disable(); - const DEBUG_HISTOGRAMS = true; if (needToDrawHistogram && DEBUG_HISTOGRAMS) { const outputBuffers = this.dataHistogramSpecifications.getFramebuffers(gl); @@ -1106,7 +1104,7 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); tempBuffer2[j] = tempBuffer[j * 4]; } console.log("histogram", tempBuffer2.join(" ")); - renderContext.bindFramebuffer(); + restoreFrameBuffer(); } } From 537be3a3dd789bd100565a316ac874cf68c704c5 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 20 May 2024 16:16:31 +0200 Subject: [PATCH 29/41] fix: change histogram debug default to false --- src/volume_rendering/volume_render_layer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index b272e783d..769b1190f 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -113,7 +113,7 @@ const HISTOGRAM_SAMPLES_PER_INSTANCE = 512; // Number of points to sample in computing the histogram. Increasing this increases the precision // of the histogram but also slows down rendering. const NUM_HISTOGRAM_SAMPLES = 4096; -const DEBUG_HISTOGRAMS = true; +const DEBUG_HISTOGRAMS = false; const depthSamplerTextureUnit = Symbol("depthSamplerTextureUnit"); From 706be232fab54c206597bc30bd612fe31a9aba48 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 20 May 2024 16:18:35 +0200 Subject: [PATCH 30/41] fix: correct fragment shader for regular chunk render --- src/sliceview/uncompressed_chunk_format.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sliceview/uncompressed_chunk_format.ts b/src/sliceview/uncompressed_chunk_format.ts index f9fb310a9..440e6c28d 100644 --- a/src/sliceview/uncompressed_chunk_format.ts +++ b/src/sliceview/uncompressed_chunk_format.ts @@ -187,7 +187,7 @@ ${shaderType} getDataValueAt(highp ivec3 p`; builder.addVertexCode(textureSamplerCode); builder.addVertexCode(dataAccessCode); } else { - builder.addVertexCode(textureSamplerCode); + builder.addFragmentCode(textureSamplerCode); builder.addFragmentCode(dataAccessCode); } } From e53325e88437d2ed7bef6441d31511cd70b499b9 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 20 May 2024 16:44:49 +0200 Subject: [PATCH 31/41] chore: add comment --- src/volume_rendering/volume_render_layer.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 769b1190f..331e9a201 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -112,6 +112,7 @@ const HISTOGRAM_SAMPLES_PER_INSTANCE = 512; // Number of points to sample in computing the histogram. Increasing this increases the precision // of the histogram but also slows down rendering. +// Here, we use 4096 samples per chunk to compute the histogram. const NUM_HISTOGRAM_SAMPLES = 4096; const DEBUG_HISTOGRAMS = false; From b7aac119e1e75baa8e068670d3d4c43912901250 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 20 May 2024 17:00:50 +0200 Subject: [PATCH 32/41] feat: allow render context to know camera move --- src/perspective_view/panel.ts | 1 + src/perspective_view/render_layer.ts | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/src/perspective_view/panel.ts b/src/perspective_view/panel.ts index 949c5f582..7bbf45a8a 100644 --- a/src/perspective_view/panel.ts +++ b/src/perspective_view/panel.ts @@ -924,6 +924,7 @@ export class PerspectivePanel extends RenderedDataPanel { alreadyEmittedPickID: false, bindFramebuffer, frameNumber: this.context.frameNumber, + cameraMovementInProgress: this.isCameraMoving, }; mat4.copy( diff --git a/src/perspective_view/render_layer.ts b/src/perspective_view/render_layer.ts index c758e79d4..a29d30903 100644 --- a/src/perspective_view/render_layer.ts +++ b/src/perspective_view/render_layer.ts @@ -55,6 +55,16 @@ export interface PerspectiveViewRenderContext * Specifies the ID of the depth frame buffer texture to query during rendering. */ depthBufferTexture?: WebGLTexture | null; + + /** + * Specifies if the camera is moving + */ + cameraMovementInProgress: boolean; + + /** + * Specifices how to bind the max projection buffer + */ + bindMaxProjectionBuffer?: () => void | undefined; } // eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging From 8b58e74f666ace46fcb31390bced995c1f04248a Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 21 May 2024 16:59:57 +0200 Subject: [PATCH 33/41] debug: add frame rate monitor to help compare performance --- src/util/framerate.ts | 94 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 src/util/framerate.ts diff --git a/src/util/framerate.ts b/src/util/framerate.ts new file mode 100644 index 000000000..dcb2e4ec8 --- /dev/null +++ b/src/util/framerate.ts @@ -0,0 +1,94 @@ +/** + * @license + * Copyright 2024 Google Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export class FramerateMonitor { + private timeElapsedQueries: (WebGLQuery | null)[] = []; + private warnedAboutMissingExtension = false; + + constructor(private queryPoolSize: number = 10) { + if (this.queryPoolSize < 1) { + throw new Error( + `Query pool size must be at least 1, but got ${queryPoolSize}.`, + ); + } + } + + getTimingExtension(gl: WebGL2RenderingContext) { + const ext = gl.getExtension("EXT_disjoint_timer_query_webgl2"); + if (ext === null && !this.warnedAboutMissingExtension) { + console.warn( + "EXT_disjoint_timer_query_webgl2 extension not available. " + + "Cannot measure frame time.", + ); + this.warnedAboutMissingExtension = true; + } + return ext; + } + + startFrameTimeQuery(gl: WebGL2RenderingContext, ext: any) { + if (ext === null) { + return null; + } + const query = gl.createQuery(); + if (query !== null) { + gl.beginQuery(ext.TIME_ELAPSED_EXT, query); + } + return query; + } + + endFrameTimeQuery( + gl: WebGL2RenderingContext, + ext: any, + query: WebGLQuery | null, + ) { + if (ext !== null && query !== null) { + gl.endQuery(ext.TIME_ELAPSED_EXT); + } + if (this.timeElapsedQueries.length >= this.queryPoolSize) { + const oldestQuery = this.timeElapsedQueries.shift(); + if (oldestQuery !== null) { + gl.deleteQuery(oldestQuery!); + } + } + this.timeElapsedQueries.push(query); + } + + getLastFrameTimesInMs( + gl: WebGL2RenderingContext, + numberOfFrames: number = 5, + ) { + const { timeElapsedQueries } = this; + const results: number[] = []; + for (let i = timeElapsedQueries.length - 1; i >= 0; i--) { + const timeElapsedQuery = timeElapsedQueries[i]; + if (timeElapsedQuery !== null) { + const available = gl.getQueryParameter( + timeElapsedQuery, + gl.QUERY_RESULT_AVAILABLE, + ); + if (available) { + const result = + gl.getQueryParameter(timeElapsedQuery, gl.QUERY_RESULT) / 1e6; + results.push(result); + } + } + if (results.length >= numberOfFrames) { + break; + } + } + return results; + } + } \ No newline at end of file From 645df6c8cc1121eca36b92ae59915aaf08c798ff Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 21 May 2024 17:16:34 +0200 Subject: [PATCH 34/41] debug: track time for VR render --- src/volume_rendering/volume_render_layer.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 4fc40fafb..69d56d360 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -54,6 +54,7 @@ import { registerNested, } from "#src/trackable_value.js"; import type { RefCountedValue } from "#src/util/disposable.js"; +import { FramerateMonitor } from "#src/util/framerate.js"; import { getFrustrumPlanes, mat4, vec3 } from "#src/util/geom.js"; import { clampToInterval } from "#src/util/lerp.js"; import { getObjectId } from "#src/util/object_id.js"; @@ -115,6 +116,7 @@ const HISTOGRAM_SAMPLES_PER_INSTANCE = 512; // Here, we use 4096 samples per chunk to compute the histogram. const NUM_HISTOGRAM_SAMPLES = 4096; const DEBUG_HISTOGRAMS = false; +const CHECK_PERFORMANCE = true; const depthSamplerTextureUnit = Symbol("depthSamplerTextureUnit"); @@ -221,6 +223,7 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { } private histogramIndexBuffer: RefCountedValue; + private framerateMonitor = new FramerateMonitor(); constructor(options: VolumeRenderingRenderLayerOptions) { super(); @@ -730,6 +733,13 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); const chunkDataDisplaySize = vec3.create(); const { gl } = this; + let query: WebGLQuery | null = null; + let ext: any = null; + const frameRateMonitor = this.framerateMonitor; + if (CHECK_PERFORMANCE) { + ext = frameRateMonitor.getTimingExtension(gl); + query = frameRateMonitor.startFrameTimeQuery(gl, ext); + } this.vertexIdHelper.enable(); const { chunkResolutionHistogram: renderScaleHistogram } = this; @@ -1116,6 +1126,10 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); console.log("histogram", tempBuffer2.join(" ")); restoreFrameBuffer(); } + if (CHECK_PERFORMANCE) { + frameRateMonitor.endFrameTimeQuery(gl, ext, query); + console.log(frameRateMonitor.getLastFrameTimesInMs(gl, ext)); + } } isReady( From c85a779308fd7dbd323b23a8378ce33f011b4c0a Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 21 May 2024 18:22:25 +0200 Subject: [PATCH 35/41] feat: faster histogram rendering by avoiding shader and buffer flippign --- src/volume_rendering/volume_render_layer.ts | 198 +++++++++++--------- 1 file changed, 105 insertions(+), 93 deletions(-) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 69d56d360..9f0a38adf 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -116,7 +116,7 @@ const HISTOGRAM_SAMPLES_PER_INSTANCE = 512; // Here, we use 4096 samples per chunk to compute the histogram. const NUM_HISTOGRAM_SAMPLES = 4096; const DEBUG_HISTOGRAMS = false; -const CHECK_PERFORMANCE = true; +const CHECK_PERFORMANCE = false; const depthSamplerTextureUnit = Symbol("depthSamplerTextureUnit"); @@ -719,16 +719,11 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); activeIndex: 0, }; let shader: ShaderProgram | null = null; - let histogramShader: ShaderProgram | null = null; let prevChunkFormat: ChunkFormat | undefined | null; let shaderResult: ParameterizedShaderGetterResult< ShaderControlsBuilderState, VolumeRenderingShaderParameters >; - let histogramShaderResult: ParameterizedShaderGetterResult< - ShaderControlsBuilderState, - VolumeRenderingShaderParameters - >; // Size of chunk (in voxels) in the "display" subspace of the chunk coordinate space. const chunkDataDisplaySize = vec3.create(); @@ -767,9 +762,6 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); shader.unbindTransferFunctionTextures(); if (prevChunkFormat !== null) { prevChunkFormat!.endDrawing(gl, shader); - if (histogramShader !== null) { - prevChunkFormat!.endDrawing(gl, histogramShader); - } } const depthTextureUnit = shader.textureUnit(depthSamplerTextureUnit); gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + depthTextureUnit); @@ -828,18 +820,7 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); gl.enable(WebGL2RenderingContext.CULL_FACE); gl.cullFace(WebGL2RenderingContext.FRONT); - if (needToDrawHistogram) { - const outputBuffers = - this.dataHistogramSpecifications.getFramebuffers(gl); - const count = this.getDataHistogramCount(); - for (let i = 0; i < count; ++i) { - outputBuffers[i].bind(256, 1); - gl.clearColor(0.0, 0.0, 0.0, 1.0); - gl.clear(WebGL2RenderingContext.COLOR_BUFFER_BIT); - } - restoreFrameBuffer(); - } - + const chunkInfoForHistogram: any[] = []; const pickId = isProjectionMode(this.mode.value) ? renderContext.pickIDs.register(this) : 0; @@ -881,12 +862,6 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); wireFrame: renderContext.wireFrame, }); shader = shaderResult.shader; - if (needToDrawHistogram) { - histogramShaderResult = this.histogramShaderGetter({ - chunkFormat: chunkFormat!, - }); - histogramShader = histogramShaderResult.shader; - } if (shader !== null) { shader.bind(); if (chunkFormat !== null) { @@ -1023,78 +998,22 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); channelToChunkDimensionIndices, newSource, ); - } - gl.uniform3fv(shader.uniform("uTranslation"), chunkPosition); - gl.uniform1ui(shader.uniform("uPickId"), pickId); - drawBoxes(gl, 1, 1); - - // Draw histograms if needed - if (histogramShader !== null && needToDrawHistogram) { - // Setup the state for drawing histograms - histogramShader.bind(); - const chunkFormat = transformedSource.source.chunkFormat; - const onlyActivateTexture = !newSource; - chunkFormat.beginDrawing(gl, histogramShader, onlyActivateTexture); - chunkFormat.beginSource(gl, histogramShader); - if (prevChunkFormat != null) { - prevChunkFormat.bindChunk( - gl, - histogramShader, + if (needToDrawHistogram) { + chunkInfoForHistogram.push({ chunk, fixedPositionWithinChunk, chunkDisplayDimensionIndices, channelToChunkDimensionIndices, newSource, - ); + chunkDataDisplaySize, + chunkFormat: prevChunkFormat, + }); } - gl.uniform3fv( - histogramShader.uniform("uChunkDataSize"), - chunkDataDisplaySize, - ); - gl.disable(WebGL2RenderingContext.DEPTH_TEST); - gl.enable(WebGL2RenderingContext.BLEND); - this.histogramIndexBuffer.value.bindToVertexAttrib( - histogramShader.attribute("aInput1"), - 1, - WebGL2RenderingContext.UNSIGNED_BYTE, - /*normalized=*/ true, - ); - const { dataType, dataHistogramSpecifications } = this; - const count = this.getDataHistogramCount(); - const outputFramebuffers = - dataHistogramSpecifications.getFramebuffers(gl); - const bounds = this.dataHistogramSpecifications.bounds.value; - - // Draw each histogram - for (let i = 0; i < count; ++i) { - outputFramebuffers[i].bind(256, 1); - enableLerpShaderFunction( - histogramShader, - `invlerpForHistogram${i}`, - dataType, - bounds[i], - ); - gl.uniform1i(histogramShader.uniform("uHistogramIndex"), i); - gl.drawArraysInstanced( - WebGL2RenderingContext.POINTS, - 0, - HISTOGRAM_SAMPLES_PER_INSTANCE, - NUM_HISTOGRAM_SAMPLES / HISTOGRAM_SAMPLES_PER_INSTANCE, - ); - } - - // Reset the state back to regular drawing mode - gl.enable(WebGL2RenderingContext.DEPTH_TEST); - restoreFrameBuffer(); - shader.bind(); - this.vertexIdHelper.enable(); - chunkFormat.beginDrawing( - gl, - shader, - true /* onlyActivateTexture */, - ); - chunkFormat.beginSource(gl, shader); } + gl.uniform3fv(shader.uniform("uTranslation"), chunkPosition); + gl.uniform1ui(shader.uniform("uPickId"), pickId); + drawBoxes(gl, 1, 1); + newSource = false; ++presentCount; } else { @@ -1105,6 +1024,99 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); gl.disable(WebGL2RenderingContext.CULL_FACE); endShader(); this.vertexIdHelper.disable(); + if (!needToDrawHistogram) { + if (CHECK_PERFORMANCE) { + frameRateMonitor.endFrameTimeQuery(gl, ext, query); + console.log(frameRateMonitor.getLastFrameTimesInMs(gl, ext)); + } + return; + } + let histogramShader: ShaderProgram | null = null; + let histogramShaderResult: ParameterizedShaderGetterResult< + ShaderControlsBuilderState, + VolumeRenderingShaderParameters + >; + const endHistogramShader = () => { + if (histogramShader === null) return; + histogramShader.unbindTransferFunctionTextures(); + if (prevChunkFormat !== null) { + prevChunkFormat!.endDrawing(gl, histogramShader); + } + }; + prevChunkFormat = null; + const { dataType, dataHistogramSpecifications } = this; + const outputFramebuffers = dataHistogramSpecifications.getFramebuffers(gl); + const count = this.getDataHistogramCount(); + for (let i = 0; i < count; ++i) { + outputFramebuffers[i].bind(256, 1); + gl.clearColor(0.0, 0.0, 0.0, 1.0); + gl.clear(WebGL2RenderingContext.COLOR_BUFFER_BIT); + } + const bounds = this.dataHistogramSpecifications.bounds.value; + for (let j = 0; j < presentCount; ++j) { + const fullInfo = chunkInfoForHistogram[j]; + const chunkFormat = fullInfo.chunkFormat; + if (chunkFormat !== prevChunkFormat) { + prevChunkFormat = chunkFormat; + endHistogramShader(); + histogramShaderResult = this.histogramShaderGetter({ + chunkFormat: chunkFormat!, + }); + histogramShader = histogramShaderResult.shader; + if (histogramShader !== null) { + if (chunkFormat !== null) { + chunkFormat.beginDrawing(gl, histogramShader); + chunkFormat.beginSource(gl, histogramShader); + } + histogramShader.bind(); + } else { + return; + } + } + if (histogramShader === null) return; + gl.uniform3fv( + histogramShader.uniform("uChunkDataSize"), + fullInfo.chunkDataDisplaySize, + ); + if (prevChunkFormat != null) { + prevChunkFormat.bindChunk( + gl, + histogramShader!, + fullInfo.chunk, + fullInfo.fixedPositionWithinChunk, + fullInfo.chunkDisplayDimensionIndices, + fullInfo.channelToChunkDimensionIndices, + fullInfo.newSource, + ); + } + gl.disable(WebGL2RenderingContext.DEPTH_TEST); + gl.enable(WebGL2RenderingContext.BLEND); + this.histogramIndexBuffer.value.bindToVertexAttrib( + histogramShader.attribute("aInput1"), + 1, + WebGL2RenderingContext.UNSIGNED_BYTE, + /*normalized=*/ true, + ); + + // Draw each histogram + for (let i = 0; i < count; ++i) { + outputFramebuffers[i].bind(256, 1); + enableLerpShaderFunction( + histogramShader, + `invlerpForHistogram${i}`, + dataType, + bounds[i], + ); + gl.uniform1i(histogramShader.uniform("uHistogramIndex"), i); + gl.drawArraysInstanced( + WebGL2RenderingContext.POINTS, + 0, + HISTOGRAM_SAMPLES_PER_INSTANCE, + NUM_HISTOGRAM_SAMPLES / HISTOGRAM_SAMPLES_PER_INSTANCE, + ); + } + } + if (needToDrawHistogram && DEBUG_HISTOGRAMS) { const outputBuffers = this.dataHistogramSpecifications.getFramebuffers(gl); @@ -1124,12 +1136,12 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); tempBuffer2[j] = tempBuffer[j * 4]; } console.log("histogram", tempBuffer2.join(" ")); - restoreFrameBuffer(); } if (CHECK_PERFORMANCE) { frameRateMonitor.endFrameTimeQuery(gl, ext, query); console.log(frameRateMonitor.getLastFrameTimesInMs(gl, ext)); } + restoreFrameBuffer(); } isReady( From 12989515098cd1ff05784d3a59e1aa1ccf1c793d Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 21 May 2024 18:24:19 +0200 Subject: [PATCH 36/41] feat: don't render 3d histogram if camera movement --- src/volume_rendering/volume_render_layer.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 9f0a38adf..f8cd83da6 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -815,7 +815,8 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); const needToDrawHistogram = this.getDataHistogramCount() > 0 && !renderContext.wireFrame && - !renderContext.sliceViewsPresent; + !renderContext.sliceViewsPresent && + !renderContext.cameraMovementInProgress; gl.enable(WebGL2RenderingContext.CULL_FACE); gl.cullFace(WebGL2RenderingContext.FRONT); From dc9276f653222cfbe4358b01661764eaecb23f77 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Wed, 22 May 2024 11:43:47 +0200 Subject: [PATCH 37/41] refactor: clarify chunk info storage for histograms --- src/volume_rendering/volume_render_layer.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index f8cd83da6..0e5f99071 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -156,6 +156,15 @@ interface VolumeRenderingShaderParameters { mode: VolumeRenderingModes; } +interface StoredChunkInfoForHistogram { + chunk: VolumeChunk; + fixedPositionWithinChunk: Uint32Array; + chunkDisplayDimensionIndices: number[]; + channelToChunkDimensionIndices: readonly number[]; + chunkDataDisplaySize: vec3; + chunkFormat: ChunkFormat; +} + const tempMat4 = mat4.create(); const tempVisibleVolumetricClippingPlanes = new Float32Array(24); @@ -821,7 +830,7 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); gl.enable(WebGL2RenderingContext.CULL_FACE); gl.cullFace(WebGL2RenderingContext.FRONT); - const chunkInfoForHistogram: any[] = []; + const chunkInfoForHistogram: StoredChunkInfoForHistogram[] = []; const pickId = isProjectionMode(this.mode.value) ? renderContext.pickIDs.register(this) : 0; @@ -1005,7 +1014,6 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); fixedPositionWithinChunk, chunkDisplayDimensionIndices, channelToChunkDimensionIndices, - newSource, chunkDataDisplaySize, chunkFormat: prevChunkFormat, }); @@ -1055,6 +1063,7 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); } const bounds = this.dataHistogramSpecifications.bounds.value; for (let j = 0; j < presentCount; ++j) { + newSource = true; const fullInfo = chunkInfoForHistogram[j]; const chunkFormat = fullInfo.chunkFormat; if (chunkFormat !== prevChunkFormat) { @@ -1087,7 +1096,7 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); fullInfo.fixedPositionWithinChunk, fullInfo.chunkDisplayDimensionIndices, fullInfo.channelToChunkDimensionIndices, - fullInfo.newSource, + newSource, ); } gl.disable(WebGL2RenderingContext.DEPTH_TEST); @@ -1116,6 +1125,7 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); NUM_HISTOGRAM_SAMPLES / HISTOGRAM_SAMPLES_PER_INSTANCE, ); } + newSource = false; } if (needToDrawHistogram && DEBUG_HISTOGRAMS) { From 3fc067dd02a45a791e139f35288a89db2da34091 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Wed, 22 May 2024 12:51:38 +0200 Subject: [PATCH 38/41] feat: variable samples for each histogram based on size and chunk amount --- src/volume_rendering/volume_render_layer.ts | 45 ++++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 0e5f99071..7e2deedfb 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -109,12 +109,12 @@ import { defineVertexId, VertexIdHelper } from "#src/webgl/vertex_id.js"; export const VOLUME_RENDERING_DEPTH_SAMPLES_DEFAULT_VALUE = 64; const VOLUME_RENDERING_DEPTH_SAMPLES_LOG_SCALE_ORIGIN = 1; const VOLUME_RENDERING_RESOLUTION_INDICATOR_BAR_HEIGHT = 10; -const HISTOGRAM_SAMPLES_PER_INSTANCE = 512; +const HISTOGRAM_SAMPLES_PER_INSTANCE = 256; // Number of points to sample in computing the histogram. Increasing this increases the precision // of the histogram but also slows down rendering. // Here, we use 4096 samples per chunk to compute the histogram. -const NUM_HISTOGRAM_SAMPLES = 4096; +const NUM_HISTOGRAM_SAMPLES = 2 ** 14; const DEBUG_HISTOGRAMS = false; const CHECK_PERFORMANCE = false; @@ -1040,6 +1040,7 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); } return; } + // Handle histogram drawing let histogramShader: ShaderProgram | null = null; let histogramShaderResult: ParameterizedShaderGetterResult< ShaderControlsBuilderState, @@ -1052,6 +1053,26 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); prevChunkFormat!.endDrawing(gl, histogramShader); } }; + const determineNumHistogramInstances = ( + chunkDataSize: vec3, + numHistograms: number, + ) => { + const maxSamplesInChunk = Math.ceil( + chunkDataSize.reduce((a, b) => a * b, 1) / 2.0, + ); + const totalDesiredSamplesInChunk = NUM_HISTOGRAM_SAMPLES / numHistograms; + const desiredSamples = Math.min( + maxSamplesInChunk, + totalDesiredSamplesInChunk, + ); + + // round to nearest multiple of NUM_HISTOGRAM_SAMPLES_PER_INSTANCE + return Math.max( + Math.round(desiredSamples / HISTOGRAM_SAMPLES_PER_INSTANCE), + 1, + ); + }; + prevChunkFormat = null; const { dataType, dataHistogramSpecifications } = this; const outputFramebuffers = dataHistogramSpecifications.getFramebuffers(gl); @@ -1064,8 +1085,8 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); const bounds = this.dataHistogramSpecifications.bounds.value; for (let j = 0; j < presentCount; ++j) { newSource = true; - const fullInfo = chunkInfoForHistogram[j]; - const chunkFormat = fullInfo.chunkFormat; + const chunkInfo = chunkInfoForHistogram[j]; + const chunkFormat = chunkInfo.chunkFormat; if (chunkFormat !== prevChunkFormat) { prevChunkFormat = chunkFormat; endHistogramShader(); @@ -1086,16 +1107,16 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); if (histogramShader === null) return; gl.uniform3fv( histogramShader.uniform("uChunkDataSize"), - fullInfo.chunkDataDisplaySize, + chunkInfo.chunkDataDisplaySize, ); if (prevChunkFormat != null) { prevChunkFormat.bindChunk( gl, histogramShader!, - fullInfo.chunk, - fullInfo.fixedPositionWithinChunk, - fullInfo.chunkDisplayDimensionIndices, - fullInfo.channelToChunkDimensionIndices, + chunkInfo.chunk, + chunkInfo.fixedPositionWithinChunk, + chunkInfo.chunkDisplayDimensionIndices, + chunkInfo.channelToChunkDimensionIndices, newSource, ); } @@ -1109,6 +1130,10 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); ); // Draw each histogram + const numInstances = determineNumHistogramInstances( + chunkInfo.chunkDataDisplaySize, + presentCount, + ); for (let i = 0; i < count; ++i) { outputFramebuffers[i].bind(256, 1); enableLerpShaderFunction( @@ -1122,7 +1147,7 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); WebGL2RenderingContext.POINTS, 0, HISTOGRAM_SAMPLES_PER_INSTANCE, - NUM_HISTOGRAM_SAMPLES / HISTOGRAM_SAMPLES_PER_INSTANCE, + numInstances, ); } newSource = false; From 7226cf0517739bb59b3912ba982ec0969f2960df Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 3 Jun 2024 13:04:47 +0200 Subject: [PATCH 39/41] refactor: clean up for review --- src/perspective_view/panel.ts | 2 +- src/sliceview/single_texture_chunk_format.ts | 6 +- src/sliceview/volume/frontend.ts | 2 +- src/util/framerate.ts | 94 ------ src/volume_rendering/volume_render_layer.ts | 290 +++++++++---------- 5 files changed, 145 insertions(+), 249 deletions(-) delete mode 100644 src/util/framerate.ts diff --git a/src/perspective_view/panel.ts b/src/perspective_view/panel.ts index 02041a368..ae852fcb3 100644 --- a/src/perspective_view/panel.ts +++ b/src/perspective_view/panel.ts @@ -81,7 +81,7 @@ import { MultipleScaleBarTextures } from "#src/widget/scale_bar.js"; import type { RPC } from "#src/worker_rpc.js"; import { SharedObject } from "#src/worker_rpc.js"; -const REDRAW_DELAY_AFTER_CAMERA_MOVE = 400; +const REDRAW_DELAY_AFTER_CAMERA_MOVE = 300; export interface PerspectiveViewerState extends RenderedDataViewerState { wireFrame: WatchableValueInterface; diff --git a/src/sliceview/single_texture_chunk_format.ts b/src/sliceview/single_texture_chunk_format.ts index 71180f04d..4fe03f273 100644 --- a/src/sliceview/single_texture_chunk_format.ts +++ b/src/sliceview/single_texture_chunk_format.ts @@ -56,12 +56,10 @@ export abstract class SingleTextureChunkFormat abstract get shaderSamplerType(): ShaderSamplerType; - beginDrawing(gl: GL, shader: ShaderProgram, onlyActivateTexture: boolean = false) { + beginDrawing(gl: GL, shader: ShaderProgram) { const textureUnit = shader.textureUnit(textureUnitSymbol); gl.activeTexture(WebGL2RenderingContext.TEXTURE0 + textureUnit); - if (!onlyActivateTexture) { - (shader)[textureLayoutSymbol] = null; - } + (shader)[textureLayoutSymbol] = null; } endDrawing(gl: GL, shader: ShaderProgram) { diff --git a/src/sliceview/volume/frontend.ts b/src/sliceview/volume/frontend.ts index cb32e06ba..dea825db8 100644 --- a/src/sliceview/volume/frontend.ts +++ b/src/sliceview/volume/frontend.ts @@ -62,7 +62,7 @@ export interface ChunkFormat { * Called once per RenderLayer when starting to draw chunks, on the ChunkFormat of the first * source. This is not called before each source is drawn. */ - beginDrawing: (gl: GL, shader: ShaderProgram, onlyActivateTexture?: boolean) => void; + beginDrawing: (gl: GL, shader: ShaderProgram) => void; /** * Called once after all chunks have been drawn, on the ChunkFormat of the first source. diff --git a/src/util/framerate.ts b/src/util/framerate.ts deleted file mode 100644 index dcb2e4ec8..000000000 --- a/src/util/framerate.ts +++ /dev/null @@ -1,94 +0,0 @@ -/** - * @license - * Copyright 2024 Google Inc. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -export class FramerateMonitor { - private timeElapsedQueries: (WebGLQuery | null)[] = []; - private warnedAboutMissingExtension = false; - - constructor(private queryPoolSize: number = 10) { - if (this.queryPoolSize < 1) { - throw new Error( - `Query pool size must be at least 1, but got ${queryPoolSize}.`, - ); - } - } - - getTimingExtension(gl: WebGL2RenderingContext) { - const ext = gl.getExtension("EXT_disjoint_timer_query_webgl2"); - if (ext === null && !this.warnedAboutMissingExtension) { - console.warn( - "EXT_disjoint_timer_query_webgl2 extension not available. " + - "Cannot measure frame time.", - ); - this.warnedAboutMissingExtension = true; - } - return ext; - } - - startFrameTimeQuery(gl: WebGL2RenderingContext, ext: any) { - if (ext === null) { - return null; - } - const query = gl.createQuery(); - if (query !== null) { - gl.beginQuery(ext.TIME_ELAPSED_EXT, query); - } - return query; - } - - endFrameTimeQuery( - gl: WebGL2RenderingContext, - ext: any, - query: WebGLQuery | null, - ) { - if (ext !== null && query !== null) { - gl.endQuery(ext.TIME_ELAPSED_EXT); - } - if (this.timeElapsedQueries.length >= this.queryPoolSize) { - const oldestQuery = this.timeElapsedQueries.shift(); - if (oldestQuery !== null) { - gl.deleteQuery(oldestQuery!); - } - } - this.timeElapsedQueries.push(query); - } - - getLastFrameTimesInMs( - gl: WebGL2RenderingContext, - numberOfFrames: number = 5, - ) { - const { timeElapsedQueries } = this; - const results: number[] = []; - for (let i = timeElapsedQueries.length - 1; i >= 0; i--) { - const timeElapsedQuery = timeElapsedQueries[i]; - if (timeElapsedQuery !== null) { - const available = gl.getQueryParameter( - timeElapsedQuery, - gl.QUERY_RESULT_AVAILABLE, - ); - if (available) { - const result = - gl.getQueryParameter(timeElapsedQuery, gl.QUERY_RESULT) / 1e6; - results.push(result); - } - } - if (results.length >= numberOfFrames) { - break; - } - } - return results; - } - } \ No newline at end of file diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index 7e2deedfb..c25f587a9 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -54,7 +54,6 @@ import { registerNested, } from "#src/trackable_value.js"; import type { RefCountedValue } from "#src/util/disposable.js"; -import { FramerateMonitor } from "#src/util/framerate.js"; import { getFrustrumPlanes, mat4, vec3 } from "#src/util/geom.js"; import { clampToInterval } from "#src/util/lerp.js"; import { getObjectId } from "#src/util/object_id.js"; @@ -116,7 +115,6 @@ const HISTOGRAM_SAMPLES_PER_INSTANCE = 256; // Here, we use 4096 samples per chunk to compute the histogram. const NUM_HISTOGRAM_SAMPLES = 2 ** 14; const DEBUG_HISTOGRAMS = false; -const CHECK_PERFORMANCE = false; const depthSamplerTextureUnit = Symbol("depthSamplerTextureUnit"); @@ -232,7 +230,6 @@ export class VolumeRenderingRenderLayer extends PerspectiveViewRenderLayer { } private histogramIndexBuffer: RefCountedValue; - private framerateMonitor = new FramerateMonitor(); constructor(options: VolumeRenderingRenderLayerOptions) { super(); @@ -536,7 +533,11 @@ vec3 chunkSamplePosition; `); const numChannelDimensions = shaderParametersState.numChannelDimensions; - chunkFormat.defineShader(builder, numChannelDimensions, true); + chunkFormat.defineShader( + builder, + numChannelDimensions, + true /*inVertexShader*/, + ); const { dataType } = chunkFormat; let dataAccessChannelParams = ""; let dataAccessChannelArgs = ""; @@ -579,7 +580,7 @@ switch (uHistogramIndex) {`; builder, invlerpName, dataType, - false, + false /*clamp*/, ), ); builder.addVertexCode(` @@ -598,11 +599,11 @@ case ${i}: `; builder.addVertexCode(glsl_simpleFloatHash); builder.setVertexMain(` -vec3 rand3 = vec3(simpleFloatHash(vec2(aInput1 + float(gl_VertexID), float(gl_InstanceID))), +vec3 rand3val = vec3(simpleFloatHash(vec2(aInput1 + float(gl_VertexID), float(gl_InstanceID))), simpleFloatHash(vec2(aInput1 + float(gl_VertexID) + 10.0, 5.0 + float(gl_InstanceID))), simpleFloatHash(vec2(aInput1 + float(gl_VertexID) + 20.0, 15.0 + float(gl_InstanceID))) ); -chunkSamplePosition = rand3 * (uChunkDataSize - 1.0); +chunkSamplePosition = rand3val * (uChunkDataSize - 1.0); ${histogramFetchCode} if (x == 0.0) { gl_Position = vec4(2.0, 2.0, 2.0, 1.0); @@ -737,13 +738,6 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); const chunkDataDisplaySize = vec3.create(); const { gl } = this; - let query: WebGLQuery | null = null; - let ext: any = null; - const frameRateMonitor = this.framerateMonitor; - if (CHECK_PERFORMANCE) { - ext = frameRateMonitor.getTimingExtension(gl); - query = frameRateMonitor.startFrameTimeQuery(gl, ext); - } this.vertexIdHelper.enable(); const { chunkResolutionHistogram: renderScaleHistogram } = this; @@ -751,7 +745,7 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); this.chunkManager.chunkQueueManager.frameNumberCounter.frameNumber, ); - const restoreFrameBuffer = () => { + const restoreDrawingBuffers = () => { if (isProjectionMode(this.mode.value)) { gl.disable(WebGL2RenderingContext.BLEND); if (renderContext.bindMaxProjectionBuffer !== undefined) { @@ -1033,151 +1027,149 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); gl.disable(WebGL2RenderingContext.CULL_FACE); endShader(); this.vertexIdHelper.disable(); - if (!needToDrawHistogram) { - if (CHECK_PERFORMANCE) { - frameRateMonitor.endFrameTimeQuery(gl, ext, query); - console.log(frameRateMonitor.getLastFrameTimesInMs(gl, ext)); - } - return; - } - // Handle histogram drawing - let histogramShader: ShaderProgram | null = null; - let histogramShaderResult: ParameterizedShaderGetterResult< - ShaderControlsBuilderState, - VolumeRenderingShaderParameters - >; - const endHistogramShader = () => { - if (histogramShader === null) return; - histogramShader.unbindTransferFunctionTextures(); - if (prevChunkFormat !== null) { - prevChunkFormat!.endDrawing(gl, histogramShader); - } - }; - const determineNumHistogramInstances = ( - chunkDataSize: vec3, - numHistograms: number, - ) => { - const maxSamplesInChunk = Math.ceil( - chunkDataSize.reduce((a, b) => a * b, 1) / 2.0, - ); - const totalDesiredSamplesInChunk = NUM_HISTOGRAM_SAMPLES / numHistograms; - const desiredSamples = Math.min( - maxSamplesInChunk, - totalDesiredSamplesInChunk, - ); - - // round to nearest multiple of NUM_HISTOGRAM_SAMPLES_PER_INSTANCE - return Math.max( - Math.round(desiredSamples / HISTOGRAM_SAMPLES_PER_INSTANCE), - 1, - ); - }; - prevChunkFormat = null; - const { dataType, dataHistogramSpecifications } = this; - const outputFramebuffers = dataHistogramSpecifications.getFramebuffers(gl); - const count = this.getDataHistogramCount(); - for (let i = 0; i < count; ++i) { - outputFramebuffers[i].bind(256, 1); - gl.clearColor(0.0, 0.0, 0.0, 1.0); - gl.clear(WebGL2RenderingContext.COLOR_BUFFER_BIT); - } - const bounds = this.dataHistogramSpecifications.bounds.value; - for (let j = 0; j < presentCount; ++j) { - newSource = true; - const chunkInfo = chunkInfoForHistogram[j]; - const chunkFormat = chunkInfo.chunkFormat; - if (chunkFormat !== prevChunkFormat) { - prevChunkFormat = chunkFormat; - endHistogramShader(); - histogramShaderResult = this.histogramShaderGetter({ - chunkFormat: chunkFormat!, - }); - histogramShader = histogramShaderResult.shader; - if (histogramShader !== null) { - if (chunkFormat !== null) { - chunkFormat.beginDrawing(gl, histogramShader); - chunkFormat.beginSource(gl, histogramShader); - } - histogramShader.bind(); - } else { - return; + if (needToDrawHistogram) { + let histogramShader: ShaderProgram | null = null; + let histogramShaderResult: ParameterizedShaderGetterResult< + ShaderControlsBuilderState, + VolumeRenderingShaderParameters + >; + const endHistogramShader = () => { + if (histogramShader === null) return; + histogramShader.unbindTransferFunctionTextures(); + if (prevChunkFormat !== null) { + prevChunkFormat!.endDrawing(gl, histogramShader); } - } - if (histogramShader === null) return; - gl.uniform3fv( - histogramShader.uniform("uChunkDataSize"), - chunkInfo.chunkDataDisplaySize, - ); - if (prevChunkFormat != null) { - prevChunkFormat.bindChunk( - gl, - histogramShader!, - chunkInfo.chunk, - chunkInfo.fixedPositionWithinChunk, - chunkInfo.chunkDisplayDimensionIndices, - chunkInfo.channelToChunkDimensionIndices, - newSource, + }; + const determineNumHistogramInstances = ( + chunkDataSize: vec3, + numHistograms: number, + ) => { + const maxSamplesInChunk = Math.ceil( + chunkDataSize.reduce((a, b) => a * b, 1) / 2.0, ); + const totalDesiredSamplesInChunk = + NUM_HISTOGRAM_SAMPLES / numHistograms; + const desiredSamples = Math.min( + maxSamplesInChunk, + totalDesiredSamplesInChunk, + ); + + // round to nearest multiple of NUM_HISTOGRAM_SAMPLES_PER_INSTANCE + return Math.max( + Math.round(desiredSamples / HISTOGRAM_SAMPLES_PER_INSTANCE), + 1, + ); + }; + + prevChunkFormat = null; + const { dataType, dataHistogramSpecifications } = this; + const histogramFramebuffers = + dataHistogramSpecifications.getFramebuffers(gl); + const numHistograms = this.getDataHistogramCount(); + for (let i = 0; i < numHistograms; ++i) { + histogramFramebuffers[i].bind(256, 1); + gl.clearColor(0.0, 0.0, 0.0, 1.0); + gl.clear(WebGL2RenderingContext.COLOR_BUFFER_BIT); } - gl.disable(WebGL2RenderingContext.DEPTH_TEST); + const bounds = this.dataHistogramSpecifications.bounds.value; + // Blending on to accumulate histograms. gl.enable(WebGL2RenderingContext.BLEND); - this.histogramIndexBuffer.value.bindToVertexAttrib( - histogramShader.attribute("aInput1"), - 1, - WebGL2RenderingContext.UNSIGNED_BYTE, - /*normalized=*/ true, - ); - - // Draw each histogram - const numInstances = determineNumHistogramInstances( - chunkInfo.chunkDataDisplaySize, - presentCount, - ); - for (let i = 0; i < count; ++i) { - outputFramebuffers[i].bind(256, 1); - enableLerpShaderFunction( - histogramShader, - `invlerpForHistogram${i}`, - dataType, - bounds[i], + gl.disable(WebGL2RenderingContext.DEPTH_TEST); + for (let j = 0; j < presentCount; ++j) { + newSource = true; + const chunkInfo = chunkInfoForHistogram[j]; + const chunkFormat = chunkInfo.chunkFormat; + if (chunkFormat !== prevChunkFormat) { + prevChunkFormat = chunkFormat; + endHistogramShader(); + histogramShaderResult = this.histogramShaderGetter({ + chunkFormat: chunkFormat!, + }); + histogramShader = histogramShaderResult.shader; + console.log(histogramShader); + if (histogramShader !== null) { + if (chunkFormat !== null) { + chunkFormat.beginDrawing(gl, histogramShader); + chunkFormat.beginSource(gl, histogramShader); + } + histogramShader.bind(); + } else { + break; + } + } + if (histogramShader === null) break; + gl.uniform3fv( + histogramShader.uniform("uChunkDataSize"), + chunkInfo.chunkDataDisplaySize, + ); + if (prevChunkFormat != null) { + prevChunkFormat.bindChunk( + gl, + histogramShader, + chunkInfo.chunk, + chunkInfo.fixedPositionWithinChunk, + chunkInfo.chunkDisplayDimensionIndices, + chunkInfo.channelToChunkDimensionIndices, + newSource, + ); + } + this.histogramIndexBuffer.value.bindToVertexAttrib( + histogramShader.attribute("aInput1"), + 1, + WebGL2RenderingContext.UNSIGNED_BYTE, + /*normalized=*/ true, ); - gl.uniform1i(histogramShader.uniform("uHistogramIndex"), i); - gl.drawArraysInstanced( - WebGL2RenderingContext.POINTS, - 0, - HISTOGRAM_SAMPLES_PER_INSTANCE, - numInstances, + + // Draw each histogram + const numInstances = determineNumHistogramInstances( + chunkInfo.chunkDataDisplaySize, + presentCount, ); + for (let i = 0; i < numHistograms; ++i) { + histogramFramebuffers[i].bind(256, 1); + enableLerpShaderFunction( + histogramShader, + `invlerpForHistogram${i}`, + dataType, + bounds[i], + ); + gl.uniform1i(histogramShader.uniform("uHistogramIndex"), i); + gl.drawArraysInstanced( + WebGL2RenderingContext.POINTS, + 0, + HISTOGRAM_SAMPLES_PER_INSTANCE, + numInstances, + ); + } + newSource = false; } - newSource = false; - } - if (needToDrawHistogram && DEBUG_HISTOGRAMS) { - const outputBuffers = - this.dataHistogramSpecifications.getFramebuffers(gl); - outputBuffers[0].bind(256, 1); - const tempBuffer = new Float32Array(256 * 4); - gl.readPixels( - 0, - 0, - 256, - 1, - WebGL2RenderingContext.RGBA, - WebGL2RenderingContext.FLOAT, - tempBuffer, - ); - const tempBuffer2 = new Float32Array(256); - for (let j = 0; j < 256; ++j) { - tempBuffer2[j] = tempBuffer[j * 4]; + if (needToDrawHistogram && DEBUG_HISTOGRAMS) { + const histogramFrameBuffers = + this.dataHistogramSpecifications.getFramebuffers(gl); + for (let i = 0; i < numHistograms; ++i) { + histogramFrameBuffers[i].bind(256, 1); + const tempBuffer = new Float32Array(256 * 4); + gl.readPixels( + 0, + 0, + 256, + 1, + WebGL2RenderingContext.RGBA, + WebGL2RenderingContext.FLOAT, + tempBuffer, + ); + const tempBuffer2 = new Float32Array(256); + for (let j = 0; j < 256; ++j) { + tempBuffer2[j] = tempBuffer[j * 4]; + } + console.log("histogram%d", i, tempBuffer2.join(" ")); + } } - console.log("histogram", tempBuffer2.join(" ")); - } - if (CHECK_PERFORMANCE) { - frameRateMonitor.endFrameTimeQuery(gl, ext, query); - console.log(frameRateMonitor.getLastFrameTimesInMs(gl, ext)); + endHistogramShader(); + restoreDrawingBuffers(); } - restoreFrameBuffer(); } isReady( From 3b7c241d05b2537a6a9128649eb62dcc9d789f52 Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 3 Jun 2024 13:09:37 +0200 Subject: [PATCH 40/41] refactor: improve vertex shader tabbing --- src/volume_rendering/volume_render_layer.ts | 45 ++++++++++----------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index c25f587a9..e886fc732 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -569,8 +569,8 @@ ${getShaderType(dataType)} getDataValue() { return getDataValue(0); } shaderParametersState.dataHistogramChannelSpecifications; const numHistograms = dataHistogramChannelSpecifications.length; let histogramFetchCode = ` -float x; -switch (uHistogramIndex) {`; + float x; + switch (uHistogramIndex) {`; for (let i = 0; i < numHistograms; ++i) { const { channel } = dataHistogramChannelSpecifications[i]; const getDataValueExpr = `getDataValue(${channel.join(",")})`; @@ -589,33 +589,31 @@ float getHistogramValue${i}() { } `); histogramFetchCode += ` -case ${i}: - x = getHistogramValue${i}(); - break; -`; + case ${i}: + x = getHistogramValue${i}(); + break;`; } histogramFetchCode += ` -} + } `; builder.addVertexCode(glsl_simpleFloatHash); builder.setVertexMain(` -vec3 rand3val = vec3(simpleFloatHash(vec2(aInput1 + float(gl_VertexID), float(gl_InstanceID))), - simpleFloatHash(vec2(aInput1 + float(gl_VertexID) + 10.0, 5.0 + float(gl_InstanceID))), - simpleFloatHash(vec2(aInput1 + float(gl_VertexID) + 20.0, 15.0 + float(gl_InstanceID))) - ); -chunkSamplePosition = rand3val * (uChunkDataSize - 1.0); + vec3 rand3val = vec3( + simpleFloatHash(vec2(aInput1 + float(gl_VertexID), float(gl_InstanceID))), + simpleFloatHash(vec2(aInput1 + float(gl_VertexID) + 10.0, 5.0 + float(gl_InstanceID))), + simpleFloatHash(vec2(aInput1 + float(gl_VertexID) + 20.0, 15.0 + float(gl_InstanceID)))); + chunkSamplePosition = rand3val * (uChunkDataSize - 1.0); ${histogramFetchCode} -if (x == 0.0) { - gl_Position = vec4(2.0, 2.0, 2.0, 1.0); -} -else { - if (x < 0.0) x = 0.0; - else if (x > 1.0) x = 1.0; - else x = (1.0 + x * 253.0) / 255.0; - gl_Position = vec4(2.0 * (x * 255.0 + 0.5) / 256.0 - 1.0, 0.0, 0.0, 1.0); -} -gl_PointSize = 1.0; - `); + if (x == 0.0) { + gl_Position = vec4(2.0, 2.0, 2.0, 1.0); + } + else { + if (x < 0.0) x = 0.0; + else if (x > 1.0) x = 1.0; + else x = (1.0 + x * 253.0) / 255.0; + gl_Position = vec4(2.0 * (x * 255.0 + 0.5) / 256.0 - 1.0, 0.0, 0.0, 1.0); + } + gl_PointSize = 1.0;`); builder.setFragmentMain(` outputValue = vec4(1.0, 1.0, 1.0, 1.0); `); @@ -1087,7 +1085,6 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); chunkFormat: chunkFormat!, }); histogramShader = histogramShaderResult.shader; - console.log(histogramShader); if (histogramShader !== null) { if (chunkFormat !== null) { chunkFormat.beginDrawing(gl, histogramShader); From 2ebc60f0d406daaa33d9f89ba7145a24f2167a1b Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Mon, 3 Jun 2024 13:40:21 +0200 Subject: [PATCH 41/41] fix: re-enable depth test on restore buffers --- src/volume_rendering/volume_render_layer.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/volume_rendering/volume_render_layer.ts b/src/volume_rendering/volume_render_layer.ts index e886fc732..6c7f40d45 100644 --- a/src/volume_rendering/volume_render_layer.ts +++ b/src/volume_rendering/volume_render_layer.ts @@ -756,6 +756,7 @@ outputValue = vec4(1.0, 1.0, 1.0, 1.0); } else { renderContext.bindFramebuffer(); } + gl.enable(WebGL2RenderingContext.DEPTH_TEST); }; const endShader = () => {