From 0a154a3b21fae5f666561e087f79f01526e46e5e Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 11 Feb 2024 11:51:38 -0500 Subject: [PATCH 1/7] In webgl drawer, fall back to canvas drawer for tiled images with tainted data --- src/tiledimage.js | 20 +++ src/webgldrawer.js | 335 ++++++++++++++++++++++++++------------------- 2 files changed, 213 insertions(+), 142 deletions(-) diff --git a/src/tiledimage.js b/src/tiledimage.js index a1c3054b..84b852d9 100644 --- a/src/tiledimage.js +++ b/src/tiledimage.js @@ -166,6 +166,7 @@ $.TiledImage = function( options ) { _lastDrawn: [], // array of tiles that were last fetched by the drawer _isBlending: false, // Are any tiles still being blended? _wasBlending: false, // Were any tiles blending before the last draw? + _isTainted: false, // Has a Tile been found with tainted data? //configurable settings springStiffness: $.DEFAULT_SETTINGS.springStiffness, animationTime: $.DEFAULT_SETTINGS.animationTime, @@ -326,6 +327,25 @@ $.extend($.TiledImage.prototype, $.EventSource.prototype, /** @lends OpenSeadrag return this._needsDraw; }, + /** + * Set the internal _isTainted flag for this TiledImage. Lazy loaded - not + * checked each time a Tile is loaded, but can be set if a consumer of the + * tiles (e.g. a Drawer) discovers a Tile to have tainted data so that further + * checks are not needed and alternative rendering strategies can be used. + * @private + */ + setTainted(isTainted){ + this._isTainted = isTainted; + }, + + /** + * @private + * @returns {Boolean} whether the TiledImage has been marked as tainted + */ + isTainted(){ + return this._isTainted; + }, + /** * Destroy the TiledImage (unload current loaded tiles). */ diff --git a/src/webgldrawer.js b/src/webgldrawer.js index bf40266d..3cd46ffe 100644 --- a/src/webgldrawer.js +++ b/src/webgldrawer.js @@ -90,6 +90,7 @@ this._clippingCanvas = null; this._clippingContext = null; this._renderingCanvas = null; + this._backupCanvasDrawer = null; // Add listeners for events that require modifying the scene or camera this.viewer.addHandler("tile-ready", ev => this._tileReadyHandler(ev)); @@ -206,6 +207,25 @@ return canvas; } + /** + * Get the backup renderer (CanvasDrawer) to use if data cannot be used by webgl + * Lazy loaded + * @private + * @returns {CanvasDrawer} + */ + _getBackupCanvasDrawer(){ + if(!this._backupCanvasDrawer){ + this._backupCanvasDrawer = new $.CanvasDrawer({ + viewer: this.viewer, + viewport: this.viewport, + element: this.container, + }); + this._backupCanvasDrawer.canvas.style.setProperty('visibility', 'hidden'); + } + + return this._backupCanvasDrawer; + } + /** * * @param {Array} tiledImages Array of TiledImage objects to draw @@ -237,168 +257,187 @@ //iterate over tiled images and draw each one using a two-pass rendering pipeline if needed tiledImages.forEach( (tiledImage, tiledImageIndex) => { - let tilesToDraw = tiledImage.getTilesToDraw(); - - if(tilesToDraw.length === 0 || tiledImage.getOpacity() === 0){ - return; - } - let firstTile = tilesToDraw[0]; - - let useContext2dPipeline = ( tiledImage.compositeOperation || - this.viewer.compositeOperation || - tiledImage._clip || - tiledImage._croppingPolygons || - tiledImage.debugMode - ); - - let useTwoPassRendering = useContext2dPipeline || (tiledImage.opacity < 1) || firstTile.hasTransparency; - - // using the context2d pipeline requires a clean rendering (back) buffer to start - if(useContext2dPipeline){ - // if the rendering buffer has image data currently, write it to the output canvas now and clear it - + if(tiledImage.isTainted()){ + // first, draw any data left in the rendering buffer onto the output canvas if(renderingBufferHasImageData){ this._outputContext.drawImage(this._renderingCanvas, 0, 0); + // clear the buffer + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer + renderingBufferHasImageData = false; } - // clear the buffer - gl.bindFramebuffer(gl.FRAMEBUFFER, null); - gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer - } + // next, use the backup canvas drawer to draw this tainted image + const canvasDrawer = this._getBackupCanvasDrawer(); + canvasDrawer.draw([tiledImage]); + this._outputContext.drawImage(canvasDrawer.canvas, 0, 0); - // First rendering pass: compose tiles that make up this tiledImage - gl.useProgram(this._firstPass.shaderProgram); - - // bind to the framebuffer for render-to-texture if using two-pass rendering, otherwise back buffer (null) - if(useTwoPassRendering){ - gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer); - // clear the buffer to draw a new image - gl.clear(gl.COLOR_BUFFER_BIT); } else { - gl.bindFramebuffer(gl.FRAMEBUFFER, null); - // no need to clear, just draw on top of the existing pixels - } + let tilesToDraw = tiledImage.getTilesToDraw(); - let overallMatrix = viewMatrix; - - let imageRotation = tiledImage.getRotation(true); - // if needed, handle the tiledImage being rotated - if( imageRotation % 360 !== 0){ - let imageRotationMatrix = $.Mat3.makeRotation(-imageRotation * Math.PI / 180); - let imageCenter = tiledImage.getBoundsNoRotate(true).getCenter(); - let t1 = $.Mat3.makeTranslation(imageCenter.x, imageCenter.y); - let t2 = $.Mat3.makeTranslation(-imageCenter.x, -imageCenter.y); - - // update the view matrix to account for this image's rotation - let localMatrix = t1.multiply(imageRotationMatrix).multiply(t2); - overallMatrix = viewMatrix.multiply(localMatrix); - } - - let maxTextures = this._gl.getParameter(this._gl.MAX_TEXTURE_IMAGE_UNITS); - if(maxTextures <= 0){ - // This can apparently happen on some systems if too many WebGL contexts have been created - // in which case maxTextures can be null, leading to out of bounds errors with the array. - // For example, when viewers were created and not destroyed in the test suite, this error - // occured in the TravisCI tests, though it did not happen when testing locally either in - // a browser or on the command line via grunt test. - - throw(new Error(`WegGL error: bad value for gl parameter MAX_TEXTURE_IMAGE_UNITS (${maxTextures}). This could happen - if too many contexts have been created and not released, or there is another problem with the graphics card.`)); - } - - let texturePositionArray = new Float32Array(maxTextures * 12); // 6 vertices (2 triangles) x 2 coordinates per vertex - let textureDataArray = new Array(maxTextures); - let matrixArray = new Array(maxTextures); - let opacityArray = new Array(maxTextures); - - // iterate over tiles and add data for each one to the buffers - for(let tileIndex = 0; tileIndex < tilesToDraw.length; tileIndex++){ - let tile = tilesToDraw[tileIndex].tile; - let indexInDrawArray = tileIndex % maxTextures; - let numTilesToDraw = indexInDrawArray + 1; - let tileContext = tile.getCanvasContext(); - - let textureInfo = tileContext ? this._TextureMap.get(tileContext.canvas) : null; - if(textureInfo){ - this._getTileData(tile, tiledImage, textureInfo, overallMatrix, indexInDrawArray, texturePositionArray, textureDataArray, matrixArray, opacityArray); - } else { - // console.log('No tile info', tile); + if(tilesToDraw.length === 0 || tiledImage.getOpacity() === 0){ + return; } - if( (numTilesToDraw === maxTextures) || (tileIndex === tilesToDraw.length - 1)){ - // We've filled up the buffers: time to draw this set of tiles + let firstTile = tilesToDraw[0]; - // bind each tile's texture to the appropriate gl.TEXTURE# - for(let i = 0; i <= numTilesToDraw; i++){ - gl.activeTexture(gl.TEXTURE0 + i); - gl.bindTexture(gl.TEXTURE_2D, textureDataArray[i]); + let useContext2dPipeline = ( tiledImage.compositeOperation || + this.viewer.compositeOperation || + tiledImage._clip || + tiledImage._croppingPolygons || + tiledImage.debugMode + ); + + let useTwoPassRendering = useContext2dPipeline || (tiledImage.opacity < 1) || firstTile.hasTransparency; + + // using the context2d pipeline requires a clean rendering (back) buffer to start + if(useContext2dPipeline){ + // if the rendering buffer has image data currently, write it to the output canvas now and clear it + + if(renderingBufferHasImageData){ + this._outputContext.drawImage(this._renderingCanvas, 0, 0); } - // set the buffer data for the texture coordinates to use for each tile - gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition); - gl.bufferData(gl.ARRAY_BUFFER, texturePositionArray, gl.DYNAMIC_DRAW); + // clear the buffer + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer + } - // set the transform matrix uniform for each tile - matrixArray.forEach( (matrix, index) => { - gl.uniformMatrix3fv(this._firstPass.uTransformMatrices[index], false, matrix); - }); - // set the opacity uniform for each tile - gl.uniform1fv(this._firstPass.uOpacities, new Float32Array(opacityArray)); + // First rendering pass: compose tiles that make up this tiledImage + gl.useProgram(this._firstPass.shaderProgram); - // bind vertex buffers and (re)set attributes before calling gl.drawArrays() - gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferOutputPosition); + // bind to the framebuffer for render-to-texture if using two-pass rendering, otherwise back buffer (null) + if(useTwoPassRendering){ + gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer); + // clear the buffer to draw a new image + gl.clear(gl.COLOR_BUFFER_BIT); + } else { + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + // no need to clear, just draw on top of the existing pixels + } + + let overallMatrix = viewMatrix; + + let imageRotation = tiledImage.getRotation(true); + // if needed, handle the tiledImage being rotated + if( imageRotation % 360 !== 0){ + let imageRotationMatrix = $.Mat3.makeRotation(-imageRotation * Math.PI / 180); + let imageCenter = tiledImage.getBoundsNoRotate(true).getCenter(); + let t1 = $.Mat3.makeTranslation(imageCenter.x, imageCenter.y); + let t2 = $.Mat3.makeTranslation(-imageCenter.x, -imageCenter.y); + + // update the view matrix to account for this image's rotation + let localMatrix = t1.multiply(imageRotationMatrix).multiply(t2); + overallMatrix = viewMatrix.multiply(localMatrix); + } + + let maxTextures = this._gl.getParameter(this._gl.MAX_TEXTURE_IMAGE_UNITS); + if(maxTextures <= 0){ + // This can apparently happen on some systems if too many WebGL contexts have been created + // in which case maxTextures can be null, leading to out of bounds errors with the array. + // For example, when viewers were created and not destroyed in the test suite, this error + // occured in the TravisCI tests, though it did not happen when testing locally either in + // a browser or on the command line via grunt test. + + throw(new Error(`WegGL error: bad value for gl parameter MAX_TEXTURE_IMAGE_UNITS (${maxTextures}). This could happen + if too many contexts have been created and not released, or there is another problem with the graphics card.`)); + } + + let texturePositionArray = new Float32Array(maxTextures * 12); // 6 vertices (2 triangles) x 2 coordinates per vertex + let textureDataArray = new Array(maxTextures); + let matrixArray = new Array(maxTextures); + let opacityArray = new Array(maxTextures); + + // iterate over tiles and add data for each one to the buffers + for(let tileIndex = 0; tileIndex < tilesToDraw.length; tileIndex++){ + let tile = tilesToDraw[tileIndex].tile; + let indexInDrawArray = tileIndex % maxTextures; + let numTilesToDraw = indexInDrawArray + 1; + let tileContext = tile.getCanvasContext(); + + let textureInfo = tileContext ? this._TextureMap.get(tileContext.canvas) : null; + if(textureInfo){ + this._getTileData(tile, tiledImage, textureInfo, overallMatrix, indexInDrawArray, texturePositionArray, textureDataArray, matrixArray, opacityArray); + } else { + // console.log('No tile info', tile); + } + if( (numTilesToDraw === maxTextures) || (tileIndex === tilesToDraw.length - 1)){ + // We've filled up the buffers: time to draw this set of tiles + + // bind each tile's texture to the appropriate gl.TEXTURE# + for(let i = 0; i <= numTilesToDraw; i++){ + gl.activeTexture(gl.TEXTURE0 + i); + gl.bindTexture(gl.TEXTURE_2D, textureDataArray[i]); + } + + // set the buffer data for the texture coordinates to use for each tile + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition); + gl.bufferData(gl.ARRAY_BUFFER, texturePositionArray, gl.DYNAMIC_DRAW); + + // set the transform matrix uniform for each tile + matrixArray.forEach( (matrix, index) => { + gl.uniformMatrix3fv(this._firstPass.uTransformMatrices[index], false, matrix); + }); + // set the opacity uniform for each tile + gl.uniform1fv(this._firstPass.uOpacities, new Float32Array(opacityArray)); + + // bind vertex buffers and (re)set attributes before calling gl.drawArrays() + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferOutputPosition); + gl.vertexAttribPointer(this._firstPass.aOutputPosition, 2, gl.FLOAT, false, 0, 0); + + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition); + gl.vertexAttribPointer(this._firstPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0); + + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferIndex); + gl.vertexAttribPointer(this._firstPass.aIndex, 1, gl.FLOAT, false, 0, 0); + + // Draw! 6 vertices per tile (2 triangles per rectangle) + gl.drawArrays(gl.TRIANGLES, 0, 6 * numTilesToDraw ); + } + } + + if(useTwoPassRendering){ + // Second rendering pass: Render the tiled image from the framebuffer into the back buffer + gl.useProgram(this._secondPass.shaderProgram); + + // set the rendering target to the back buffer (null) + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + + // bind the rendered texture from the first pass to use during this second pass + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, this._renderToTexture); + + // set opacity to the value for the current tiledImage + this._gl.uniform1f(this._secondPass.uOpacityMultiplier, tiledImage.opacity); + + // bind buffers and set attributes before calling gl.drawArrays + gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferTexturePosition); + gl.vertexAttribPointer(this._secondPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0); + gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferOutputPosition); gl.vertexAttribPointer(this._firstPass.aOutputPosition, 2, gl.FLOAT, false, 0, 0); - gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition); - gl.vertexAttribPointer(this._firstPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0); + // Draw the quad (two triangles) + gl.drawArrays(gl.TRIANGLES, 0, 6); - gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferIndex); - gl.vertexAttribPointer(this._firstPass.aIndex, 1, gl.FLOAT, false, 0, 0); + } - // Draw! 6 vertices per tile (2 triangles per rectangle) - gl.drawArrays(gl.TRIANGLES, 0, 6 * numTilesToDraw ); + renderingBufferHasImageData = true; + + if(useContext2dPipeline){ + // draw from the rendering canvas onto the output canvas, clipping/cropping if needed. + this._applyContext2dPipeline(tiledImage, tilesToDraw, tiledImageIndex); + renderingBufferHasImageData = false; + // clear the buffer + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer + } + + // after drawing the first TiledImage, fire the tiled-image-drawn event (for testing) + if(tiledImageIndex === 0){ + this._raiseTiledImageDrawnEvent(tiledImage, tilesToDraw.map(info=>info.tile)); } } - if(useTwoPassRendering){ - // Second rendering pass: Render the tiled image from the framebuffer into the back buffer - gl.useProgram(this._secondPass.shaderProgram); - // set the rendering target to the back buffer (null) - gl.bindFramebuffer(gl.FRAMEBUFFER, null); - - // bind the rendered texture from the first pass to use during this second pass - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, this._renderToTexture); - - // set opacity to the value for the current tiledImage - this._gl.uniform1f(this._secondPass.uOpacityMultiplier, tiledImage.opacity); - - // bind buffers and set attributes before calling gl.drawArrays - gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferTexturePosition); - gl.vertexAttribPointer(this._secondPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0); - gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferOutputPosition); - gl.vertexAttribPointer(this._firstPass.aOutputPosition, 2, gl.FLOAT, false, 0, 0); - - // Draw the quad (two triangles) - gl.drawArrays(gl.TRIANGLES, 0, 6); - - } - - renderingBufferHasImageData = true; - - if(useContext2dPipeline){ - // draw from the rendering canvas onto the output canvas, clipping/cropping if needed. - this._applyContext2dPipeline(tiledImage, tilesToDraw, tiledImageIndex); - renderingBufferHasImageData = false; - // clear the buffer - gl.bindFramebuffer(gl.FRAMEBUFFER, null); - gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer - } - - // after drawing the first TiledImage, fire the tiled-image-drawn event (for testing) - if(tiledImageIndex === 0){ - this._raiseTiledImageDrawnEvent(tiledImage, tilesToDraw.map(info=>info.tile)); - } }); @@ -802,7 +841,19 @@ let tile = event.tile; let tiledImage = event.tiledImage; let tileContext = tile.getCanvasContext(); - let canvas = tileContext.canvas; + let canvas = tileContext && tileContext.canvas; + // if the tile doesn't provide a canvas, or is tainted by cross-origin + // data, marked the TiledImage as tainted so the canvas drawer can be + // used instead, and return immediately - data cannot be uploaded to webgl + if(!canvas || $.isCanvasTainted(canvas)){ + const wasTainted = tiledImage.isTainted(); + if(!wasTainted){ + tiledImage.setTainted(true); + $.console.warn('WebGL cannot be used to draw this TiledImage because it has tainted data. Does crossOriginPolicy need to be set?'); + } + return; + } + let textureInfo = this._TextureMap.get(canvas); // if this is a new image for us, create a texture From 8967e2bb034e62518d9c5d14d24d9849a4fec041 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 12 Feb 2024 09:30:26 -0500 Subject: [PATCH 2/7] support hot-swapping drawers with viewer.setDrawer() --- src/canvasdrawer.js | 1 + src/viewer.js | 81 ++++++++++++++++++++++++++++++--------------- src/webgldrawer.js | 28 +++++++++++++++- 3 files changed, 83 insertions(+), 27 deletions(-) diff --git a/src/canvasdrawer.js b/src/canvasdrawer.js index d4ec6231..b848c3ef 100644 --- a/src/canvasdrawer.js +++ b/src/canvasdrawer.js @@ -139,6 +139,7 @@ class CanvasDrawer extends OpenSeadragon.DrawerBase{ this.canvas.height = 1; this.sketchCanvas = null; this.sketchContext = null; + this.container.removeChild(this.canvas); } /** diff --git a/src/viewer.js b/src/viewer.js index 6a8865c8..c3c10938 100644 --- a/src/viewer.js +++ b/src/viewer.js @@ -455,35 +455,13 @@ $.Viewer = function( options ) { this.drawer = null; - for (let i = 0; i < drawerCandidates.length; i++) { - - let drawerCandidate = drawerCandidates[i]; - let Drawer = null; - - //if inherits from a drawer base, use it - if (drawerCandidate && drawerCandidate.prototype instanceof $.DrawerBase) { - Drawer = drawerCandidate; - drawerCandidate = 'custom'; - } else if (typeof drawerCandidate === "string") { - Drawer = $.determineDrawer(drawerCandidate); - } else { - $.console.warn('Unsupported drawer! Drawer must be an existing string type, or a class that extends OpenSeadragon.DrawerBase.'); - continue; - } - - // if the drawer is supported, create it and break the loop - if (Drawer && Drawer.isSupported()) { - this.drawer = new Drawer({ - viewer: this, - viewport: this.viewport, - element: this.canvas, - debugGridColor: this.debugGridColor, - options: this.drawerOptions[drawerCandidate], - }); - + for (const drawerCandidate of drawerCandidates){ + let success = this.setDrawer(drawerCandidate, false); + if(success){ break; } } + if (!this.drawer){ $.console.error('No drawer could be created!'); throw('Error with creating the selected drawer(s)'); @@ -950,6 +928,57 @@ $.extend( $.Viewer.prototype, $.EventSource.prototype, $.ControlDock.prototype, this.removeAllHandlers(); }, + /** + * Set the drawer for this viewer, as a supported string or drawer constructor. + * @param {String | OpenSeadragon.DrawerBase} drawerCandidate The type of drawer to try to construct + * @param { Boolean } [redrawImmediately] Whether to immediately draw a new frame. Default = true. + * @param { Object } [drawerOptions] Options for this drawer. If falsey, defaults to viewer.drawerOptions + * for this viewer type. See {@link OpenSeadragon.Options}. + * @returns {Boolean} whether the drawer was created successfully + */ + setDrawer(drawerCandidate, redrawImmediately = true, drawerOptions = null){ + const oldDrawer = this.drawer; + + let Drawer = null; + + //if inherits from a drawer base, use it + if (drawerCandidate && drawerCandidate.prototype instanceof $.DrawerBase) { + Drawer = drawerCandidate; + drawerCandidate = 'custom'; + } else if (typeof drawerCandidate === "string") { + Drawer = $.determineDrawer(drawerCandidate); + } + + if(!Drawer){ + $.console.warn('Unsupported drawer! Drawer must be an existing string type, or a class that extends OpenSeadragon.DrawerBase.'); + } + + // if the drawer is supported, create it and return true + if (Drawer && Drawer.isSupported()) { + + // first destroy the previous drawer + if(oldDrawer){ + oldDrawer.destroy(); + } + + // create the new drawer + this.drawer = new Drawer({ + viewer: this, + viewport: this.viewport, + element: this.canvas, + debugGridColor: this.debugGridColor, + options: drawerOptions || this.drawerOptions[drawerCandidate], + }); + + if(redrawImmediately){ + this.forceRedraw(); + } + return true; + } + + return false; + }, + /** * @function * @returns {Boolean} diff --git a/src/webgldrawer.js b/src/webgldrawer.js index 3cd46ffe..e5b8f811 100644 --- a/src/webgldrawer.js +++ b/src/webgldrawer.js @@ -158,6 +158,16 @@ // set our webgl context reference to null to enable garbage collection this._gl = null; + if(this._backupCanvasDrawer){ + this._backupCanvasDrawer.destroy(); + this._backupCanvasDrawer = null; + } + + this.container.removeChild(this.canvas); + if(this.viewer.drawer === this){ + this.viewer.drawer = null; + } + // set our destroyed flag to true this._destroyed = true; } @@ -355,6 +365,15 @@ let tileContext = tile.getCanvasContext(); let textureInfo = tileContext ? this._TextureMap.get(tileContext.canvas) : null; + if(!textureInfo){ + // tile was not processed in the tile-ready event (this can happen + // if this drawer was created after the tile was downloaded) + this._tileReadyHandler({tile: tile, tiledImage: tiledImage}); + + // retry getting textureInfo + textureInfo = tileContext ? this._TextureMap.get(tileContext.canvas) : null; + } + if(textureInfo){ this._getTileData(tile, tiledImage, textureInfo, overallMatrix, indexInDrawArray, texturePositionArray, textureDataArray, matrixArray, opacityArray); } else { @@ -840,11 +859,18 @@ _tileReadyHandler(event){ let tile = event.tile; let tiledImage = event.tiledImage; + + // If a tiledImage is already known to be tainted, don't try to upload any + // textures to webgl, because they won't be used even if it succeeds + if(tiledImage.isTainted()){ + return; + } + let tileContext = tile.getCanvasContext(); let canvas = tileContext && tileContext.canvas; // if the tile doesn't provide a canvas, or is tainted by cross-origin // data, marked the TiledImage as tainted so the canvas drawer can be - // used instead, and return immediately - data cannot be uploaded to webgl + // used instead, and return immediately - tainted data cannot be uploaded to webgl if(!canvas || $.isCanvasTainted(canvas)){ const wasTainted = tiledImage.isTainted(); if(!wasTainted){ From 6e4914ada33c697242d765bb0c8fb21b0049f22c Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 12 Feb 2024 09:32:17 -0500 Subject: [PATCH 3/7] better cleanup for html drawer in destroy --- src/htmldrawer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/htmldrawer.js b/src/htmldrawer.js index 3d04f444..80f851e4 100644 --- a/src/htmldrawer.js +++ b/src/htmldrawer.js @@ -126,7 +126,7 @@ class HTMLDrawer extends OpenSeadragon.DrawerBase{ * Destroy the drawer (unload current loaded tiles) */ destroy() { - this.canvas.innerHTML = ""; + this.container.removeChild(this.canvas); } /** From 02898cfc173e076b1153ebb954ec8f7dc8a611da Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 12 Feb 2024 18:05:33 -0500 Subject: [PATCH 4/7] clean up bound event handlers in webgldrawer.destroy --- src/webgldrawer.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/webgldrawer.js b/src/webgldrawer.js index e5b8f811..b747779e 100644 --- a/src/webgldrawer.js +++ b/src/webgldrawer.js @@ -93,8 +93,10 @@ this._backupCanvasDrawer = null; // Add listeners for events that require modifying the scene or camera - this.viewer.addHandler("tile-ready", ev => this._tileReadyHandler(ev)); - this.viewer.addHandler("image-unloaded", ev => this._imageUnloadedHandler(ev)); + this._boundToTileReady = ev => this._tileReadyHandler(ev); + this._boundToImageUnloaded = ev => this._imageUnloadedHandler(ev); + this.viewer.addHandler("tile-ready", this._boundToTileReady); + this.viewer.addHandler("image-unloaded", this._boundToImageUnloaded); // Reject listening for the tile-drawing and tile-drawn events, which this drawer does not fire this.viewer.rejectEventHandler("tile-drawn", "The WebGLDrawer does not raise the tile-drawn event"); @@ -155,6 +157,10 @@ ext.loseContext(); } + // unbind our event listeners from the viewer + this.viewer.removeHandler("tile-ready", this._boundToTileReady); + this.viewer.removeHandler("image-unloaded", this._boundToImageUnloaded); + // set our webgl context reference to null to enable garbage collection this._gl = null; From 99c35aa3cb5ef825e2661e4fbfe860e023c512e0 Mon Sep 17 00:00:00 2001 From: Tom Date: Wed, 21 Feb 2024 16:01:18 -0500 Subject: [PATCH 5/7] consolidate drawer creation into viewer api. add drawer-error event --- src/drawerbase.js | 31 +++++++++++++++++++++++++++++++ src/viewer.js | 29 +++++++++++++++++------------ src/webgldrawer.js | 7 ++----- test/demo/drawercomparison.js | 2 +- 4 files changed, 51 insertions(+), 18 deletions(-) diff --git a/src/drawerbase.js b/src/drawerbase.js index 29d7a3b4..0a0d04ce 100644 --- a/src/drawerbase.js +++ b/src/drawerbase.js @@ -280,6 +280,37 @@ OpenSeadragon.DrawerBase = class DrawerBase{ }); } + /** + * Called by implementations to fire the drawer-error event + * @private + */ + _raiseDrawerErrorEvent(tiledImage, errorMessage){ + if(!this.viewer) { + return; + } + + /** + * Raised when a tiled image is drawn to the canvas. Used internally for testing. + * The update-viewport event is preferred if you want to know when a frame has been drawn. + * + * @event drawer-error + * @memberof OpenSeadragon.Viewer + * @type {object} + * @property {OpenSeadragon.Viewer} eventSource - A reference to the Viewer which raised the event. + * @property {OpenSeadragon.TiledImage} tiledImage - Which TiledImage is being drawn. + * @property {OpenSeadragon.DrawerBase} drawer - The drawer that raised the error. + * @property {String} error - A message describing the error. + * @property {?Object} userData - Arbitrary subscriber-defined object. + * @private + */ + this.viewer.raiseEvent( 'drawer-error', { + tiledImage: tiledImage, + drawer: this, + error: errorMessage, + }); + } + + }; }( OpenSeadragon )); diff --git a/src/viewer.js b/src/viewer.js index c3c10938..c2168c1b 100644 --- a/src/viewer.js +++ b/src/viewer.js @@ -456,7 +456,7 @@ $.Viewer = function( options ) { this.drawer = null; for (const drawerCandidate of drawerCandidates){ - let success = this.setDrawer(drawerCandidate, false); + let success = this.requestDrawer(drawerCandidate, true, false); if(success){ break; } @@ -929,19 +929,20 @@ $.extend( $.Viewer.prototype, $.EventSource.prototype, $.ControlDock.prototype, }, /** - * Set the drawer for this viewer, as a supported string or drawer constructor. - * @param {String | OpenSeadragon.DrawerBase} drawerCandidate The type of drawer to try to construct - * @param { Boolean } [redrawImmediately] Whether to immediately draw a new frame. Default = true. + * Request a drawer for this viewer, as a supported string or drawer constructor. + * @param {String | OpenSeadragon.DrawerBase} drawerCandidate The type of drawer to try to construct. + * @param { Boolean } [mainDrawer] Whether to use this as the viewer's main drawer. Default = true. + * @param { Boolean } [redrawImmediately] Whether to immediately draw a new frame. Only used if mainDrawer = true. Default = true. * @param { Object } [drawerOptions] Options for this drawer. If falsey, defaults to viewer.drawerOptions * for this viewer type. See {@link OpenSeadragon.Options}. - * @returns {Boolean} whether the drawer was created successfully + * @returns {Object | Boolean} The drawer that was created, or false if the requestd drawer is not supported */ - setDrawer(drawerCandidate, redrawImmediately = true, drawerOptions = null){ + requestDrawer(drawerCandidate, mainDrawer = true, redrawImmediately = true, drawerOptions = null){ const oldDrawer = this.drawer; let Drawer = null; - //if inherits from a drawer base, use it + //if the candidate inherits from a drawer base, use it if (drawerCandidate && drawerCandidate.prototype instanceof $.DrawerBase) { Drawer = drawerCandidate; drawerCandidate = 'custom'; @@ -957,12 +958,12 @@ $.extend( $.Viewer.prototype, $.EventSource.prototype, $.ControlDock.prototype, if (Drawer && Drawer.isSupported()) { // first destroy the previous drawer - if(oldDrawer){ + if(oldDrawer && mainDrawer){ oldDrawer.destroy(); } // create the new drawer - this.drawer = new Drawer({ + const newDrawer = new Drawer({ viewer: this, viewport: this.viewport, element: this.canvas, @@ -970,10 +971,14 @@ $.extend( $.Viewer.prototype, $.EventSource.prototype, $.ControlDock.prototype, options: drawerOptions || this.drawerOptions[drawerCandidate], }); - if(redrawImmediately){ - this.forceRedraw(); + if(mainDrawer){ + this.drawer = newDrawer; + if(redrawImmediately){ + this.forceRedraw(); + } } - return true; + + return newDrawer; } return false; diff --git a/src/webgldrawer.js b/src/webgldrawer.js index b747779e..af3ff198 100644 --- a/src/webgldrawer.js +++ b/src/webgldrawer.js @@ -231,11 +231,7 @@ */ _getBackupCanvasDrawer(){ if(!this._backupCanvasDrawer){ - this._backupCanvasDrawer = new $.CanvasDrawer({ - viewer: this.viewer, - viewport: this.viewport, - element: this.container, - }); + this._backupCanvasDrawer = this.viewer.requestDrawer('canvas', false); this._backupCanvasDrawer.canvas.style.setProperty('visibility', 'hidden'); } @@ -882,6 +878,7 @@ if(!wasTainted){ tiledImage.setTainted(true); $.console.warn('WebGL cannot be used to draw this TiledImage because it has tainted data. Does crossOriginPolicy need to be set?'); + this._raiseDrawerErrorEvent(tiledImage, 'Tainted data cannot be used by the WebGLDrawer. Falling back to CanvasDrawer for this TiledImage.'); } return; } diff --git a/test/demo/drawercomparison.js b/test/demo/drawercomparison.js index 96793915..76c780a4 100644 --- a/test/demo/drawercomparison.js +++ b/test/demo/drawercomparison.js @@ -49,7 +49,7 @@ let viewer2 = window.viewer2 = OpenSeadragon({ minZoomImageRatio:0.01, maxZoomPixelRatio:100, smoothTileEdgesMinZoom:1.1, - crossOriginPolicy: 'Anonymous', + // crossOriginPolicy: 'Anonymous', ajaxWithCredentials: false, // maxImageCacheCount: 30, drawer:drawer2, From 1442e5d4e495de4fb85f5152b34f6d45e335e7d4 Mon Sep 17 00:00:00 2001 From: Tom Date: Wed, 21 Feb 2024 16:02:29 -0500 Subject: [PATCH 6/7] revert change in drawercomparison.js used for testing --- test/demo/drawercomparison.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/demo/drawercomparison.js b/test/demo/drawercomparison.js index 76c780a4..96793915 100644 --- a/test/demo/drawercomparison.js +++ b/test/demo/drawercomparison.js @@ -49,7 +49,7 @@ let viewer2 = window.viewer2 = OpenSeadragon({ minZoomImageRatio:0.01, maxZoomPixelRatio:100, smoothTileEdgesMinZoom:1.1, - // crossOriginPolicy: 'Anonymous', + crossOriginPolicy: 'Anonymous', ajaxWithCredentials: false, // maxImageCacheCount: 30, drawer:drawer2, From b6501a3786b3d48c5b628bacb552ac9b4f6c0968 Mon Sep 17 00:00:00 2001 From: Tom Date: Thu, 22 Feb 2024 13:30:05 -0500 Subject: [PATCH 7/7] fix typo in docs. change to options object for requestDrawer API --- src/viewer.js | 23 +++++++++++++++++------ src/webgldrawer.js | 2 +- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/viewer.js b/src/viewer.js index c2168c1b..037e3016 100644 --- a/src/viewer.js +++ b/src/viewer.js @@ -456,7 +456,7 @@ $.Viewer = function( options ) { this.drawer = null; for (const drawerCandidate of drawerCandidates){ - let success = this.requestDrawer(drawerCandidate, true, false); + let success = this.requestDrawer(drawerCandidate, {mainDrawer: true, redrawImmediately: false}); if(success){ break; } @@ -931,13 +931,24 @@ $.extend( $.Viewer.prototype, $.EventSource.prototype, $.ControlDock.prototype, /** * Request a drawer for this viewer, as a supported string or drawer constructor. * @param {String | OpenSeadragon.DrawerBase} drawerCandidate The type of drawer to try to construct. - * @param { Boolean } [mainDrawer] Whether to use this as the viewer's main drawer. Default = true. - * @param { Boolean } [redrawImmediately] Whether to immediately draw a new frame. Only used if mainDrawer = true. Default = true. - * @param { Object } [drawerOptions] Options for this drawer. If falsey, defaults to viewer.drawerOptions + * @param { Object } options + * @param { Boolean } [options.mainDrawer] Whether to use this as the viewer's main drawer. Default = true. + * @param { Boolean } [options.redrawImmediately] Whether to immediately draw a new frame. Only used if options.mainDrawer = true. Default = true. + * @param { Object } [options.drawerOptions] Options for this drawer. Defaults to viewer.drawerOptions. * for this viewer type. See {@link OpenSeadragon.Options}. - * @returns {Object | Boolean} The drawer that was created, or false if the requestd drawer is not supported + * @returns {Object | Boolean} The drawer that was created, or false if the requested drawer is not supported */ - requestDrawer(drawerCandidate, mainDrawer = true, redrawImmediately = true, drawerOptions = null){ + requestDrawer(drawerCandidate, options){ + const defaultOpts = { + mainDrawer: true, + redrawImmediately: true, + drawerOptions: null + }; + options = $.extend(true, defaultOpts, options); + const mainDrawer = options.mainDrawer; + const redrawImmediately = options.redrawImmediately; + const drawerOptions = options.drawerOptions; + const oldDrawer = this.drawer; let Drawer = null; diff --git a/src/webgldrawer.js b/src/webgldrawer.js index e7a23fc9..d236a944 100644 --- a/src/webgldrawer.js +++ b/src/webgldrawer.js @@ -231,7 +231,7 @@ */ _getBackupCanvasDrawer(){ if(!this._backupCanvasDrawer){ - this._backupCanvasDrawer = this.viewer.requestDrawer('canvas', false); + this._backupCanvasDrawer = this.viewer.requestDrawer('canvas', {mainDrawer: false}); this._backupCanvasDrawer.canvas.style.setProperty('visibility', 'hidden'); }