From 0a154a3b21fae5f666561e087f79f01526e46e5e Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 11 Feb 2024 11:51:38 -0500 Subject: [PATCH] In webgl drawer, fall back to canvas drawer for tiled images with tainted data --- src/tiledimage.js | 20 +++ src/webgldrawer.js | 335 ++++++++++++++++++++++++++------------------- 2 files changed, 213 insertions(+), 142 deletions(-) diff --git a/src/tiledimage.js b/src/tiledimage.js index a1c3054b..84b852d9 100644 --- a/src/tiledimage.js +++ b/src/tiledimage.js @@ -166,6 +166,7 @@ $.TiledImage = function( options ) { _lastDrawn: [], // array of tiles that were last fetched by the drawer _isBlending: false, // Are any tiles still being blended? _wasBlending: false, // Were any tiles blending before the last draw? + _isTainted: false, // Has a Tile been found with tainted data? //configurable settings springStiffness: $.DEFAULT_SETTINGS.springStiffness, animationTime: $.DEFAULT_SETTINGS.animationTime, @@ -326,6 +327,25 @@ $.extend($.TiledImage.prototype, $.EventSource.prototype, /** @lends OpenSeadrag return this._needsDraw; }, + /** + * Set the internal _isTainted flag for this TiledImage. Lazy loaded - not + * checked each time a Tile is loaded, but can be set if a consumer of the + * tiles (e.g. a Drawer) discovers a Tile to have tainted data so that further + * checks are not needed and alternative rendering strategies can be used. + * @private + */ + setTainted(isTainted){ + this._isTainted = isTainted; + }, + + /** + * @private + * @returns {Boolean} whether the TiledImage has been marked as tainted + */ + isTainted(){ + return this._isTainted; + }, + /** * Destroy the TiledImage (unload current loaded tiles). */ diff --git a/src/webgldrawer.js b/src/webgldrawer.js index bf40266d..3cd46ffe 100644 --- a/src/webgldrawer.js +++ b/src/webgldrawer.js @@ -90,6 +90,7 @@ this._clippingCanvas = null; this._clippingContext = null; this._renderingCanvas = null; + this._backupCanvasDrawer = null; // Add listeners for events that require modifying the scene or camera this.viewer.addHandler("tile-ready", ev => this._tileReadyHandler(ev)); @@ -206,6 +207,25 @@ return canvas; } + /** + * Get the backup renderer (CanvasDrawer) to use if data cannot be used by webgl + * Lazy loaded + * @private + * @returns {CanvasDrawer} + */ + _getBackupCanvasDrawer(){ + if(!this._backupCanvasDrawer){ + this._backupCanvasDrawer = new $.CanvasDrawer({ + viewer: this.viewer, + viewport: this.viewport, + element: this.container, + }); + this._backupCanvasDrawer.canvas.style.setProperty('visibility', 'hidden'); + } + + return this._backupCanvasDrawer; + } + /** * * @param {Array} tiledImages Array of TiledImage objects to draw @@ -237,168 +257,187 @@ //iterate over tiled images and draw each one using a two-pass rendering pipeline if needed tiledImages.forEach( (tiledImage, tiledImageIndex) => { - let tilesToDraw = tiledImage.getTilesToDraw(); - - if(tilesToDraw.length === 0 || tiledImage.getOpacity() === 0){ - return; - } - let firstTile = tilesToDraw[0]; - - let useContext2dPipeline = ( tiledImage.compositeOperation || - this.viewer.compositeOperation || - tiledImage._clip || - tiledImage._croppingPolygons || - tiledImage.debugMode - ); - - let useTwoPassRendering = useContext2dPipeline || (tiledImage.opacity < 1) || firstTile.hasTransparency; - - // using the context2d pipeline requires a clean rendering (back) buffer to start - if(useContext2dPipeline){ - // if the rendering buffer has image data currently, write it to the output canvas now and clear it - + if(tiledImage.isTainted()){ + // first, draw any data left in the rendering buffer onto the output canvas if(renderingBufferHasImageData){ this._outputContext.drawImage(this._renderingCanvas, 0, 0); + // clear the buffer + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer + renderingBufferHasImageData = false; } - // clear the buffer - gl.bindFramebuffer(gl.FRAMEBUFFER, null); - gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer - } + // next, use the backup canvas drawer to draw this tainted image + const canvasDrawer = this._getBackupCanvasDrawer(); + canvasDrawer.draw([tiledImage]); + this._outputContext.drawImage(canvasDrawer.canvas, 0, 0); - // First rendering pass: compose tiles that make up this tiledImage - gl.useProgram(this._firstPass.shaderProgram); - - // bind to the framebuffer for render-to-texture if using two-pass rendering, otherwise back buffer (null) - if(useTwoPassRendering){ - gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer); - // clear the buffer to draw a new image - gl.clear(gl.COLOR_BUFFER_BIT); } else { - gl.bindFramebuffer(gl.FRAMEBUFFER, null); - // no need to clear, just draw on top of the existing pixels - } + let tilesToDraw = tiledImage.getTilesToDraw(); - let overallMatrix = viewMatrix; - - let imageRotation = tiledImage.getRotation(true); - // if needed, handle the tiledImage being rotated - if( imageRotation % 360 !== 0){ - let imageRotationMatrix = $.Mat3.makeRotation(-imageRotation * Math.PI / 180); - let imageCenter = tiledImage.getBoundsNoRotate(true).getCenter(); - let t1 = $.Mat3.makeTranslation(imageCenter.x, imageCenter.y); - let t2 = $.Mat3.makeTranslation(-imageCenter.x, -imageCenter.y); - - // update the view matrix to account for this image's rotation - let localMatrix = t1.multiply(imageRotationMatrix).multiply(t2); - overallMatrix = viewMatrix.multiply(localMatrix); - } - - let maxTextures = this._gl.getParameter(this._gl.MAX_TEXTURE_IMAGE_UNITS); - if(maxTextures <= 0){ - // This can apparently happen on some systems if too many WebGL contexts have been created - // in which case maxTextures can be null, leading to out of bounds errors with the array. - // For example, when viewers were created and not destroyed in the test suite, this error - // occured in the TravisCI tests, though it did not happen when testing locally either in - // a browser or on the command line via grunt test. - - throw(new Error(`WegGL error: bad value for gl parameter MAX_TEXTURE_IMAGE_UNITS (${maxTextures}). This could happen - if too many contexts have been created and not released, or there is another problem with the graphics card.`)); - } - - let texturePositionArray = new Float32Array(maxTextures * 12); // 6 vertices (2 triangles) x 2 coordinates per vertex - let textureDataArray = new Array(maxTextures); - let matrixArray = new Array(maxTextures); - let opacityArray = new Array(maxTextures); - - // iterate over tiles and add data for each one to the buffers - for(let tileIndex = 0; tileIndex < tilesToDraw.length; tileIndex++){ - let tile = tilesToDraw[tileIndex].tile; - let indexInDrawArray = tileIndex % maxTextures; - let numTilesToDraw = indexInDrawArray + 1; - let tileContext = tile.getCanvasContext(); - - let textureInfo = tileContext ? this._TextureMap.get(tileContext.canvas) : null; - if(textureInfo){ - this._getTileData(tile, tiledImage, textureInfo, overallMatrix, indexInDrawArray, texturePositionArray, textureDataArray, matrixArray, opacityArray); - } else { - // console.log('No tile info', tile); + if(tilesToDraw.length === 0 || tiledImage.getOpacity() === 0){ + return; } - if( (numTilesToDraw === maxTextures) || (tileIndex === tilesToDraw.length - 1)){ - // We've filled up the buffers: time to draw this set of tiles + let firstTile = tilesToDraw[0]; - // bind each tile's texture to the appropriate gl.TEXTURE# - for(let i = 0; i <= numTilesToDraw; i++){ - gl.activeTexture(gl.TEXTURE0 + i); - gl.bindTexture(gl.TEXTURE_2D, textureDataArray[i]); + let useContext2dPipeline = ( tiledImage.compositeOperation || + this.viewer.compositeOperation || + tiledImage._clip || + tiledImage._croppingPolygons || + tiledImage.debugMode + ); + + let useTwoPassRendering = useContext2dPipeline || (tiledImage.opacity < 1) || firstTile.hasTransparency; + + // using the context2d pipeline requires a clean rendering (back) buffer to start + if(useContext2dPipeline){ + // if the rendering buffer has image data currently, write it to the output canvas now and clear it + + if(renderingBufferHasImageData){ + this._outputContext.drawImage(this._renderingCanvas, 0, 0); } - // set the buffer data for the texture coordinates to use for each tile - gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition); - gl.bufferData(gl.ARRAY_BUFFER, texturePositionArray, gl.DYNAMIC_DRAW); + // clear the buffer + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer + } - // set the transform matrix uniform for each tile - matrixArray.forEach( (matrix, index) => { - gl.uniformMatrix3fv(this._firstPass.uTransformMatrices[index], false, matrix); - }); - // set the opacity uniform for each tile - gl.uniform1fv(this._firstPass.uOpacities, new Float32Array(opacityArray)); + // First rendering pass: compose tiles that make up this tiledImage + gl.useProgram(this._firstPass.shaderProgram); - // bind vertex buffers and (re)set attributes before calling gl.drawArrays() - gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferOutputPosition); + // bind to the framebuffer for render-to-texture if using two-pass rendering, otherwise back buffer (null) + if(useTwoPassRendering){ + gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer); + // clear the buffer to draw a new image + gl.clear(gl.COLOR_BUFFER_BIT); + } else { + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + // no need to clear, just draw on top of the existing pixels + } + + let overallMatrix = viewMatrix; + + let imageRotation = tiledImage.getRotation(true); + // if needed, handle the tiledImage being rotated + if( imageRotation % 360 !== 0){ + let imageRotationMatrix = $.Mat3.makeRotation(-imageRotation * Math.PI / 180); + let imageCenter = tiledImage.getBoundsNoRotate(true).getCenter(); + let t1 = $.Mat3.makeTranslation(imageCenter.x, imageCenter.y); + let t2 = $.Mat3.makeTranslation(-imageCenter.x, -imageCenter.y); + + // update the view matrix to account for this image's rotation + let localMatrix = t1.multiply(imageRotationMatrix).multiply(t2); + overallMatrix = viewMatrix.multiply(localMatrix); + } + + let maxTextures = this._gl.getParameter(this._gl.MAX_TEXTURE_IMAGE_UNITS); + if(maxTextures <= 0){ + // This can apparently happen on some systems if too many WebGL contexts have been created + // in which case maxTextures can be null, leading to out of bounds errors with the array. + // For example, when viewers were created and not destroyed in the test suite, this error + // occured in the TravisCI tests, though it did not happen when testing locally either in + // a browser or on the command line via grunt test. + + throw(new Error(`WegGL error: bad value for gl parameter MAX_TEXTURE_IMAGE_UNITS (${maxTextures}). This could happen + if too many contexts have been created and not released, or there is another problem with the graphics card.`)); + } + + let texturePositionArray = new Float32Array(maxTextures * 12); // 6 vertices (2 triangles) x 2 coordinates per vertex + let textureDataArray = new Array(maxTextures); + let matrixArray = new Array(maxTextures); + let opacityArray = new Array(maxTextures); + + // iterate over tiles and add data for each one to the buffers + for(let tileIndex = 0; tileIndex < tilesToDraw.length; tileIndex++){ + let tile = tilesToDraw[tileIndex].tile; + let indexInDrawArray = tileIndex % maxTextures; + let numTilesToDraw = indexInDrawArray + 1; + let tileContext = tile.getCanvasContext(); + + let textureInfo = tileContext ? this._TextureMap.get(tileContext.canvas) : null; + if(textureInfo){ + this._getTileData(tile, tiledImage, textureInfo, overallMatrix, indexInDrawArray, texturePositionArray, textureDataArray, matrixArray, opacityArray); + } else { + // console.log('No tile info', tile); + } + if( (numTilesToDraw === maxTextures) || (tileIndex === tilesToDraw.length - 1)){ + // We've filled up the buffers: time to draw this set of tiles + + // bind each tile's texture to the appropriate gl.TEXTURE# + for(let i = 0; i <= numTilesToDraw; i++){ + gl.activeTexture(gl.TEXTURE0 + i); + gl.bindTexture(gl.TEXTURE_2D, textureDataArray[i]); + } + + // set the buffer data for the texture coordinates to use for each tile + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition); + gl.bufferData(gl.ARRAY_BUFFER, texturePositionArray, gl.DYNAMIC_DRAW); + + // set the transform matrix uniform for each tile + matrixArray.forEach( (matrix, index) => { + gl.uniformMatrix3fv(this._firstPass.uTransformMatrices[index], false, matrix); + }); + // set the opacity uniform for each tile + gl.uniform1fv(this._firstPass.uOpacities, new Float32Array(opacityArray)); + + // bind vertex buffers and (re)set attributes before calling gl.drawArrays() + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferOutputPosition); + gl.vertexAttribPointer(this._firstPass.aOutputPosition, 2, gl.FLOAT, false, 0, 0); + + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition); + gl.vertexAttribPointer(this._firstPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0); + + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferIndex); + gl.vertexAttribPointer(this._firstPass.aIndex, 1, gl.FLOAT, false, 0, 0); + + // Draw! 6 vertices per tile (2 triangles per rectangle) + gl.drawArrays(gl.TRIANGLES, 0, 6 * numTilesToDraw ); + } + } + + if(useTwoPassRendering){ + // Second rendering pass: Render the tiled image from the framebuffer into the back buffer + gl.useProgram(this._secondPass.shaderProgram); + + // set the rendering target to the back buffer (null) + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + + // bind the rendered texture from the first pass to use during this second pass + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, this._renderToTexture); + + // set opacity to the value for the current tiledImage + this._gl.uniform1f(this._secondPass.uOpacityMultiplier, tiledImage.opacity); + + // bind buffers and set attributes before calling gl.drawArrays + gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferTexturePosition); + gl.vertexAttribPointer(this._secondPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0); + gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferOutputPosition); gl.vertexAttribPointer(this._firstPass.aOutputPosition, 2, gl.FLOAT, false, 0, 0); - gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition); - gl.vertexAttribPointer(this._firstPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0); + // Draw the quad (two triangles) + gl.drawArrays(gl.TRIANGLES, 0, 6); - gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferIndex); - gl.vertexAttribPointer(this._firstPass.aIndex, 1, gl.FLOAT, false, 0, 0); + } - // Draw! 6 vertices per tile (2 triangles per rectangle) - gl.drawArrays(gl.TRIANGLES, 0, 6 * numTilesToDraw ); + renderingBufferHasImageData = true; + + if(useContext2dPipeline){ + // draw from the rendering canvas onto the output canvas, clipping/cropping if needed. + this._applyContext2dPipeline(tiledImage, tilesToDraw, tiledImageIndex); + renderingBufferHasImageData = false; + // clear the buffer + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer + } + + // after drawing the first TiledImage, fire the tiled-image-drawn event (for testing) + if(tiledImageIndex === 0){ + this._raiseTiledImageDrawnEvent(tiledImage, tilesToDraw.map(info=>info.tile)); } } - if(useTwoPassRendering){ - // Second rendering pass: Render the tiled image from the framebuffer into the back buffer - gl.useProgram(this._secondPass.shaderProgram); - // set the rendering target to the back buffer (null) - gl.bindFramebuffer(gl.FRAMEBUFFER, null); - - // bind the rendered texture from the first pass to use during this second pass - gl.activeTexture(gl.TEXTURE0); - gl.bindTexture(gl.TEXTURE_2D, this._renderToTexture); - - // set opacity to the value for the current tiledImage - this._gl.uniform1f(this._secondPass.uOpacityMultiplier, tiledImage.opacity); - - // bind buffers and set attributes before calling gl.drawArrays - gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferTexturePosition); - gl.vertexAttribPointer(this._secondPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0); - gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferOutputPosition); - gl.vertexAttribPointer(this._firstPass.aOutputPosition, 2, gl.FLOAT, false, 0, 0); - - // Draw the quad (two triangles) - gl.drawArrays(gl.TRIANGLES, 0, 6); - - } - - renderingBufferHasImageData = true; - - if(useContext2dPipeline){ - // draw from the rendering canvas onto the output canvas, clipping/cropping if needed. - this._applyContext2dPipeline(tiledImage, tilesToDraw, tiledImageIndex); - renderingBufferHasImageData = false; - // clear the buffer - gl.bindFramebuffer(gl.FRAMEBUFFER, null); - gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer - } - - // after drawing the first TiledImage, fire the tiled-image-drawn event (for testing) - if(tiledImageIndex === 0){ - this._raiseTiledImageDrawnEvent(tiledImage, tilesToDraw.map(info=>info.tile)); - } }); @@ -802,7 +841,19 @@ let tile = event.tile; let tiledImage = event.tiledImage; let tileContext = tile.getCanvasContext(); - let canvas = tileContext.canvas; + let canvas = tileContext && tileContext.canvas; + // if the tile doesn't provide a canvas, or is tainted by cross-origin + // data, marked the TiledImage as tainted so the canvas drawer can be + // used instead, and return immediately - data cannot be uploaded to webgl + if(!canvas || $.isCanvasTainted(canvas)){ + const wasTainted = tiledImage.isTainted(); + if(!wasTainted){ + tiledImage.setTainted(true); + $.console.warn('WebGL cannot be used to draw this TiledImage because it has tainted data. Does crossOriginPolicy need to be set?'); + } + return; + } + let textureInfo = this._TextureMap.get(canvas); // if this is a new image for us, create a texture