mirror/openseadragon
1
0
Fork 0
mirror of https://github.com/openseadragon/openseadragon.git synced 2024-11-21 20:56:09 +03:00
Code Issues Projects Releases Wiki Activity

In webgl drawer, fall back to canvas drawer for tiled images with tainted data

Browse Source
This commit is contained in:
Tom 2024-02-11 11:51:38 -05:00
parent f7d86ac244
commit 0a154a3b21
2 changed files with 213 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/tiledimage.js
View File

@ -166,6 +166,7 @@ $.TiledImage = function( options ) {
_lastDrawn: [], // array of tiles that were last fetched by the drawer
_isBlending: false, // Are any tiles still being blended?
_wasBlending: false, // Were any tiles blending before the last draw?
_isTainted: false, // Has a Tile been found with tainted data?
//configurable settings
springStiffness: $.DEFAULT_SETTINGS.springStiffness,
animationTime: $.DEFAULT_SETTINGS.animationTime,
@ -326,6 +327,25 @@ $.extend($.TiledImage.prototype, $.EventSource.prototype, /** @lends OpenSeadrag
return this._needsDraw;
},
/**
* Set the internal _isTainted flag for this TiledImage. Lazy loaded - not
* checked each time a Tile is loaded, but can be set if a consumer of the
* tiles (e.g. a Drawer) discovers a Tile to have tainted data so that further
* checks are not needed and alternative rendering strategies can be used.
* @private
*/
setTainted(isTainted){
this._isTainted = isTainted;
},
/**
* @private
* @returns {Boolean} whether the TiledImage has been marked as tainted
*/
isTainted(){
return this._isTainted;
},
/**
* Destroy the TiledImage (unload current loaded tiles).
*/

335
src/webgldrawer.js
View File

@ -90,6 +90,7 @@
this._clippingCanvas = null;
this._clippingContext = null;
this._renderingCanvas = null;
this._backupCanvasDrawer = null;
// Add listeners for events that require modifying the scene or camera
this.viewer.addHandler("tile-ready", ev => this._tileReadyHandler(ev));
@ -206,6 +207,25 @@
return canvas;
}
/**
* Get the backup renderer (CanvasDrawer) to use if data cannot be used by webgl
* Lazy loaded
* @private
* @returns {CanvasDrawer}
*/
_getBackupCanvasDrawer(){
if(!this._backupCanvasDrawer){
this._backupCanvasDrawer = new $.CanvasDrawer({
viewer: this.viewer,
viewport: this.viewport,
element: this.container,
});
this._backupCanvasDrawer.canvas.style.setProperty('visibility', 'hidden');
}
return this._backupCanvasDrawer;
}
/**
*
* @param {Array} tiledImages Array of TiledImage objects to draw
@ -237,168 +257,187 @@
//iterate over tiled images and draw each one using a two-pass rendering pipeline if needed
tiledImages.forEach( (tiledImage, tiledImageIndex) => {
let tilesToDraw = tiledImage.getTilesToDraw();
if(tilesToDraw.length === 0 || tiledImage.getOpacity() === 0){
return;
}
let firstTile = tilesToDraw[0];
let useContext2dPipeline = ( tiledImage.compositeOperation ||
this.viewer.compositeOperation ||
tiledImage._clip ||
tiledImage._croppingPolygons ||
tiledImage.debugMode
);
let useTwoPassRendering = useContext2dPipeline || (tiledImage.opacity < 1) || firstTile.hasTransparency;
// using the context2d pipeline requires a clean rendering (back) buffer to start
if(useContext2dPipeline){
// if the rendering buffer has image data currently, write it to the output canvas now and clear it
if(tiledImage.isTainted()){
// first, draw any data left in the rendering buffer onto the output canvas
if(renderingBufferHasImageData){
this._outputContext.drawImage(this._renderingCanvas, 0, 0);
// clear the buffer
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer
renderingBufferHasImageData = false;
}
// clear the buffer
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer
}
// next, use the backup canvas drawer to draw this tainted image
const canvasDrawer = this._getBackupCanvasDrawer();
canvasDrawer.draw([tiledImage]);
this._outputContext.drawImage(canvasDrawer.canvas, 0, 0);
// First rendering pass: compose tiles that make up this tiledImage
gl.useProgram(this._firstPass.shaderProgram);
// bind to the framebuffer for render-to-texture if using two-pass rendering, otherwise back buffer (null)
if(useTwoPassRendering){
gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer);
// clear the buffer to draw a new image
gl.clear(gl.COLOR_BUFFER_BIT);
} else {
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
// no need to clear, just draw on top of the existing pixels
}
let tilesToDraw = tiledImage.getTilesToDraw();
let overallMatrix = viewMatrix;
let imageRotation = tiledImage.getRotation(true);
// if needed, handle the tiledImage being rotated
if( imageRotation % 360 !== 0){
let imageRotationMatrix = $.Mat3.makeRotation(-imageRotation * Math.PI / 180);
let imageCenter = tiledImage.getBoundsNoRotate(true).getCenter();
let t1 = $.Mat3.makeTranslation(imageCenter.x, imageCenter.y);
let t2 = $.Mat3.makeTranslation(-imageCenter.x, -imageCenter.y);
// update the view matrix to account for this image's rotation
let localMatrix = t1.multiply(imageRotationMatrix).multiply(t2);
overallMatrix = viewMatrix.multiply(localMatrix);
}
let maxTextures = this._gl.getParameter(this._gl.MAX_TEXTURE_IMAGE_UNITS);
if(maxTextures <= 0){
// This can apparently happen on some systems if too many WebGL contexts have been created
// in which case maxTextures can be null, leading to out of bounds errors with the array.
// For example, when viewers were created and not destroyed in the test suite, this error
// occured in the TravisCI tests, though it did not happen when testing locally either in
// a browser or on the command line via grunt test.
throw(new Error(`WegGL error: bad value for gl parameter MAX_TEXTURE_IMAGE_UNITS (${maxTextures}). This could happen
if too many contexts have been created and not released, or there is another problem with the graphics card.`));
}
let texturePositionArray = new Float32Array(maxTextures * 12); // 6 vertices (2 triangles) x 2 coordinates per vertex
let textureDataArray = new Array(maxTextures);
let matrixArray = new Array(maxTextures);
let opacityArray = new Array(maxTextures);
// iterate over tiles and add data for each one to the buffers
for(let tileIndex = 0; tileIndex < tilesToDraw.length; tileIndex++){
let tile = tilesToDraw[tileIndex].tile;
let indexInDrawArray = tileIndex % maxTextures;
let numTilesToDraw = indexInDrawArray + 1;
let tileContext = tile.getCanvasContext();
let textureInfo = tileContext ? this._TextureMap.get(tileContext.canvas) : null;
if(textureInfo){
this._getTileData(tile, tiledImage, textureInfo, overallMatrix, indexInDrawArray, texturePositionArray, textureDataArray, matrixArray, opacityArray);
} else {
// console.log('No tile info', tile);
if(tilesToDraw.length === 0 || tiledImage.getOpacity() === 0){
return;
}
if( (numTilesToDraw === maxTextures) || (tileIndex === tilesToDraw.length - 1)){
// We've filled up the buffers: time to draw this set of tiles
let firstTile = tilesToDraw[0];
// bind each tile's texture to the appropriate gl.TEXTURE#
for(let i = 0; i <= numTilesToDraw; i++){
gl.activeTexture(gl.TEXTURE0 + i);
gl.bindTexture(gl.TEXTURE_2D, textureDataArray[i]);
let useContext2dPipeline = ( tiledImage.compositeOperation ||
this.viewer.compositeOperation ||
tiledImage._clip ||
tiledImage._croppingPolygons ||
tiledImage.debugMode
);
let useTwoPassRendering = useContext2dPipeline || (tiledImage.opacity < 1) || firstTile.hasTransparency;
// using the context2d pipeline requires a clean rendering (back) buffer to start
if(useContext2dPipeline){
// if the rendering buffer has image data currently, write it to the output canvas now and clear it
if(renderingBufferHasImageData){
this._outputContext.drawImage(this._renderingCanvas, 0, 0);
}
// set the buffer data for the texture coordinates to use for each tile
gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition);
gl.bufferData(gl.ARRAY_BUFFER, texturePositionArray, gl.DYNAMIC_DRAW);
// clear the buffer
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer
}
// set the transform matrix uniform for each tile
matrixArray.forEach( (matrix, index) => {
gl.uniformMatrix3fv(this._firstPass.uTransformMatrices[index], false, matrix);
});
// set the opacity uniform for each tile
gl.uniform1fv(this._firstPass.uOpacities, new Float32Array(opacityArray));
// First rendering pass: compose tiles that make up this tiledImage
gl.useProgram(this._firstPass.shaderProgram);
// bind vertex buffers and (re)set attributes before calling gl.drawArrays()
gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferOutputPosition);
// bind to the framebuffer for render-to-texture if using two-pass rendering, otherwise back buffer (null)
if(useTwoPassRendering){
gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer);
// clear the buffer to draw a new image
gl.clear(gl.COLOR_BUFFER_BIT);
} else {
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
// no need to clear, just draw on top of the existing pixels
}
let overallMatrix = viewMatrix;
let imageRotation = tiledImage.getRotation(true);
// if needed, handle the tiledImage being rotated
if( imageRotation % 360 !== 0){
let imageRotationMatrix = $.Mat3.makeRotation(-imageRotation * Math.PI / 180);
let imageCenter = tiledImage.getBoundsNoRotate(true).getCenter();
let t1 = $.Mat3.makeTranslation(imageCenter.x, imageCenter.y);
let t2 = $.Mat3.makeTranslation(-imageCenter.x, -imageCenter.y);
// update the view matrix to account for this image's rotation
let localMatrix = t1.multiply(imageRotationMatrix).multiply(t2);
overallMatrix = viewMatrix.multiply(localMatrix);
}
let maxTextures = this._gl.getParameter(this._gl.MAX_TEXTURE_IMAGE_UNITS);
if(maxTextures <= 0){
// This can apparently happen on some systems if too many WebGL contexts have been created
// in which case maxTextures can be null, leading to out of bounds errors with the array.
// For example, when viewers were created and not destroyed in the test suite, this error
// occured in the TravisCI tests, though it did not happen when testing locally either in
// a browser or on the command line via grunt test.
throw(new Error(`WegGL error: bad value for gl parameter MAX_TEXTURE_IMAGE_UNITS (${maxTextures}). This could happen
if too many contexts have been created and not released, or there is another problem with the graphics card.`));
}
let texturePositionArray = new Float32Array(maxTextures * 12); // 6 vertices (2 triangles) x 2 coordinates per vertex
let textureDataArray = new Array(maxTextures);
let matrixArray = new Array(maxTextures);
let opacityArray = new Array(maxTextures);
// iterate over tiles and add data for each one to the buffers
for(let tileIndex = 0; tileIndex < tilesToDraw.length; tileIndex++){
let tile = tilesToDraw[tileIndex].tile;
let indexInDrawArray = tileIndex % maxTextures;
let numTilesToDraw = indexInDrawArray + 1;
let tileContext = tile.getCanvasContext();
let textureInfo = tileContext ? this._TextureMap.get(tileContext.canvas) : null;
if(textureInfo){
this._getTileData(tile, tiledImage, textureInfo, overallMatrix, indexInDrawArray, texturePositionArray, textureDataArray, matrixArray, opacityArray);
} else {
// console.log('No tile info', tile);
}
if( (numTilesToDraw === maxTextures) || (tileIndex === tilesToDraw.length - 1)){
// We've filled up the buffers: time to draw this set of tiles
// bind each tile's texture to the appropriate gl.TEXTURE#
for(let i = 0; i <= numTilesToDraw; i++){
gl.activeTexture(gl.TEXTURE0 + i);
gl.bindTexture(gl.TEXTURE_2D, textureDataArray[i]);
}
// set the buffer data for the texture coordinates to use for each tile
gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition);
gl.bufferData(gl.ARRAY_BUFFER, texturePositionArray, gl.DYNAMIC_DRAW);
// set the transform matrix uniform for each tile
matrixArray.forEach( (matrix, index) => {
gl.uniformMatrix3fv(this._firstPass.uTransformMatrices[index], false, matrix);
});
// set the opacity uniform for each tile
gl.uniform1fv(this._firstPass.uOpacities, new Float32Array(opacityArray));
// bind vertex buffers and (re)set attributes before calling gl.drawArrays()
gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferOutputPosition);
gl.vertexAttribPointer(this._firstPass.aOutputPosition, 2, gl.FLOAT, false, 0, 0);
gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition);
gl.vertexAttribPointer(this._firstPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0);
gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferIndex);
gl.vertexAttribPointer(this._firstPass.aIndex, 1, gl.FLOAT, false, 0, 0);
// Draw! 6 vertices per tile (2 triangles per rectangle)
gl.drawArrays(gl.TRIANGLES, 0, 6 * numTilesToDraw );
}
}
if(useTwoPassRendering){
// Second rendering pass: Render the tiled image from the framebuffer into the back buffer
gl.useProgram(this._secondPass.shaderProgram);
// set the rendering target to the back buffer (null)
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
// bind the rendered texture from the first pass to use during this second pass
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, this._renderToTexture);
// set opacity to the value for the current tiledImage
this._gl.uniform1f(this._secondPass.uOpacityMultiplier, tiledImage.opacity);
// bind buffers and set attributes before calling gl.drawArrays
gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferTexturePosition);
gl.vertexAttribPointer(this._secondPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0);
gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferOutputPosition);
gl.vertexAttribPointer(this._firstPass.aOutputPosition, 2, gl.FLOAT, false, 0, 0);
gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition);
gl.vertexAttribPointer(this._firstPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0);
// Draw the quad (two triangles)
gl.drawArrays(gl.TRIANGLES, 0, 6);
gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferIndex);
gl.vertexAttribPointer(this._firstPass.aIndex, 1, gl.FLOAT, false, 0, 0);
}
// Draw! 6 vertices per tile (2 triangles per rectangle)
gl.drawArrays(gl.TRIANGLES, 0, 6 * numTilesToDraw );
renderingBufferHasImageData = true;
if(useContext2dPipeline){
// draw from the rendering canvas onto the output canvas, clipping/cropping if needed.
this._applyContext2dPipeline(tiledImage, tilesToDraw, tiledImageIndex);
renderingBufferHasImageData = false;
// clear the buffer
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer
}
// after drawing the first TiledImage, fire the tiled-image-drawn event (for testing)
if(tiledImageIndex === 0){
this._raiseTiledImageDrawnEvent(tiledImage, tilesToDraw.map(info=>info.tile));
}
}
if(useTwoPassRendering){
// Second rendering pass: Render the tiled image from the framebuffer into the back buffer
gl.useProgram(this._secondPass.shaderProgram);
// set the rendering target to the back buffer (null)
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
// bind the rendered texture from the first pass to use during this second pass
gl.activeTexture(gl.TEXTURE0);
gl.bindTexture(gl.TEXTURE_2D, this._renderToTexture);
// set opacity to the value for the current tiledImage
this._gl.uniform1f(this._secondPass.uOpacityMultiplier, tiledImage.opacity);
// bind buffers and set attributes before calling gl.drawArrays
gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferTexturePosition);
gl.vertexAttribPointer(this._secondPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0);
gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferOutputPosition);
gl.vertexAttribPointer(this._firstPass.aOutputPosition, 2, gl.FLOAT, false, 0, 0);
// Draw the quad (two triangles)
gl.drawArrays(gl.TRIANGLES, 0, 6);
}
renderingBufferHasImageData = true;
if(useContext2dPipeline){
// draw from the rendering canvas onto the output canvas, clipping/cropping if needed.
this._applyContext2dPipeline(tiledImage, tilesToDraw, tiledImageIndex);
renderingBufferHasImageData = false;
// clear the buffer
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer
}
// after drawing the first TiledImage, fire the tiled-image-drawn event (for testing)
if(tiledImageIndex === 0){
this._raiseTiledImageDrawnEvent(tiledImage, tilesToDraw.map(info=>info.tile));
}
});
@ -802,7 +841,19 @@
let tile = event.tile;
let tiledImage = event.tiledImage;
let tileContext = tile.getCanvasContext();
let canvas = tileContext.canvas;
let canvas = tileContext && tileContext.canvas;
// if the tile doesn't provide a canvas, or is tainted by cross-origin
// data, marked the TiledImage as tainted so the canvas drawer can be
// used instead, and return immediately - data cannot be uploaded to webgl
if(!canvas || $.isCanvasTainted(canvas)){
const wasTainted = tiledImage.isTainted();
if(!wasTainted){
tiledImage.setTainted(true);
$.console.warn('WebGL cannot be used to draw this TiledImage because it has tainted data. Does crossOriginPolicy need to be set?');
}
return;
}
let textureInfo = this._TextureMap.get(canvas);
// if this is a new image for us, create a texture