diff --git a/src/canvasdrawer.js b/src/canvasdrawer.js index be0e4cbb..6366e431 100644 --- a/src/canvasdrawer.js +++ b/src/canvasdrawer.js @@ -324,7 +324,9 @@ class CanvasDrawer extends $.DrawerBase{ if (tiledImage._croppingPolygons) { var self = this; - this._saveContext(useSketch); + if(!usedClip){ + this._saveContext(useSketch); + } try { var polygons = tiledImage._croppingPolygons.map(function (polygon) { return polygon.map(function (coord) { diff --git a/src/webgldrawer.js b/src/webgldrawer.js index d634b260..4b35e19f 100644 --- a/src/webgldrawer.js +++ b/src/webgldrawer.js @@ -1,3 +1,4 @@ + /* * OpenSeadragon - WebGLDrawer * @@ -142,11 +143,10 @@ this._TileMap = new Map(); this._gl = null; - this._glLocs = null; - this._glProgram = null; - this._glUnitQuadBuffer = null; + this._firstPass = null; + this._secondPass = null; this._glFrameBuffer = null; - this._glTiledImageTexture = null; + this._renderToTexture = null; this._glFramebufferToCanvasTransform = null; this._outputCanvas = null; this._outputContext = null; @@ -196,7 +196,7 @@ }); // Delete all our created resources - gl.deleteBuffer(this._glUnitQuadBuffer); + gl.deleteBuffer(this._secondPass.bufferOutputPosition); gl.deleteFramebuffer(this._glFrameBuffer); // TO DO: if/when render buffers or frame buffers are used, release them: // gl.deleteRenderbuffer(someRenderbuffer); @@ -282,35 +282,59 @@ let rotMatrix = Mat3.makeRotation(-viewport.rotation); let viewMatrix = scaleMatrix.multiply(rotMatrix).multiply(posMatrix); + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer + // clear the output canvas this._outputContext.clearRect(0, 0, this._outputCanvas.width, this._outputCanvas.height); - // TO DO: further optimization is possible. - // If no clipping and no composite operation, the tiled images - // can all be drawn onto the rendering canvas at the same time, avoiding - // unnecessary clearing and copying of the pixel data. - // For now, I'm doing it this way to replicate full functionality - // of the context2d drawer + let renderingBufferHasImageData = false; + + //iterate over tiled images and draw each one using a two-pass rendering pipeline if needed + tiledImages.forEach( (tiledImage, tiledImageIndex) => { + + let useContext2dPipeline = ( tiledImage.compositeOperation || + this.viewer.compositeOperation || + tiledImage._clip || + tiledImage._croppingPolygons || + tiledImage.debugMode + ); + let useTwoPassRendering = useContext2dPipeline || (tiledImage.opacity < 1); // TO DO: check hasTransparency in addition to opacity - //iterate over tiled imagesget the list of tiles to draw - tiledImages.forEach( (tiledImage, i) => { - //get the list of tiles to draw let tilesToDraw = tiledImage.getTilesToDraw(); if(tilesToDraw.length === 0){ return; } - // bind to the framebuffer for render-to-texture - gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer); + // using the context2d pipeline requires a clean rendering (back) buffer to start + if(useContext2dPipeline){ + // if the rendering buffer has image data currently, write it to the output canvas now and clear it - // clear the buffer - gl.clear(gl.COLOR_BUFFER_BIT); + if(renderingBufferHasImageData){ + this._outputContext.drawImage(this._renderingCanvas, 0, 0); + } + + // clear the buffer + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer + } + + // First rendering pass: compose tiles that make up this tiledImage + gl.useProgram(this._firstPass.shaderProgram); + + // bind to the framebuffer for render-to-texture if using two-pass rendering, otherwise back buffer (null) + if(useTwoPassRendering){ + gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer); + // clear the buffer to draw a new image + gl.clear(gl.COLOR_BUFFER_BIT); + } else { + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + // no need to clear, just draw on top of the existing pixels + } - // set opacity for this image - gl.uniform1f(this._glLocs.uOpacityMultiplier, tiledImage.opacity); @@ -329,58 +353,142 @@ overallMatrix = viewMatrix.multiply(localMatrix); } - // iterate over tiles and draw each one to the buffer - for(let ti = 0; ti < tilesToDraw.length; ti++){ - let tile = tilesToDraw[ti].tile; - let textureInfo = this._TextureMap.get(tile.getCanvasContext().canvas); + let maxTextures = this._gl.getParameter(this._gl.MAX_TEXTURE_IMAGE_UNITS); + let texturePositionArray = new Float32Array(maxTextures * 12); // 6 vertices (2 triangles) x 2 coordinates per vertex + let textureDataArray = new Array(maxTextures); + let matrixArray = new Array(maxTextures); + let opacityArray = new Array(maxTextures); + + // iterate over tiles and add data for each one to the buffers + for(let tileIndex = 0; tileIndex < tilesToDraw.length; tileIndex++){ + let tile = tilesToDraw[tileIndex].tile; + let index = tileIndex % maxTextures; + let tileContext = tile.getCanvasContext(); + + let textureInfo = tileContext ? this._TextureMap.get(tileContext.canvas) : null; if(textureInfo){ - this._drawTile(tile, tiledImage, textureInfo, overallMatrix, tiledImage.opacity); + this._getTileData(tile, tiledImage, textureInfo, overallMatrix, index, texturePositionArray, textureDataArray, matrixArray, opacityArray); } else { // console.log('No tile info', tile); } - } + if( (index === maxTextures - 1) || (tileIndex === tilesToDraw.length - 1)){ + // We've filled up the buffers: time to draw this set of tiles + // bind each tile's texture to the appropriate gl.TEXTURE# + for(let i = 0; i <= index; i++){ + gl.activeTexture(gl.TEXTURE0 + i); + gl.bindTexture(gl.TEXTURE_2D, textureDataArray[i]); + } - // Draw from the Framebuffer onto the rendering canvas buffer + // set the buffer data for the texture coordinates to use for each tile + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition); + gl.bufferData(gl.ARRAY_BUFFER, texturePositionArray, gl.DYNAMIC_DRAW); - gl.flush(); // finish drawing to the texture - gl.bindFramebuffer(gl.FRAMEBUFFER, null); // null means bind to the backbuffer for drawing - gl.bindTexture(gl.TEXTURE_2D, this._glTiledImageTexture); // bind the rendered texture to use - gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer + // set the transform matrix uniform for each tile + matrixArray.forEach( (matrix, index) => { + gl.uniformMatrix3fv(this._firstPass.uTransformMatrices[index], false, matrix); + }); + // set the opacity uniform for each tile + gl.uniform1fv(this._firstPass.uOpacities, new Float32Array(opacityArray)); - // set up the matrix to draw the whole framebuffer to the entire clip space - gl.uniformMatrix3fv(this._glLocs.uMatrix, false, this._glFramebufferToCanvasTransform); + // bind vertex buffers and (re)set attributes before calling gl.drawArrays() + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferOutputPosition); + gl.vertexAttribPointer(this._firstPass.aOutputPosition, 2, gl.FLOAT, false, 0, 0); - // reset texturebuffer to unit quad - gl.bindBuffer(gl.ARRAY_BUFFER, this._glTextureBuffer); - gl.bufferData(gl.ARRAY_BUFFER, this._glUnitQuad, gl.DYNAMIC_DRAW); + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition); + gl.vertexAttribPointer(this._firstPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0); - // set opacity to the value for the current tiledImage - this._gl.uniform1f(this._glLocs.uOpacityMultiplier, tiledImage.opacity); - gl.drawArrays(gl.TRIANGLES, 0, 6); + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferIndex); + gl.vertexAttribPointer(this._firstPass.aIndex, 1, gl.FLOAT, false, 0, 0); - // iterate over any filters - filters can use this._glTiledImageTexture to get rendered data if desired - let filters = this.filters || []; - for(let fi = 0; fi < filters.length; fi++){ - let filter = this.filters[fi]; - if(filter.apply){ - filter.apply(gl); // filter.apply should write data on top of the backbuffer (bound above) + // Draw! 6 vertices per tile (2 triangles per rectangle) + gl.drawArrays(gl.TRIANGLES, 0, 6 * (index + 1) ); } } - gl.flush(); //make sure drawing to the output buffer of the rendering canvas is complete. Is this necessary? - // draw from the rendering canvas onto the output canvas, clipping/cropping if needed. - this._renderToOutputCanvas(tiledImage, tilesToDraw, i); + // gl.flush(); // is this necessary? + + if(useTwoPassRendering){ + // Second rendering pass: Render the tiled image from the framebuffer into the back buffer + gl.useProgram(this._secondPass.shaderProgram); + + // set the rendering target to the back buffer (null) + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + + // bind the rendered texture from the first pass to use during this second pass + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, this._renderToTexture); + + // set opacity to the value for the current tiledImage + this._gl.uniform1f(this._secondPass.uOpacityMultiplier, tiledImage.opacity); + + // bind buffers and set attributes before calling gl.drawArrays + gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferTexturePosition); + gl.vertexAttribPointer(this._secondPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0); + gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferOutputPosition); + gl.vertexAttribPointer(this._firstPass.aOutputPosition, 2, gl.FLOAT, false, 0, 0); + + // Draw the quad (two triangles) + gl.drawArrays(gl.TRIANGLES, 0, 6); + + // TO DO: is this the mechanism we want to use here? + // iterate over any filters - filters can use this._renderToTexture to get rendered data if desired + let filters = this.filters || []; + for(let fi = 0; fi < filters.length; fi++){ + let filter = this.filters[fi]; + if(filter.apply){ + filter.apply(gl); // filter.apply should write data on top of the backbuffer (bound above) + } + } + } + + renderingBufferHasImageData = true; + + // gl.flush(); //make sure drawing to the output buffer of the rendering canvas is complete. Is this necessary? + + if(useContext2dPipeline){ + // draw from the rendering canvas onto the output canvas, clipping/cropping if needed. + this._applyContext2dPipeline(tiledImage, tilesToDraw, tiledImageIndex); + renderingBufferHasImageData = false; + // clear the buffer + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer + } + + // Fire tiled-image-drawn event. + // TO DO: the image data may not be on the output canvas yet!! + if( this.viewer ){ + /** + * Raised when a tiled image is drawn to the canvas. Only valid + * for webgl drawer. + * + * @event tiled-image-drawn + * @memberof OpenSeadragon.Viewer + * @type {object} + * @property {OpenSeadragon.Viewer} eventSource - A reference to the Viewer which raised the event. + * @property {OpenSeadragon.TiledImage} tiledImage - Which TiledImage is being drawn. + * @property {Array} tiles - An array of Tile objects that were drawn. + * @property {?Object} userData - Arbitrary subscriber-defined object. + */ + this.viewer.raiseEvent( 'tiled-image-drawn', { + tiledImage: tiledImage, + tiles: tilesToDraw.map(info => info.tile), + }); + } }); + // TO DO: the line below is a test! + if(renderingBufferHasImageData){ + this._outputContext.drawImage(this._renderingCanvas, 0, 0); + } } // Public API required by all Drawer implementations /** - * Set the context2d imageSmoothingEnabled parameter - * @param {Boolean} enabled - */ + * Set the context2d imageSmoothingEnabled parameter + * @param {Boolean} enabled + */ setImageSmoothingEnabled(enabled){ this._clippingContext.imageSmoothingEnabled = enabled; this._outputContext.imageSmoothingEnabled = enabled; @@ -412,13 +520,13 @@ } /** - * Draw data from the rendering canvas onto the output canvas, with clipping, - * cropping and/or debug info as requested. - * @private - * @param {OpenSeadragon.TiledImage} tiledImage - the tiledImage to draw - * @param {Array} tilesToDraw - array of objects containing tiles that were drawn - */ - _renderToOutputCanvas(tiledImage, tilesToDraw, tiledImageIndex){ + * Draw data from the rendering canvas onto the output canvas, with clipping, + * cropping and/or debug info as requested. + * @private + * @param {OpenSeadragon.TiledImage} tiledImage - the tiledImage to draw + * @param {Array} tilesToDraw - array of objects containing tiles that were drawn + */ + _applyContext2dPipeline(tiledImage, tilesToDraw, tiledImageIndex){ // composite onto the output canvas, clipping if necessary this._outputContext.save(); @@ -439,39 +547,19 @@ this._drawDebugInfo(tilesToDraw, tiledImage, strokeStyle, fillStyle); } - // Fire tiled-image-drawn event now that the data is on the output canvas - if( this.viewer ){ - /** - * Raised when a tiled image is drawn to the canvas. Only valid - * for webgl drawer. - * - * @event tiled-image-drawn - * @memberof OpenSeadragon.Viewer - * @type {object} - * @property {OpenSeadragon.Viewer} eventSource - A reference to the Viewer which raised the event. - * @property {OpenSeadragon.TiledImage} tiledImage - Which TiledImage is being drawn. - * @property {Array} tiles - An array of Tile objects that were drawn. - * @property {?Object} userData - Arbitrary subscriber-defined object. - */ - this.viewer.raiseEvent( 'tiled-image-drawn', { - tiledImage: tiledImage, - tiles: tilesToDraw.map(info => info.tile), - }); - } + } // private - _drawTile(tile, tiledImage, textureInfo, viewMatrix, imageOpacity){ + _getTileData(tile, tiledImage, textureInfo, viewMatrix, index, texturePositionArray, textureDataArray, matrixArray, opacityArray){ - let gl = this._gl; let texture = textureInfo.texture; let textureQuad = textureInfo.position; - // set the vertices into the non-overlapped portion of the texture - gl.bindBuffer(gl.ARRAY_BUFFER, this._glTextureBuffer); - gl.bufferData(gl.ARRAY_BUFFER, textureQuad, gl.DYNAMIC_DRAW); + // set the position of this texture + texturePositionArray.set(textureQuad, index * 12); - // compute offsets for overlap + // compute offsets that account for tile overlap; needed for calculating the transform matrix appropriately let overlapFraction = this._calculateOverlapFraction(tile, tiledImage); let xOffset = tile.positionedBounds.width * overlapFraction.x; let yOffset = tile.positionedBounds.height * overlapFraction.y; @@ -502,41 +590,164 @@ let overallMatrix = viewMatrix.multiply(matrix); - // set opacity for this image - this._gl.uniform1f(this._glLocs.uOpacityMultiplier, tile.opacity); // imageOpacity * - - gl.uniformMatrix3fv(this._glLocs.uMatrix, false, overallMatrix.values); - gl.bindTexture(gl.TEXTURE_2D, texture); + opacityArray[index] = tile.opacity;// * tiledImage.opacity; + textureDataArray[index] = texture; + matrixArray[index] = overallMatrix.values; if(this.continuousTileRefresh){ - // Upload the image into the texture (already bound to TEXTURE_2D above) + // Upload the image into the texture + // TO DO: test if this works appropriately let tileContext = tile.getCanvasContext(); this._raiseTileDrawingEvent(tiledImage, this._outputContext, tile, tileContext); this._uploadImageData(tileContext, tile, tiledImage); } - - gl.drawArrays(gl.TRIANGLES, 0, 6); } _setupRenderer(){ - - if(!this._gl){ + let gl = this._gl; + if(!gl){ $.console.error('_setupCanvases must be called before _setupRenderer'); } + this._unitQuad = this._makeQuadVertexBuffer(0, 1, 0, 1); // used a few places; create once and store the result + this._makeFirstPassShaderProgram(); + this._makeSecondPassShaderProgram(); + + // set up the texture to render to in the first pass, and which will be used for rendering the second pass + this._renderToTexture = gl.createTexture(); + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, this._renderToTexture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, this._renderingCanvas.width, this._renderingCanvas.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + + // set up the framebuffer for render-to-texture + this._glFrameBuffer = gl.createFramebuffer(); + gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer); + gl.framebufferTexture2D( + gl.FRAMEBUFFER, + gl.COLOR_ATTACHMENT0, // attach texture as COLOR_ATTACHMENT0 + gl.TEXTURE_2D, // attach a 2D texture + this._renderToTexture, // the texture to attach + 0 + ); + + gl.enable(gl.BLEND); + gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA); + + } + + _makeFirstPassShaderProgram(){ + let numTextures = this._glNumTextures = this._gl.getParameter(this._gl.MAX_TEXTURE_IMAGE_UNITS); + let makeMatrixUniforms = () => { + return [...Array(numTextures).keys()].map(index => `uniform mat3 u_matrix_${index};`).join('\n'); + }; + let makeConditionals = () => { + return [...Array(numTextures).keys()].map(index => `${index > 0 ? 'else ' : ''}if(int(a_index) == ${index}) { transform_matrix = u_matrix_${index}; }`).join('\n'); + }; + + const vertexShaderProgram = ` + attribute vec2 a_output_position; + attribute vec2 a_texture_position; + attribute float a_index; + + ${makeMatrixUniforms()} // create a uniform mat3 for each potential tile to draw + + varying vec2 v_texture_position; + varying float v_image_index; + + void main() { + + mat3 transform_matrix; // value will be set by the if/elses in makeConditional() + + ${makeConditionals()} + + gl_Position = vec4(transform_matrix * vec3(a_output_position, 1), 1); + + v_texture_position = a_texture_position; + v_image_index = a_index; + } + `; + + const fragmentShaderProgram = ` + precision mediump float; + + // our textures + uniform sampler2D u_images[${numTextures}]; + // our opacities + uniform float u_opacities[${numTextures}]; + + // the varyings passed in from the vertex shader. + varying vec2 v_texture_position; + varying float v_image_index; + + void main() { + // can't index directly with a variable, need to use a loop iterator hack + for(int i = 0; i < ${numTextures}; ++i){ + if(i == int(v_image_index)){ + gl_FragColor = texture2D(u_images[i], v_texture_position) * u_opacities[i]; + } + } + } + `; + + let gl = this._gl; + + let program = this.constructor.initShaderProgram(gl, vertexShaderProgram, fragmentShaderProgram); + gl.useProgram(program); + + // get locations of attributes and uniforms, and create buffers for each attribute + this._firstPass = { + shaderProgram: program, + aOutputPosition: gl.getAttribLocation(program, 'a_output_position'), + aTexturePosition: gl.getAttribLocation(program, 'a_texture_position'), + aIndex: gl.getAttribLocation(program, 'a_index'), + uTransformMatrices: [...Array(this._glNumTextures).keys()].map(i=>gl.getUniformLocation(program, `u_matrix_${i}`)), + uImages: gl.getUniformLocation(program, 'u_images'), + uOpacities: gl.getUniformLocation(program, 'u_opacities'), + bufferOutputPosition: gl.createBuffer(), + bufferTexturePosition: gl.createBuffer(), + bufferIndex: gl.createBuffer(), + }; + + gl.uniform1iv(this._firstPass.uImages, [...Array(numTextures).keys()]); + + // provide coordinates for the rectangle in output space, i.e. a unit quad for each one. + let outputQuads = new Float32Array(numTextures * 12); + for(let i = 0; i < numTextures; ++i){ + outputQuads.set(Float32Array.from(this._unitQuad), i * 12); + } + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferOutputPosition); + gl.bufferData(gl.ARRAY_BUFFER, outputQuads, gl.STATIC_DRAW); // bind data statically here, since it's unchanging + gl.enableVertexAttribArray(this._firstPass.aOutputPosition); + + // provide texture coordinates for the rectangle in image (texture) space. Data will be set later. + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition); + gl.enableVertexAttribArray(this._firstPass.aTexturePosition); + + // for each vertex, provide an index into the array of textures/matrices to use for the correct tile + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferIndex); + let indices = [...Array(this._glNumTextures).keys()].map(i => Array(6).fill(i)).flat(); // repeat each index 6 times, for the 6 vertices per tile (2 triangles) + gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(indices), gl.STATIC_DRAW); // bind data statically here, since it's unchanging + gl.enableVertexAttribArray(this._firstPass.aIndex); + + } + + _makeSecondPassShaderProgram(){ const vertexShaderProgram = ` attribute vec2 a_output_position; attribute vec2 a_texture_position; uniform mat3 u_matrix; - varying vec2 v_texCoord; + varying vec2 v_texture_position; void main() { - gl_Position = vec4(u_matrix * vec3(a_output_position, 1), 1); + gl_Position = vec4(u_matrix * vec3(a_output_position, 1), 1); - v_texCoord = a_texture_position; + v_texture_position = a_texture_position; } `; @@ -547,65 +758,48 @@ uniform sampler2D u_image; // the texCoords passed in from the vertex shader. - varying vec2 v_texCoord; + varying vec2 v_texture_position; // the opacity multiplier for the image uniform float u_opacity_multiplier; void main() { - gl_FragColor = texture2D(u_image, v_texCoord); - gl_FragColor *= u_opacity_multiplier; + gl_FragColor = texture2D(u_image, v_texture_position); + gl_FragColor *= u_opacity_multiplier; } `; - let gl = this._gl; - this._glProgram = this.constructor.initShaderProgram(gl, vertexShaderProgram, fragmentShaderProgram); - gl.useProgram(this._glProgram); - gl.enable(gl.BLEND); - gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA); - this._glLocs = { - aOutputPosition: gl.getAttribLocation(this._glProgram, 'a_output_position'), - aTexturePosition: gl.getAttribLocation(this._glProgram, 'a_texture_position'), - uMatrix: gl.getUniformLocation(this._glProgram, 'u_matrix'), - uImage: gl.getUniformLocation(this._glProgram, 'u_image'), - uOpacityMultiplier: gl.getUniformLocation(this._glProgram, 'u_opacity_multiplier') + let gl = this._gl; + + let program = this.constructor.initShaderProgram(gl, vertexShaderProgram, fragmentShaderProgram); + gl.useProgram(program); + + // get locations of attributes and uniforms, and create buffers for each attribute + this._secondPass = { + shaderProgram: program, + aOutputPosition: gl.getAttribLocation(program, 'a_output_position'), + aTexturePosition: gl.getAttribLocation(program, 'a_texture_position'), + uMatrix: gl.getUniformLocation(program, 'u_matrix'), + uImage: gl.getUniformLocation(program, 'u_image'), + uOpacityMultiplier: gl.getUniformLocation(program, 'u_opacity_multiplier'), + bufferOutputPosition: gl.createBuffer(), + bufferTexturePosition: gl.createBuffer(), }; - this._glUnitQuad = this._makeQuadVertexBuffer(0, 1, 0, 1); - // provide texture coordinates for the rectangle in output space. - this._glUnitQuadBuffer = gl.createBuffer(); //keep reference to clear it later - gl.bindBuffer(gl.ARRAY_BUFFER, this._glUnitQuadBuffer); - gl.bufferData(gl.ARRAY_BUFFER, this._glUnitQuad, gl.STATIC_DRAW); - gl.enableVertexAttribArray(this._glLocs.aOutputPosition); - gl.vertexAttribPointer(this._glLocs.aOutputPosition, 2, gl.FLOAT, false, 0, 0); + + // provide coordinates for the rectangle in output space, i.e. a unit quad for each one. + gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferOutputPosition); + gl.bufferData(gl.ARRAY_BUFFER, this._unitQuad, gl.STATIC_DRAW); // bind data statically here since it's unchanging + gl.enableVertexAttribArray(this._secondPass.aOutputPosition); // provide texture coordinates for the rectangle in image (texture) space. - this._glTextureBuffer = gl.createBuffer(); //keep reference to clear it later - gl.bindBuffer(gl.ARRAY_BUFFER, this._glTextureBuffer); - gl.bufferData(gl.ARRAY_BUFFER, this._glUnitQuad, gl.DYNAMIC_DRAW); // use unit quad to start, will be updated per tile - gl.enableVertexAttribArray(this._glLocs.aTexturePosition); - gl.vertexAttribPointer(this._glLocs.aTexturePosition, 2, gl.FLOAT, false, 0, 0); - - // setup the framebuffer - this._glTiledImageTexture = gl.createTexture(); - gl.bindTexture(gl.TEXTURE_2D, this._glTiledImageTexture); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, this._renderingCanvas.width, this._renderingCanvas.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - this._glFrameBuffer = gl.createFramebuffer(); - gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer); - gl.framebufferTexture2D( - gl.FRAMEBUFFER, - gl.COLOR_ATTACHMENT0, // attach texture as COLOR_ATTACHMENT0 - gl.TEXTURE_2D, // attach a 2D texture - this._glTiledImageTexture, // the texture to attach - 0 - ); - - this._glFramebufferToCanvasTransform = Mat3.makeScaling(2, 2).multiply(Mat3.makeTranslation(-0.5, -0.5)).values; - + gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferTexturePosition); + gl.bufferData(gl.ARRAY_BUFFER, this._unitQuad, gl.DYNAMIC_DRAW); // bind data statically here since it's unchanging + gl.enableVertexAttribArray(this._secondPass.aTexturePosition); + // set the matrix that transforms the framebuffer to clip space + let matrix = Mat3.makeScaling(2, 2).multiply(Mat3.makeTranslation(-0.5, -0.5)); + gl.uniformMatrix3fv(this._secondPass.uMatrix, false, matrix.values); } _resizeRenderer(){ @@ -615,10 +809,11 @@ gl.viewport(0, 0, w, h); //release the old texture - gl.deleteTexture(this._glTiledImageTexture); + gl.deleteTexture(this._renderToTexture); //create a new texture and set it up - this._glTiledImageTexture = gl.createTexture(); - gl.bindTexture(gl.TEXTURE_2D, this._glTiledImageTexture); + this._renderToTexture = gl.createTexture(); + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, this._renderToTexture); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, w, h, 0, gl.RGBA, gl.UNSIGNED_BYTE, null); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); @@ -626,7 +821,7 @@ //bind the frame buffer to the new texture gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer); - gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, this._glTiledImageTexture, 0); + gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, this._renderToTexture, 0); } @@ -708,7 +903,7 @@ position = this._makeQuadVertexBuffer(left, right, top, bottom); } else { // no overlap: this texture can use the unit quad as it's position data - position = this._glUnitQuad; + position = this._unitQuad; } let textureInfo = { @@ -718,7 +913,7 @@ // add it to our _TextureMap this._TextureMap.set(canvas, textureInfo); - + gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, texture); // Set the parameters so we can render any size image. gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); @@ -1035,14 +1230,6 @@ }; - // $.WebGLDrawer.Filter = class{ - // constructor(gl){ - // this.gl = gl; - // } - // apply(){ - - // } - // }; }( OpenSeadragon )); diff --git a/test/demo/drawercomparison.html b/test/demo/drawercomparison.html index 7c09a3c0..3e007419 100644 --- a/test/demo/drawercomparison.html +++ b/test/demo/drawercomparison.html @@ -6,6 +6,7 @@ + + + +
+ +

Compare performance of drawer implementations

+
+ + + + + +
+
+ + +
+ + + + + + diff --git a/test/demo/drawerperformance.js b/test/demo/drawerperformance.js new file mode 100644 index 00000000..25cd52e5 --- /dev/null +++ b/test/demo/drawerperformance.js @@ -0,0 +1,90 @@ +const sources = { + "rainbow":"../data/testpattern.dzi", + "leaves":"../data/iiif_2_0_sizes/info.json", + "bblue":{ + type:'image', + url: "../data/BBlue.png", + }, + "duomo":"https://openseadragon.github.io/example-images/duomo/duomo.dzi", +} +const labels = { + rainbow: 'Rainbow Grid', + leaves: 'Leaves', + bblue: 'Blue B', + duomo: 'Duomo', +} +let viewer; + +(function(){ + var script=document.createElement('script'); + script.onload=function(){ + var stats=new Stats(); + document.body.appendChild(stats.dom); + requestAnimationFrame(function loop(){stats.update();requestAnimationFrame(loop)}); + }; + script.src='https://mrdoob.github.io/stats.js/build/stats.min.js'; + document.head.appendChild(script); +})(); + + +$('#create-drawer').on('click',function(){ + let drawerType = $('#select-drawer').val(); + let num = Math.floor($('#input-number').val()); + + if(viewer){ + viewer.destroy(); + } + viewer = window.viewer = makeViewer(drawerType); + let tileSources = makeTileSources(num); + + + + tileSources.forEach((ts, i) => { + viewer.addTiledImage({ + tileSource: ts, + x: (i % 10) / 20, + y: Math.floor(i / 10) / 20, + width: 1, + opacity: (i % 3) === 0 ? 0.4 : 1 + }); + }); + + let movingLeft = false; + window.setInterval(()=>{ + let m = movingLeft ? 1 : -1; + movingLeft = m === -1; + let dist = viewer.viewport.getBounds().width / 2 / viewer.viewport.getZoom(); + viewer.viewport.panBy(new OpenSeadragon.Point( dist * m/2, 0)); + + }, 1000); + +}); + +function makeViewer(drawerType){ + let viewer = OpenSeadragon({ + id: "drawer", + prefixUrl: "../../build/openseadragon/images/", + minZoomImageRatio:0.01, + maxZoomPixelRatio:100, + smoothTileEdgesMinZoom:1.1, + crossOriginPolicy: 'Anonymous', + ajaxWithCredentials: false, + drawer:drawerType, + blendTime:0 + }); + + return viewer; +} + +function makeTileSources(num){ + + let keys = Object.keys(sources); + + let indices = Array.from(Array(num).keys()); + + return indices.map(index => { + let ts = sources[keys[index % keys.length]]; + return ts; + }) + +} diff --git a/test/demo/webgldemodrawer.js b/test/demo/webgldemodrawer.js index 61e7c13c..1128e4a2 100644 --- a/test/demo/webgldemodrawer.js +++ b/test/demo/webgldemodrawer.js @@ -1,7 +1,6 @@ // THIS CODE OVERWRITES THE ORIGINAL VERSION FOR FASTER TESTING // i.e. it doesn't need to be re-built with grunt after every save. - /* * OpenSeadragon - WebGLDrawer * @@ -146,11 +145,10 @@ this._TileMap = new Map(); this._gl = null; - this._glLocs = null; - this._glProgram = null; - this._glUnitQuadBuffer = null; + this._firstPass = null; + this._secondPass = null; this._glFrameBuffer = null; - this._glTiledImageTexture = null; + this._renderToTexture = null; this._glFramebufferToCanvasTransform = null; this._outputCanvas = null; this._outputContext = null; @@ -200,8 +198,8 @@ }); // Delete all our created resources - gl.deleteBuffer(this._glUnitQuadBuffer); - gl.deleteBuffer(this._glFrameBuffer); + gl.deleteBuffer(this._secondPass.bufferOutputPosition); + gl.deleteFramebuffer(this._glFrameBuffer); // TO DO: if/when render buffers or frame buffers are used, release them: // gl.deleteRenderbuffer(someRenderbuffer); // gl.deleteFramebuffer(someFramebuffer); @@ -238,8 +236,7 @@ // Public API required by all Drawer implementations /** - * @returns {Boolean} returns true if canvas and webgl are supported and - * three.js has been exposed as a global variable named THREE + * @returns {Boolean} returns true if canvas and webgl are supported */ static isSupported(){ let canvasElement = document.createElement( 'canvas' ); @@ -252,6 +249,10 @@ return !!( webglContext ); } + getType(){ + return 'webgl'; + } + /** * create the HTML element (canvas in this case) that the image will be drawn into * @returns {Element} the canvas to draw into @@ -283,35 +284,59 @@ let rotMatrix = Mat3.makeRotation(-viewport.rotation); let viewMatrix = scaleMatrix.multiply(rotMatrix).multiply(posMatrix); + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer + // clear the output canvas this._outputContext.clearRect(0, 0, this._outputCanvas.width, this._outputCanvas.height); - // TO DO: further optimization is possible. - // If no clipping and no composite operation, the tiled images - // can all be drawn onto the rendering canvas at the same time, avoiding - // unnecessary clearing and copying of the pixel data. - // For now, I'm doing it this way to replicate full functionality - // of the context2d drawer + let renderingBufferHasImageData = false; + + //iterate over tiled images and draw each one using a two-pass rendering pipeline if needed + tiledImages.forEach( (tiledImage, tiledImageIndex) => { + + let useContext2dPipeline = ( tiledImage.compositeOperation || + this.viewer.compositeOperation || + tiledImage._clip || + tiledImage._croppingPolygons || + tiledImage.debugMode + ); + let useTwoPassRendering = useContext2dPipeline ||(tiledImage.opacity < 1); // TO DO: check hasTransparency in addition to opacity - //iterate over tiled imagesget the list of tiles to draw - tiledImages.forEach( (tiledImage, i) => { - //get the list of tiles to draw let tilesToDraw = tiledImage.getTilesToDraw(); if(tilesToDraw.length === 0){ return; } - // bind to the framebuffer for render-to-texture - gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer); + // using the context2d pipeline requires a clean rendering (back) buffer to start + if(useContext2dPipeline){ + // if the rendering buffer has image data currently, write it to the output canvas now and clear it - // clear the buffer - gl.clear(gl.COLOR_BUFFER_BIT); + if(renderingBufferHasImageData){ + this._outputContext.drawImage(this._renderingCanvas, 0, 0); + } + + // clear the buffer + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer + } + + // First rendering pass: compose tiles that make up this tiledImage + gl.useProgram(this._firstPass.shaderProgram); + + // bind to the framebuffer for render-to-texture if using two-pass rendering, otherwise back buffer (null) + if(useTwoPassRendering){ + gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer); + // clear the buffer to draw a new image + gl.clear(gl.COLOR_BUFFER_BIT); + } else { + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + // no need to clear, just draw on top of the existing pixels + } - // set opacity for this image - gl.uniform1f(this._glLocs.uOpacityMultiplier, tiledImage.opacity); @@ -330,62 +355,142 @@ overallMatrix = viewMatrix.multiply(localMatrix); } - // iterate over tiles and draw each one to the buffer - for(let ti = 0; ti < tilesToDraw.length; ti++){ - let tile = tilesToDraw[ti].tile; - let textureInfo = this._TextureMap.get(tile.getCanvasContext().canvas); + let maxTextures = this._gl.getParameter(this._gl.MAX_TEXTURE_IMAGE_UNITS); + let texturePositionArray = new Float32Array(maxTextures * 12); // 6 vertices (2 triangles) x 2 coordinates per vertex + let textureDataArray = new Array(maxTextures); + let matrixArray = new Array(maxTextures); + let opacityArray = new Array(maxTextures); + + // iterate over tiles and add data for each one to the buffers + for(let tileIndex = 0; tileIndex < tilesToDraw.length; tileIndex++){ + let tile = tilesToDraw[tileIndex].tile; + let index = tileIndex % maxTextures; + let tileContext = tile.getCanvasContext(); + + let textureInfo = tileContext ? this._TextureMap.get(tileContext.canvas) : null; if(textureInfo){ - this._drawTile(tile, tiledImage, textureInfo, overallMatrix, tiledImage.opacity); + this._getTileData(tile, tiledImage, textureInfo, overallMatrix, index, texturePositionArray, textureDataArray, matrixArray, opacityArray); } else { // console.log('No tile info', tile); } - } + if( (index === maxTextures - 1) || (tileIndex === tilesToDraw.length - 1)){ + // We've filled up the buffers: time to draw this set of tiles + // bind each tile's texture to the appropriate gl.TEXTURE# + for(let i = 0; i <= index; i++){ + gl.activeTexture(gl.TEXTURE0 + i); + gl.bindTexture(gl.TEXTURE_2D, textureDataArray[i]); + } - // Draw from the Framebuffer onto the rendering canvas buffer + // set the buffer data for the texture coordinates to use for each tile + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition); + gl.bufferData(gl.ARRAY_BUFFER, texturePositionArray, gl.DYNAMIC_DRAW); - gl.flush(); // finish drawing to the texture - gl.bindFramebuffer(gl.FRAMEBUFFER, null); // null means bind to the backbuffer for drawing - gl.bindTexture(gl.TEXTURE_2D, this._glTiledImageTexture); // bind the rendered texture to use - gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer + // set the transform matrix uniform for each tile + matrixArray.forEach( (matrix, index) => { + gl.uniformMatrix3fv(this._firstPass.uTransformMatrices[index], false, matrix); + }); + // set the opacity uniform for each tile + gl.uniform1fv(this._firstPass.uOpacities, new Float32Array(opacityArray)); - // set up the matrix to draw the whole framebuffer to the entire clip space - gl.uniformMatrix3fv(this._glLocs.uMatrix, false, this._glFramebufferToCanvasTransform); + // bind vertex buffers and (re)set attributes before calling gl.drawArrays() + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferOutputPosition); + gl.vertexAttribPointer(this._firstPass.aOutputPosition, 2, gl.FLOAT, false, 0, 0); - // reset texturebuffer to unit quad - gl.bindBuffer(gl.ARRAY_BUFFER, this._glTextureBuffer); - gl.bufferData(gl.ARRAY_BUFFER, this._glUnitQuad, gl.DYNAMIC_DRAW); + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition); + gl.vertexAttribPointer(this._firstPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0); - // set opacity to the value for the current tiledImage - this._gl.uniform1f(this._glLocs.uOpacityMultiplier, tiledImage.opacity); - gl.drawArrays(gl.TRIANGLES, 0, 6); + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferIndex); + gl.vertexAttribPointer(this._firstPass.aIndex, 1, gl.FLOAT, false, 0, 0); - // iterate over any filters - filters can use this._glTiledImageTexture to get rendered data if desired - let filters = this.filters || []; - for(let fi = 0; fi < filters.length; fi++){ - let filter = this.filters[fi]; - if(filter.createProgram && !filter.program){ - filter.createProgram(gl); - } - if(filter.apply){ - filter.apply(gl, this._glTiledImageTexture, tiledImage); // filter.apply should write data on top of the backbuffer (bound above) + // Draw! 6 vertices per tile (2 triangles per rectangle) + gl.drawArrays(gl.TRIANGLES, 0, 6 * (index + 1) ); } } - gl.flush(); //make sure drawing to the output buffer of the rendering canvas is complete. Is this necessary? + // gl.flush(); // is this necessary? - // draw from the rendering canvas onto the output canvas, clipping/cropping if needed. - this._renderToOutputCanvas(tiledImage, tilesToDraw, i); + if(useTwoPassRendering){ + // Second rendering pass: Render the tiled image from the framebuffer into the back buffer + gl.useProgram(this._secondPass.shaderProgram); + + // set the rendering target to the back buffer (null) + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + + // bind the rendered texture from the first pass to use during this second pass + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, this._renderToTexture); + + // set opacity to the value for the current tiledImage + this._gl.uniform1f(this._secondPass.uOpacityMultiplier, tiledImage.opacity); + + // bind buffers and set attributes before calling gl.drawArrays + gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferTexturePosition); + gl.vertexAttribPointer(this._secondPass.aTexturePosition, 2, gl.FLOAT, false, 0, 0); + gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferOutputPosition); + gl.vertexAttribPointer(this._firstPass.aOutputPosition, 2, gl.FLOAT, false, 0, 0); + + // Draw the quad (two triangles) + gl.drawArrays(gl.TRIANGLES, 0, 6); + + // TO DO: is this the mechanism we want to use here? + // iterate over any filters - filters can use this._renderToTexture to get rendered data if desired + let filters = this.filters || []; + for(let fi = 0; fi < filters.length; fi++){ + let filter = this.filters[fi]; + if(filter.apply){ + filter.apply(gl); // filter.apply should write data on top of the backbuffer (bound above) + } + } + } + + renderingBufferHasImageData = true; + + // gl.flush(); //make sure drawing to the output buffer of the rendering canvas is complete. Is this necessary? + + if(useContext2dPipeline){ + // draw from the rendering canvas onto the output canvas, clipping/cropping if needed. + this._applyContext2dPipeline(tiledImage, tilesToDraw, tiledImageIndex); + renderingBufferHasImageData = false; + // clear the buffer + gl.bindFramebuffer(gl.FRAMEBUFFER, null); + gl.clear(gl.COLOR_BUFFER_BIT); // clear the back buffer + } + + // Fire tiled-image-drawn event. + // TO DO: the image data may not be on the output canvas yet!! + if( this.viewer ){ + /** + * Raised when a tiled image is drawn to the canvas. Only valid + * for webgl drawer. + * + * @event tiled-image-drawn + * @memberof OpenSeadragon.Viewer + * @type {object} + * @property {OpenSeadragon.Viewer} eventSource - A reference to the Viewer which raised the event. + * @property {OpenSeadragon.TiledImage} tiledImage - Which TiledImage is being drawn. + * @property {Array} tiles - An array of Tile objects that were drawn. + * @property {?Object} userData - Arbitrary subscriber-defined object. + */ + this.viewer.raiseEvent( 'tiled-image-drawn', { + tiledImage: tiledImage, + tiles: tilesToDraw.map(info => info.tile), + }); + } }); + // TO DO: the line below is a test! + if(renderingBufferHasImageData){ + this._outputContext.drawImage(this._renderingCanvas, 0, 0); + } } // Public API required by all Drawer implementations /** - * Set the context2d imageSmoothingEnabled parameter - * @param {Boolean} enabled - */ + * Set the context2d imageSmoothingEnabled parameter + * @param {Boolean} enabled + */ setImageSmoothingEnabled(enabled){ this._clippingContext.imageSmoothingEnabled = enabled; this._outputContext.imageSmoothingEnabled = enabled; @@ -417,13 +522,13 @@ } /** - * Draw data from the rendering canvas onto the output canvas, with clipping, - * cropping and/or debug info as requested. - * @private - * @param {OpenSeadragon.TiledImage} tiledImage - the tiledImage to draw - * @param {Array} tilesToDraw - array of objects containing tiles that were drawn - */ - _renderToOutputCanvas(tiledImage, tilesToDraw, tiledImageIndex){ + * Draw data from the rendering canvas onto the output canvas, with clipping, + * cropping and/or debug info as requested. + * @private + * @param {OpenSeadragon.TiledImage} tiledImage - the tiledImage to draw + * @param {Array} tilesToDraw - array of objects containing tiles that were drawn + */ + _applyContext2dPipeline(tiledImage, tilesToDraw, tiledImageIndex){ // composite onto the output canvas, clipping if necessary this._outputContext.save(); @@ -444,39 +549,19 @@ this._drawDebugInfo(tilesToDraw, tiledImage, strokeStyle, fillStyle); } - // Fire tiled-image-drawn event now that the data is on the output canvas - if( this.viewer ){ - /** - * Raised when a tiled image is drawn to the canvas. Only valid - * for webgl drawer. - * - * @event tiled-image-drawn - * @memberof OpenSeadragon.Viewer - * @type {object} - * @property {OpenSeadragon.Viewer} eventSource - A reference to the Viewer which raised the event. - * @property {OpenSeadragon.TiledImage} tiledImage - Which TiledImage is being drawn. - * @property {Array} tiles - An array of Tile objects that were drawn. - * @property {?Object} userData - Arbitrary subscriber-defined object. - */ - this.viewer.raiseEvent( 'tiled-image-drawn', { - tiledImage: tiledImage, - tiles: tilesToDraw.map(info => info.tile), - }); - } + } // private - _drawTile(tile, tiledImage, textureInfo, viewMatrix, imageOpacity){ + _getTileData(tile, tiledImage, textureInfo, viewMatrix, index, texturePositionArray, textureDataArray, matrixArray, opacityArray){ - let gl = this._gl; let texture = textureInfo.texture; let textureQuad = textureInfo.position; - // set the vertices into the non-overlapped portion of the texture - gl.bindBuffer(gl.ARRAY_BUFFER, this._glTextureBuffer); - gl.bufferData(gl.ARRAY_BUFFER, textureQuad, gl.DYNAMIC_DRAW); + // set the position of this texture + texturePositionArray.set(textureQuad, index * 12); - // compute offsets for overlap + // compute offsets that account for tile overlap; needed for calculating the transform matrix appropriately let overlapFraction = this._calculateOverlapFraction(tile, tiledImage); let xOffset = tile.positionedBounds.width * overlapFraction.x; let yOffset = tile.positionedBounds.height * overlapFraction.y; @@ -507,41 +592,164 @@ let overallMatrix = viewMatrix.multiply(matrix); - // set opacity for this image - this._gl.uniform1f(this._glLocs.uOpacityMultiplier, tile.opacity); // imageOpacity * - - gl.uniformMatrix3fv(this._glLocs.uMatrix, false, overallMatrix.values); - gl.bindTexture(gl.TEXTURE_2D, texture); + opacityArray[index] = tile.opacity;// * tiledImage.opacity; + textureDataArray[index] = texture; + matrixArray[index] = overallMatrix.values; if(this.continuousTileRefresh){ - // Upload the image into the texture (already bound to TEXTURE_2D above) + // Upload the image into the texture + // TO DO: test if this works appropriately let tileContext = tile.getCanvasContext(); this._raiseTileDrawingEvent(tiledImage, this._outputContext, tile, tileContext); this._uploadImageData(tileContext, tile, tiledImage); } - - gl.drawArrays(gl.TRIANGLES, 0, 6); } _setupRenderer(){ - - if(!this._gl){ + let gl = this._gl; + if(!gl){ $.console.error('_setupCanvases must be called before _setupRenderer'); } + this._unitQuad = this._makeQuadVertexBuffer(0, 1, 0, 1); // used a few places; create once and store the result + this._makeFirstPassShaderProgram(); + this._makeSecondPassShaderProgram(); + + // set up the texture to render to in the first pass, and which will be used for rendering the second pass + this._renderToTexture = gl.createTexture(); + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, this._renderToTexture); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, this._renderingCanvas.width, this._renderingCanvas.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + + // set up the framebuffer for render-to-texture + this._glFrameBuffer = gl.createFramebuffer(); + gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer); + gl.framebufferTexture2D( + gl.FRAMEBUFFER, + gl.COLOR_ATTACHMENT0, // attach texture as COLOR_ATTACHMENT0 + gl.TEXTURE_2D, // attach a 2D texture + this._renderToTexture, // the texture to attach + 0 + ); + + gl.enable(gl.BLEND); + gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA); + + } + + _makeFirstPassShaderProgram(){ + let numTextures = this._glNumTextures = this._gl.getParameter(this._gl.MAX_TEXTURE_IMAGE_UNITS); + let makeMatrixUniforms = () => { + return [...Array(numTextures).keys()].map(index => `uniform mat3 u_matrix_${index};`).join('\n'); + }; + let makeConditionals = () => { + return [...Array(numTextures).keys()].map(index => `${index > 0 ? 'else ' : ''}if(int(a_index) == ${index}) { transform_matrix = u_matrix_${index}; }`).join('\n'); + }; + + const vertexShaderProgram = ` + attribute vec2 a_output_position; + attribute vec2 a_texture_position; + attribute float a_index; + + ${makeMatrixUniforms()} // create a uniform mat3 for each potential tile to draw + + varying vec2 v_texture_position; + varying float v_image_index; + + void main() { + + mat3 transform_matrix; // value will be set by the if/elses in makeConditional() + + ${makeConditionals()} + + gl_Position = vec4(transform_matrix * vec3(a_output_position, 1), 1); + + v_texture_position = a_texture_position; + v_image_index = a_index; + } + `; + + const fragmentShaderProgram = ` + precision mediump float; + + // our textures + uniform sampler2D u_images[${numTextures}]; + // our opacities + uniform float u_opacities[${numTextures}]; + + // the varyings passed in from the vertex shader. + varying vec2 v_texture_position; + varying float v_image_index; + + void main() { + // can't index directly with a variable, need to use a loop iterator hack + for(int i = 0; i < ${numTextures}; ++i){ + if(i == int(v_image_index)){ + gl_FragColor = texture2D(u_images[i], v_texture_position) * u_opacities[i]; + } + } + } + `; + + let gl = this._gl; + + let program = this.constructor.initShaderProgram(gl, vertexShaderProgram, fragmentShaderProgram); + gl.useProgram(program); + + // get locations of attributes and uniforms, and create buffers for each attribute + this._firstPass = { + shaderProgram: program, + aOutputPosition: gl.getAttribLocation(program, 'a_output_position'), + aTexturePosition: gl.getAttribLocation(program, 'a_texture_position'), + aIndex: gl.getAttribLocation(program, 'a_index'), + uTransformMatrices: [...Array(this._glNumTextures).keys()].map(i=>gl.getUniformLocation(program, `u_matrix_${i}`)), + uImages: gl.getUniformLocation(program, 'u_images'), + uOpacities: gl.getUniformLocation(program, 'u_opacities'), + bufferOutputPosition: gl.createBuffer(), + bufferTexturePosition: gl.createBuffer(), + bufferIndex: gl.createBuffer(), + }; + + gl.uniform1iv(this._firstPass.uImages, [...Array(numTextures).keys()]); + + // provide coordinates for the rectangle in output space, i.e. a unit quad for each one. + let outputQuads = new Float32Array(numTextures * 12); + for(let i = 0; i < numTextures; ++i){ + outputQuads.set(Float32Array.from(this._unitQuad), i * 12); + } + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferOutputPosition); + gl.bufferData(gl.ARRAY_BUFFER, outputQuads, gl.STATIC_DRAW); // bind data statically here, since it's unchanging + gl.enableVertexAttribArray(this._firstPass.aOutputPosition); + + // provide texture coordinates for the rectangle in image (texture) space. Data will be set later. + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferTexturePosition); + gl.enableVertexAttribArray(this._firstPass.aTexturePosition); + + // for each vertex, provide an index into the array of textures/matrices to use for the correct tile + gl.bindBuffer(gl.ARRAY_BUFFER, this._firstPass.bufferIndex); + let indices = [...Array(this._glNumTextures).keys()].map(i => Array(6).fill(i)).flat(); // repeat each index 6 times, for the 6 vertices per tile (2 triangles) + gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(indices), gl.STATIC_DRAW); // bind data statically here, since it's unchanging + gl.enableVertexAttribArray(this._firstPass.aIndex); + + } + + _makeSecondPassShaderProgram(){ const vertexShaderProgram = ` attribute vec2 a_output_position; attribute vec2 a_texture_position; uniform mat3 u_matrix; - varying vec2 v_texCoord; + varying vec2 v_texture_position; void main() { - gl_Position = vec4(u_matrix * vec3(a_output_position, 1), 1); + gl_Position = vec4(u_matrix * vec3(a_output_position, 1), 1); - v_texCoord = a_texture_position; + v_texture_position = a_texture_position; } `; @@ -552,66 +760,48 @@ uniform sampler2D u_image; // the texCoords passed in from the vertex shader. - varying vec2 v_texCoord; + varying vec2 v_texture_position; // the opacity multiplier for the image uniform float u_opacity_multiplier; void main() { - gl_FragColor = texture2D(u_image, v_texCoord); - gl_FragColor *= u_opacity_multiplier; + gl_FragColor = texture2D(u_image, v_texture_position); + gl_FragColor *= u_opacity_multiplier; } `; let gl = this._gl; - this._glProgram = this.constructor.initShaderProgram(gl, vertexShaderProgram, fragmentShaderProgram); - gl.useProgram(this._glProgram); - gl.enable(gl.BLEND); - gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA); - this._glLocs = { - aOutputPosition: gl.getAttribLocation(this._glProgram, 'a_output_position'), - aTexturePosition: gl.getAttribLocation(this._glProgram, 'a_texture_position'), - uMatrix: gl.getUniformLocation(this._glProgram, 'u_matrix'), - uImage: gl.getUniformLocation(this._glProgram, 'u_image'), - uOpacityMultiplier: gl.getUniformLocation(this._glProgram, 'u_opacity_multiplier') + let program = this.constructor.initShaderProgram(gl, vertexShaderProgram, fragmentShaderProgram); + gl.useProgram(program); + + // get locations of attributes and uniforms, and create buffers for each attribute + this._secondPass = { + shaderProgram: program, + aOutputPosition: gl.getAttribLocation(program, 'a_output_position'), + aTexturePosition: gl.getAttribLocation(program, 'a_texture_position'), + uMatrix: gl.getUniformLocation(program, 'u_matrix'), + uImage: gl.getUniformLocation(program, 'u_image'), + uOpacityMultiplier: gl.getUniformLocation(program, 'u_opacity_multiplier'), + bufferOutputPosition: gl.createBuffer(), + bufferTexturePosition: gl.createBuffer(), }; - this._glUnitQuad = this._makeQuadVertexBuffer(0, 1, 0, 1); - // provide texture coordinates for the rectangle in output space. - this._glUnitQuadBuffer = gl.createBuffer(); //keep reference to clear it later - gl.bindBuffer(gl.ARRAY_BUFFER, this._glUnitQuadBuffer); - gl.bufferData(gl.ARRAY_BUFFER, this._glUnitQuad, gl.STATIC_DRAW); - gl.enableVertexAttribArray(this._glLocs.aOutputPosition); - gl.vertexAttribPointer(this._glLocs.aOutputPosition, 2, gl.FLOAT, false, 0, 0); + + // provide coordinates for the rectangle in output space, i.e. a unit quad for each one. + gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferOutputPosition); + gl.bufferData(gl.ARRAY_BUFFER, this._unitQuad, gl.STATIC_DRAW); // bind data statically here since it's unchanging + gl.enableVertexAttribArray(this._secondPass.aOutputPosition); // provide texture coordinates for the rectangle in image (texture) space. - this._glTextureBuffer = gl.createBuffer(); //keep reference to clear it later - gl.bindBuffer(gl.ARRAY_BUFFER, this._glTextureBuffer); - gl.bufferData(gl.ARRAY_BUFFER, this._glUnitQuad, gl.DYNAMIC_DRAW); // use unit quad to start, will be updated per tile - gl.enableVertexAttribArray(this._glLocs.aTexturePosition); - gl.vertexAttribPointer(this._glLocs.aTexturePosition, 2, gl.FLOAT, false, 0, 0); - - // setup the framebuffer - this._glTiledImageTexture = gl.createTexture(); - gl.bindTexture(gl.TEXTURE_2D, this._glTiledImageTexture); - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, this._renderingCanvas.width, this._renderingCanvas.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - this._glFrameBuffer = gl.createFramebuffer(); - gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer); - gl.framebufferTexture2D( - gl.FRAMEBUFFER, - gl.COLOR_ATTACHMENT0, // attach texture as COLOR_ATTACHMENT0 - gl.TEXTURE_2D, // attach a 2D texture - this._glTiledImageTexture, // the texture to attach - 0 - ); - - this._glFramebufferToCanvasTransform = Mat3.makeScaling(2, 2).multiply(Mat3.makeTranslation(-0.5, -0.5)).values; - + gl.bindBuffer(gl.ARRAY_BUFFER, this._secondPass.bufferTexturePosition); + gl.bufferData(gl.ARRAY_BUFFER, this._unitQuad, gl.DYNAMIC_DRAW); // bind data statically here since it's unchanging + gl.enableVertexAttribArray(this._secondPass.aTexturePosition); + // set the matrix that transforms the framebuffer to clip space + let matrix = Mat3.makeScaling(2, 2).multiply(Mat3.makeTranslation(-0.5, -0.5)); + gl.uniformMatrix3fv(this._secondPass.uMatrix, false, matrix.values); } _resizeRenderer(){ @@ -621,10 +811,11 @@ gl.viewport(0, 0, w, h); //release the old texture - gl.deleteTexture(this._glTiledImageTexture); + gl.deleteTexture(this._renderToTexture); //create a new texture and set it up - this._glTiledImageTexture = gl.createTexture(); - gl.bindTexture(gl.TEXTURE_2D, this._glTiledImageTexture); + this._renderToTexture = gl.createTexture(); + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, this._renderToTexture); gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, w, h, 0, gl.RGBA, gl.UNSIGNED_BYTE, null); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); @@ -632,7 +823,7 @@ //bind the frame buffer to the new texture gl.bindFramebuffer(gl.FRAMEBUFFER, this._glFrameBuffer); - gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, this._glTiledImageTexture, 0); + gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, this._renderToTexture, 0); } @@ -714,7 +905,7 @@ position = this._makeQuadVertexBuffer(left, right, top, bottom); } else { // no overlap: this texture can use the unit quad as it's position data - position = this._glUnitQuad; + position = this._unitQuad; } let textureInfo = { @@ -724,7 +915,7 @@ // add it to our _TextureMap this._TextureMap.set(canvas, textureInfo); - + gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, texture); // Set the parameters so we can render any size image. gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); @@ -758,9 +949,17 @@ let gl = this._gl; let canvas = tileContext.canvas; - // This depends on gl.TEXTURE_2D being bound to the texture - // associated with this canvas before calling this function - gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, canvas); + try{ + if(!canvas){ + throw('Tile context does not have a canvas', tileContext); + } + // This depends on gl.TEXTURE_2D being bound to the texture + // associated with this canvas before calling this function + gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, canvas); + } catch (e){ + $.console.error('Error uploading image data to WebGL', e); + } + } @@ -772,13 +971,13 @@ _cleanupImageData(tileCanvas){ let textureInfo = this._TextureMap.get(tileCanvas); //remove from the map + this._TextureMap.delete(tileCanvas); + //release the texture from the GPU if(textureInfo){ this._gl.deleteTexture(textureInfo.texture); } - //release the texture from the GPU - this._gl.deleteTexture(textureInfo.texture); // release the position buffer from the GPU // TO DO: do this! } @@ -1033,14 +1232,6 @@ }; - // $.WebGLDrawer.Filter = class{ - // constructor(gl){ - // this.gl = gl; - // } - // apply(){ - - // } - // }; }( OpenSeadragon ));