diff --git a/lib/Doctrine/ORM/Tools/Pagination/CountSqlWalker.php b/lib/Doctrine/ORM/Tools/Pagination/CountSqlWalker.php new file mode 100644 index 000000000..51e6c2e4b --- /dev/null +++ b/lib/Doctrine/ORM/Tools/Pagination/CountSqlWalker.php @@ -0,0 +1,120 @@ + FROM ()) + * + * Works with composite keys but cannot deal with queries that have multiple + * root entities (e.g. `SELECT f, b from Foo, Bar`) + * + * @author Sander Marechal + */ +class CountSqlWalker extends SqlWalker +{ + /** + * @var Doctrine\DBAL\Platforms\AbstractPlatform + */ + private $platform; + + /** + * @var Doctrine\ORM\Query\ResultSetMapping + */ + private $rsm; + + /** + * @var array + */ + private $queryComponents; + + /** + * Constructor. Stores various parameters that are otherwise unavailable + * because Doctrine\ORM\Query\SqlWalker keeps everything private without + * accessors. + * + * @param Doctrine\ORM\Query $query + * @param Doctrine\ORM\Query\ParserResult $parserResult + * @param array $queryComponents + */ + public function __construct($query, $parserResult, array $queryComponents) + { + $this->platform = $query->getEntityManager()->getConnection()->getDatabasePlatform(); + $this->rsm = $parserResult->getResultSetMapping(); + $this->queryComponents = $queryComponents; + + parent::__construct($query, $parserResult, $queryComponents); + } + + /** + * Walks down a SelectStatement AST node, wrapping it in a COUNT (SELECT DISTINCT) + * + * Note that the ORDER BY clause is not removed. Many SQL implementations (e.g. MySQL) + * are able to cache subqueries. By keeping the ORDER BY clause intact, the limitSubQuery + * that will most likely be executed next can be read from the native SQL cache. + * + * @param SelectStatement $AST + * @return string + */ + public function walkSelectStatement(SelectStatement $AST) + { + $sql = parent::walkSelectStatement($AST); + + // Find out the SQL alias of the identifier column of the root entity + // It may be possible to make this work with multiple root entities but that + // would probably require issuing multiple queries or doing a UNION SELECT + // so for now, It's not supported. + + // Get the root entity and alias from the AST fromClause + $from = $AST->fromClause->identificationVariableDeclarations; + if (count($from) > 1) { + throw new \RuntimeException("Cannot count query which selects two FROM components, cannot make distinction"); + } + + $rootClass = $from[0]->rangeVariableDeclaration->abstractSchemaName; + $rootAlias = $from[0]->rangeVariableDeclaration->aliasIdentificationVariable; + + // Get the identity properties from the metadata + $rootIdentifier = $this->queryComponents[$rootAlias]['metadata']->identifier; + + // For every identifier, find out the SQL alias by combing through the ResultSetMapping + $sqlIdentifier = array(); + foreach ($rootIdentifier as $property) { + foreach (array_keys($this->rsm->fieldMappings, $property) as $alias) { + if ($this->rsm->columnOwnerMap[$alias] == $rootAlias) { + $sqlIdentifier[$property] = $alias; + } + } + } + + if (count($rootIdentifier) != count($sqlIdentifier)) { + throw new \RuntimeException(sprintf( + 'Not all identifier properties can be found in the ResultSetMapping: %s', + implode(', ', array_diff($rootIdentifier, array_keys($sqlIdentifier))) + )); + } + + // Build the counter query + return sprintf('SELECT %s AS _dctrn_count FROM (SELECT DISTINCT %s FROM (%s) AS _dctrn_result) AS _dctrn_table', + $this->platform->getCountExpression('*'), + implode(', ', $sqlIdentifier), + $sql + ); + } +} diff --git a/lib/Doctrine/ORM/Tools/Pagination/LimitSubquerySqlWalker.php b/lib/Doctrine/ORM/Tools/Pagination/LimitSubquerySqlWalker.php new file mode 100644 index 000000000..cbea28c65 --- /dev/null +++ b/lib/Doctrine/ORM/Tools/Pagination/LimitSubquerySqlWalker.php @@ -0,0 +1,144 @@ + FROM () LIMIT x OFFSET y + * + * Works with composite keys but cannot deal with queries that have multiple + * root entities (e.g. `SELECT f, b from Foo, Bar`) + * + * @author Sander Marechal + */ +class LimitSubquerySqlWalker extends SqlWalker +{ + /** + * @var Doctrine\DBAL\Platforms\AbstractPlatform + */ + private $platform; + + /** + * @var Doctrine\ORM\Query\ResultSetMapping + */ + private $rsm; + + /** + * @var array + */ + private $queryComponents; + + /** + * @var int + */ + private $firstResult; + + /** + * @var int + */ + private $maxResults; + + /** + * Constructor. Stores various parameters that are otherwise unavailable + * because Doctrine\ORM\Query\SqlWalker keeps everything private without + * accessors. + * + * @param Doctrine\ORM\Query $query + * @param Doctrine\ORM\Query\ParserResult $parserResult + * @param array $queryComponents + */ + public function __construct($query, $parserResult, array $queryComponents) + { + $this->platform = $query->getEntityManager()->getConnection()->getDatabasePlatform(); + $this->rsm = $parserResult->getResultSetMapping(); + $this->queryComponents = $queryComponents; + + // Reset limit and offset + $this->firstResult = $query->getFirstResult(); + $this->maxResults = $query->getMaxResults(); + $query->setFirstResult(null)->setMaxResults(null); + + parent::__construct($query, $parserResult, $queryComponents); + } + + /** + * Walks down a SelectStatement AST node, wrapping it in a SELECT DISTINCT + * + * @param SelectStatement $AST + * @return string + */ + public function walkSelectStatement(SelectStatement $AST) + { + $sql = parent::walkSelectStatement($AST); + + // Find out the SQL alias of the identifier column of the root entity + // It may be possible to make this work with multiple root entities but that + // would probably require issuing multiple queries or doing a UNION SELECT + // so for now, It's not supported. + + // Get the root entity and alias from the AST fromClause + $from = $AST->fromClause->identificationVariableDeclarations; + if (count($from) !== 1) { + throw new \RuntimeException("Cannot count query which selects two FROM components, cannot make distinction"); + } + + $rootClass = $from[0]->rangeVariableDeclaration->abstractSchemaName; + $rootAlias = $from[0]->rangeVariableDeclaration->aliasIdentificationVariable; + + // Get the identity properties from the metadata + $metadata = $this->queryComponents[$rootAlias]['metadata']; + $rootIdentifier = $metadata->identifier; + + // For every identifier, find out the SQL alias by combing through the ResultSetMapping + $sqlIdentifier = array(); + foreach ($rootIdentifier as $property) { + foreach (array_keys($this->rsm->fieldMappings, $property) as $alias) { + if ($this->rsm->columnOwnerMap[$alias] == $rootAlias) { + $sqlIdentifier[$property] = $alias; + } + } + } + + if (count($rootIdentifier) != count($sqlIdentifier)) { + throw new \RuntimeException(sprintf( + 'Not all identifier properties can be found in the ResultSetMapping: %s', + implode(', ', array_diff($rootIdentifier, array_keys($sqlIdentifier))) + )); + } + + // Build the counter query + $sql = sprintf('SELECT DISTINCT %s FROM (%s) AS _dctrn_result', + implode(', ', $sqlIdentifier), $sql); + + // Apply the limit and offset + $sql = $this->platform->modifyLimitQuery( + $sql, $this->maxResults, $this->firstResult + ); + + // Add the columns to the ResultSetMapping. It's not really nice but + // it works. Preferably I'd clear the RSM or simply create a new one + // but that is not possible from inside the output walker, so we dirty + // up the one we have. + foreach ($sqlIdentifier as $property => $alias) { + $this->rsm->addScalarResult($alias, $property); + } + + return $sql; + } +} diff --git a/lib/Doctrine/ORM/Tools/Pagination/Paginator.php b/lib/Doctrine/ORM/Tools/Pagination/Paginator.php index 760d7de13..fbd004b3c 100644 --- a/lib/Doctrine/ORM/Tools/Pagination/Paginator.php +++ b/lib/Doctrine/ORM/Tools/Pagination/Paginator.php @@ -21,9 +21,12 @@ namespace Doctrine\ORM\Tools\Pagination; use Doctrine\ORM\QueryBuilder; use Doctrine\ORM\Query; +use Doctrine\ORM\Query\ResultSetMapping; use Doctrine\ORM\NoResultException; use Doctrine\ORM\Tools\Pagination\WhereInWalker; +use Doctrine\ORM\Tools\Pagination\WhereInSqlWalker; use Doctrine\ORM\Tools\Pagination\CountWalker; +use Doctrine\ORM\Tools\Pagination\CountSqlWalker; use Countable; use IteratorAggregate; use ArrayIterator; @@ -49,6 +52,11 @@ class Paginator implements \Countable, \IteratorAggregate */ private $fetchJoinCollection; + /** + * @var bool|null + */ + private $useSqlWalkers; + /** * @var int */ @@ -90,6 +98,28 @@ class Paginator implements \Countable, \IteratorAggregate return $this->fetchJoinCollection; } + /** + * Returns whether the paginator will use an SQL TreeWalker + * + * @return bool|null + */ + public function getUseSqlWalkers() + { + return $this->useSqlWalkers; + } + + /** + * Set whether the paginator will use an SQL TreeWalker + * + * @param bool|null $useSqlWalkers + * @return $this + */ + public function setUseSqlWalkers($useSqlWalkers) + { + $this->useSqlWalkers = $useSqlWalkers; + return $this; + } + /** * {@inheritdoc} */ @@ -103,7 +133,16 @@ class Paginator implements \Countable, \IteratorAggregate $countQuery->setHint(CountWalker::HINT_DISTINCT, true); } - $countQuery->setHint(Query::HINT_CUSTOM_TREE_WALKERS, array('Doctrine\ORM\Tools\Pagination\CountWalker')); + if ($this->useSqlWalker($countQuery)) { + $rsm = new ResultSetMapping(); + $rsm->addScalarResult('_dctrn_count', 'count'); + + $countQuery->setHint(Query::HINT_CUSTOM_OUTPUT_WALKER, 'Doctrine\ORM\Tools\Pagination\CountSqlWalker'); + $countQuery->setResultSetMapping($rsm); + } else { + $countQuery->setHint(Query::HINT_CUSTOM_TREE_WALKERS, array('Doctrine\ORM\Tools\Pagination\CountWalker')); + } + $countQuery->setFirstResult(null)->setMaxResults(null); try { @@ -127,9 +166,14 @@ class Paginator implements \Countable, \IteratorAggregate if ($this->fetchJoinCollection) { $subQuery = $this->cloneQuery($this->query); - $subQuery->setHint(Query::HINT_CUSTOM_TREE_WALKERS, array('Doctrine\ORM\Tools\Pagination\LimitSubqueryWalker')) - ->setFirstResult($offset) - ->setMaxResults($length); + + if ($this->useSqlWalker($subQuery)) { + $subQuery->setHint(Query::HINT_CUSTOM_OUTPUT_WALKER, 'Doctrine\ORM\Tools\Pagination\LimitSubquerySqlWalker'); + } else { + $subQuery->setHint(Query::HINT_CUSTOM_TREE_WALKERS, array('Doctrine\ORM\Tools\Pagination\LimitSubqueryWalker')); + } + + $subQuery->setFirstResult($offset)->setMaxResults($length); $ids = array_map('current', $subQuery->getScalarResult()); @@ -176,5 +220,21 @@ class Paginator implements \Countable, \IteratorAggregate return $cloneQuery; } + + /** + * Determine whether to use an SQL TreeWalker for the query + * + * @param Query $query The query. + * + * @return bool + */ + private function useSqlWalker(Query $query) + { + if ($this->useSqlWalkers === null) { + return (Boolean) $query->getHint(Query::HINT_CUSTOM_OUTPUT_WALKER) == false; + } + + return $this->useSqlWalkers; + } } diff --git a/tests/Doctrine/Tests/ORM/Functional/PaginationTest.php b/tests/Doctrine/Tests/ORM/Functional/PaginationTest.php index 990353519..cc5354dce 100644 --- a/tests/Doctrine/Tests/ORM/Functional/PaginationTest.php +++ b/tests/Doctrine/Tests/ORM/Functional/PaginationTest.php @@ -30,6 +30,7 @@ class PaginationTest extends \Doctrine\Tests\OrmFunctionalTestCase $query = $this->_em->createQuery($dql); $paginator = new Paginator($query); + $paginator->setUseSqlWalkers(false); $this->assertEquals(3, count($paginator)); } @@ -39,15 +40,27 @@ class PaginationTest extends \Doctrine\Tests\OrmFunctionalTestCase $query = $this->_em->createQuery($dql); $paginator = new Paginator($query); + $paginator->setUseSqlWalkers(false); $this->assertEquals(3, count($paginator)); } + public function testCountComplexWithSqlWalker() + { + $dql = "SELECT g, COUNT(u.id) AS userCount FROM Doctrine\Tests\Models\CMS\CmsGroup g LEFT JOIN g.users u GROUP BY g.id HAVING userCount > 0"; + $query = $this->_em->createQuery($dql); + + $paginator = new Paginator($query); + $paginator->setUseSqlWalkers(true); + $this->assertEquals(9, count($paginator)); + } + public function testIterateSimpleWithoutJoinFetchJoinHandlingOff() { $dql = "SELECT u FROM Doctrine\Tests\Models\CMS\CmsUser u"; $query = $this->_em->createQuery($dql); $paginator = new Paginator($query, false); + $paginator->setUseSqlWalkers(false); $data = array(); foreach ($paginator as $user) { @@ -62,6 +75,7 @@ class PaginationTest extends \Doctrine\Tests\OrmFunctionalTestCase $query = $this->_em->createQuery($dql); $paginator = new Paginator($query, true); + $paginator->setUseSqlWalkers(false); $data = array(); foreach ($paginator as $user) { @@ -76,6 +90,7 @@ class PaginationTest extends \Doctrine\Tests\OrmFunctionalTestCase $query = $this->_em->createQuery($dql); $paginator = new Paginator($query, true); + $paginator->setUseSqlWalkers(false); $data = array(); foreach ($paginator as $user) { @@ -84,6 +99,40 @@ class PaginationTest extends \Doctrine\Tests\OrmFunctionalTestCase $this->assertEquals(3, count($data)); } + public function testIterateComplexWithSqlWalker() + { + $dql = "SELECT g, COUNT(u.id) AS userCount FROM Doctrine\Tests\Models\CMS\CmsGroup g LEFT JOIN g.users u GROUP BY g.id HAVING userCount > 0"; + $query = $this->_em->createQuery($dql); + + $paginator = new Paginator($query); + $paginator->setUseSqlWalkers(true); + + $data = array(); + foreach ($paginator as $user) { + $data[] = $user; + } + $this->assertEquals(9, count($data)); + } + + public function testDetectSqlWalker() + { + // This query works using the SQL walkers but causes an exception using the TreeWalker + $dql = "SELECT g, COUNT(u.id) AS userCount FROM Doctrine\Tests\Models\CMS\CmsGroup g LEFT JOIN g.users u GROUP BY g.id HAVING userCount > 0"; + $query = $this->_em->createQuery($dql); + + // If the Paginator detects the custom SQL walker it should fall back to using the + // Tree walkers for pagination, which leads to an exception. If the query works, the SQL walkers were used + $query->setHint(Query::HINT_CUSTOM_OUTPUT_WALKER, 'Doctrine\ORM\Query\SqlWalker'); + $paginator = new Paginator($query); + + try { + count($paginator); + $this->fail('Paginator did not detect custom SQL walker'); + } catch (\PHPUnit_Framework_Error_Notice $e) { + $this->assertEquals('Undefined index: userCount', $e->getMessage()); + } + } + public function populate() { for ($i = 0; $i < 3; $i++) { diff --git a/tests/Doctrine/Tests/ORM/Tools/Pagination/CountSqlWalkerTest.php b/tests/Doctrine/Tests/ORM/Tools/Pagination/CountSqlWalkerTest.php new file mode 100644 index 000000000..79f9e6f6d --- /dev/null +++ b/tests/Doctrine/Tests/ORM/Tools/Pagination/CountSqlWalkerTest.php @@ -0,0 +1,48 @@ +entityManager->createQuery( + 'SELECT p, c, a FROM Doctrine\Tests\ORM\Tools\Pagination\BlogPost p JOIN p.category c JOIN p.author a'); + $query->setHint(Query::HINT_CUSTOM_OUTPUT_WALKER, 'Doctrine\ORM\Tools\Pagination\CountSqlWalker'); + $query->setHint(CountWalker::HINT_DISTINCT, true); + $query->setFirstResult(null)->setMaxResults(null); + + $this->assertEquals( + "SELECT COUNT(*) AS _dctrn_count FROM (SELECT DISTINCT id0 FROM (SELECT b0_.id AS id0, c1_.id AS id1, a2_.id AS id2, a2_.name AS name3, b0_.author_id AS author_id4, b0_.category_id AS category_id5 FROM BlogPost b0_ INNER JOIN Category c1_ ON b0_.category_id = c1_.id INNER JOIN Author a2_ ON b0_.author_id = a2_.id) AS _dctrn_result) AS _dctrn_table", $query->getSql() + ); + } + + public function testCountQuery_MixedResultsWithName() + { + $query = $this->entityManager->createQuery( + 'SELECT a, sum(a.name) as foo FROM Doctrine\Tests\ORM\Tools\Pagination\Author a'); + $query->setHint(Query::HINT_CUSTOM_OUTPUT_WALKER, 'Doctrine\ORM\Tools\Pagination\CountSqlWalker'); + $query->setHint(CountWalker::HINT_DISTINCT, true); + $query->setFirstResult(null)->setMaxResults(null); + + $this->assertEquals( + "SELECT COUNT(*) AS _dctrn_count FROM (SELECT DISTINCT id0 FROM (SELECT a0_.id AS id0, a0_.name AS name1, sum(a0_.name) AS sclr2 FROM Author a0_) AS _dctrn_result) AS _dctrn_table", $query->getSql() + ); + } + + public function testCountQuery_Having() + { + $query = $this->entityManager->createQuery( + 'SELECT g, u, count(u.id) AS userCount FROM Doctrine\Tests\ORM\Tools\Pagination\Group g LEFT JOIN g.users u GROUP BY g.id HAVING userCount > 0'); + $query->setHint(Query::HINT_CUSTOM_OUTPUT_WALKER, 'Doctrine\ORM\Tools\Pagination\CountSqlWalker'); + $query->setFirstResult(null)->setMaxResults(null); + + $this->assertEquals( + "SELECT COUNT(*) AS _dctrn_count FROM (SELECT DISTINCT id1 FROM (SELECT count(u0_.id) AS sclr0, g1_.id AS id1, u0_.id AS id2 FROM groups g1_ LEFT JOIN user_group u2_ ON g1_.id = u2_.group_id LEFT JOIN User u0_ ON u0_.id = u2_.user_id GROUP BY g1_.id HAVING sclr0 > 0) AS _dctrn_result) AS _dctrn_table", $query->getSql() + ); + } +} + diff --git a/tests/Doctrine/Tests/ORM/Tools/Pagination/LimitSubquerySqlWalkerTest.php b/tests/Doctrine/Tests/ORM/Tools/Pagination/LimitSubquerySqlWalkerTest.php new file mode 100644 index 000000000..8bf3b7a18 --- /dev/null +++ b/tests/Doctrine/Tests/ORM/Tools/Pagination/LimitSubquerySqlWalkerTest.php @@ -0,0 +1,33 @@ +entityManager->createQuery( + 'SELECT p, c, a FROM Doctrine\Tests\ORM\Tools\Pagination\MyBlogPost p JOIN p.category c JOIN p.author a'); + $limitQuery = clone $query; + $limitQuery->setHint(Query::HINT_CUSTOM_OUTPUT_WALKER, 'Doctrine\ORM\Tools\Pagination\LimitSubquerySqlWalker'); + + $this->assertEquals( + "SELECT DISTINCT id0 FROM (SELECT m0_.id AS id0, c1_.id AS id1, a2_.id AS id2, a2_.name AS name3, m0_.author_id AS author_id4, m0_.category_id AS category_id5 FROM MyBlogPost m0_ INNER JOIN Category c1_ ON m0_.category_id = c1_.id INNER JOIN Author a2_ ON m0_.author_id = a2_.id) AS _dctrn_result", $limitQuery->getSql() + ); + } + + public function testCountQuery_MixedResultsWithName() + { + $query = $this->entityManager->createQuery( + 'SELECT a, sum(a.name) as foo FROM Doctrine\Tests\ORM\Tools\Pagination\Author a'); + $limitQuery = clone $query; + $limitQuery->setHint(Query::HINT_CUSTOM_OUTPUT_WALKER, 'Doctrine\ORM\Tools\Pagination\LimitSubquerySqlWalker'); + + $this->assertEquals( + "SELECT DISTINCT id0 FROM (SELECT a0_.id AS id0, a0_.name AS name1, sum(a0_.name) AS sclr2 FROM Author a0_) AS _dctrn_result", $limitQuery->getSql() + ); + } +} +