From 89a625021ac94cd7b3576992a442dfcaebc967c9 Mon Sep 17 00:00:00 2001 From: romanb Date: Fri, 20 Mar 2009 20:53:14 +0000 Subject: [PATCH] [2.0] Refactored and reenabled Lexer tests. --- lib/Doctrine/ORM/Query/Lexer.php | 30 +- lib/Doctrine/ORM/Query/Parser.php | 11 +- tests/Doctrine/Tests/ORM/Query/AllTests.php | 3 +- tests/Doctrine/Tests/ORM/Query/LexerTest.php | 288 ++++++++++++++++++ .../Doctrine/Tests/ORM/Query/ScannerTest.php | 280 ----------------- 5 files changed, 323 insertions(+), 289 deletions(-) create mode 100644 tests/Doctrine/Tests/ORM/Query/LexerTest.php delete mode 100644 tests/Doctrine/Tests/ORM/Query/ScannerTest.php diff --git a/lib/Doctrine/ORM/Query/Lexer.php b/lib/Doctrine/ORM/Query/Lexer.php index 1b499235d..d42d9412b 100644 --- a/lib/Doctrine/ORM/Query/Lexer.php +++ b/lib/Doctrine/ORM/Query/Lexer.php @@ -114,7 +114,7 @@ class Lexer public $lookahead; /** - * @var array The last matched token. + * @var array The last matched/seen token. */ public $token; @@ -151,17 +151,43 @@ class Lexer * * @return array|null the next token; null if there is no more tokens left */ - public function next() + public function moveNext() { $this->token = $this->lookahead; $this->_peek = 0; if (isset($this->_tokens[$this->_position])) { $this->lookahead = $this->_tokens[$this->_position++]; + return true; } else { $this->lookahead = null; + return false; } } + /** + * Attempts to match the given token with the current lookahead token. + * + * If they match, the lexer moves on to the next token, otherwise a syntax error + * is raised. + * + * @param int|string token type or value + * @return bool True, if tokens match; false otherwise. + */ + /*public function match($token) + { + if (is_string($token)) { + $isMatch = ($this->lookahead['value'] === $token); + } else { + $isMatch = ($this->lookahead['type'] === $token); + } + + if ( ! $isMatch) { + $this->syntaxError($this->getLiteral($token)); + } + + $this->moveNext(); + }*/ + /** * Checks if an identifier is a keyword and returns its correct type. * diff --git a/lib/Doctrine/ORM/Query/Parser.php b/lib/Doctrine/ORM/Query/Parser.php index dddc18ff1..1a6303938 100644 --- a/lib/Doctrine/ORM/Query/Parser.php +++ b/lib/Doctrine/ORM/Query/Parser.php @@ -60,21 +60,21 @@ class Parser * * @var Doctrine_ORM_Query_Scanner */ - protected $_lexer; + private $_lexer; /** * The Parser Result object. * * @var Doctrine_ORM_Query_ParserResult */ - protected $_parserResult; + private $_parserResult; /** * The EntityManager. * * @var EnityManager */ - protected $_em; + private $_em; /** * Creates a new query parser object. @@ -131,8 +131,7 @@ class Parser $this->syntaxError($this->_lexer->getLiteral($token)); } - $this->_lexer->next(); - return true; + $this->_lexer->moveNext(); } public function isA($value, $token) @@ -322,7 +321,7 @@ class Parser */ private function _QueryLanguage() { - $this->_lexer->next(); + $this->_lexer->moveNext(); switch ($this->_lexer->lookahead['type']) { case Lexer::T_SELECT: return $this->_SelectStatement(); diff --git a/tests/Doctrine/Tests/ORM/Query/AllTests.php b/tests/Doctrine/Tests/ORM/Query/AllTests.php index 2e4740466..a07fc93b8 100644 --- a/tests/Doctrine/Tests/ORM/Query/AllTests.php +++ b/tests/Doctrine/Tests/ORM/Query/AllTests.php @@ -22,9 +22,10 @@ class AllTests $suite->addTestSuite('Doctrine\Tests\ORM\Query\IdentifierRecognitionTest'); $suite->addTestSuite('Doctrine\Tests\ORM\Query\SelectSqlGenerationTest'); $suite->addTestSuite('Doctrine\Tests\ORM\Query\LanguageRecognitionTest'); + $suite->addTestSuite('Doctrine\Tests\ORM\Query\LexerTest'); /* - $suite->addTestSuite('Orm_Query_ScannerTest'); + $suite->addTestSuite('Orm_Query_DqlGenerationTest'); $suite->addTestSuite('Orm_Query_DeleteSqlGenerationTest'); $suite->addTestSuite('Orm_Query_UpdateSqlGenerationTest');*/ diff --git a/tests/Doctrine/Tests/ORM/Query/LexerTest.php b/tests/Doctrine/Tests/ORM/Query/LexerTest.php new file mode 100644 index 000000000..f2ef583b4 --- /dev/null +++ b/tests/Doctrine/Tests/ORM/Query/LexerTest.php @@ -0,0 +1,288 @@ +moveNext(); + $token = $lexer->lookahead; + + $this->assertEquals(Lexer::T_IDENTIFIER, $token['type']); + $this->assertEquals('u', $token['value']); + } + + public function testScannerRecognizesIdentifierConsistingOfLetters() + { + $lexer = new Lexer('someIdentifier'); + + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_IDENTIFIER, $token['type']); + $this->assertEquals('someIdentifier', $token['value']); + } + + public function testScannerRecognizesIdentifierIncludingDigits() + { + $lexer = new Lexer('s0m31d3nt1f13r'); + + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_IDENTIFIER, $token['type']); + $this->assertEquals('s0m31d3nt1f13r', $token['value']); + } + + public function testScannerRecognizesIdentifierIncludingUnderscore() + { + $lexer = new Lexer('some_identifier'); + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_IDENTIFIER, $token['type']); + $this->assertEquals('some_identifier', $token['value']); + } + + public function testScannerRecognizesDecimalInteger() + { + $lexer = new Lexer('1234'); + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_INTEGER, $token['type']); + $this->assertEquals(1234, $token['value']); + } + + public function testScannerRecognizesFloat() + { + $lexer = new Lexer('1.234'); + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_FLOAT, $token['type']); + $this->assertEquals(1.234, $token['value']); + } + + public function testScannerRecognizesFloatWithExponent() + { + $lexer = new Lexer('1.2e3'); + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_FLOAT, $token['type']); + $this->assertEquals(1.2e3, $token['value']); + } + + public function testScannerRecognizesFloatWithExponent2() + { + $lexer = new Lexer('0.2e3'); + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_FLOAT, $token['type']); + $this->assertEquals(.2e3, $token['value']); + } + + public function testScannerRecognizesFloatWithNegativeExponent() + { + $lexer = new Lexer('7E-10'); + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_FLOAT, $token['type']); + $this->assertEquals(7E-10, $token['value']); + } + + public function testScannerRecognizesFloatBig() + { + $lexer = new Lexer('1,234,567.89'); + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_FLOAT, $token['type']); + $this->assertEquals(1.23456789e6, $token['value']); + } + + public function testScannerRecognizesFloatBigWrongPoint() + { + $lexer = new Lexer('12,34,56,7.89'); + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_FLOAT, $token['type']); + $this->assertEquals(1.23456789e6, $token['value']); + } + + public function testScannerRecognizesFloatLocaleSpecific() + { + $lexer = new Lexer('1,234'); + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_FLOAT, $token['type']); + $this->assertEquals(1.234, $token['value']); + } + + public function testScannerRecognizesFloatLocaleSpecificBig() + { + $lexer = new Lexer('1.234.567,89'); + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_FLOAT, $token['type']); + $this->assertEquals(1.23456789e6, $token['value']); + } + + public function testScannerRecognizesFloatLocaleSpecificBigWrongPoint() + { + $lexer = new Lexer('12.34.56.7,89'); + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_FLOAT, $token['type']); + $this->assertEquals(1.23456789e6, $token['value']); + } + + public function testScannerRecognizesFloatLocaleSpecificExponent() + { + $lexer = new Lexer('1,234e2'); + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_FLOAT, $token['type']); + $this->assertEquals(1.234e2, $token['value']); + } + + public function testScannerRecognizesFloatLocaleSpecificExponent2() + { + $lexer = new Lexer('0,234e2'); + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_FLOAT, $token['type']); + $this->assertEquals(.234e2, $token['value']); + } + + public function testScannerRecognizesFloatContainingWhitespace() + { + $lexer = new Lexer('- 1.234e2'); + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_NONE, $token['type']); + $this->assertEquals('-', $token['value']); + + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_FLOAT, $token['type']); + $this->assertNotEquals(-1.234e2, $token['value']); + $this->assertEquals(1.234e2, $token['value']); + } + + public function testScannerRecognizesStringContainingWhitespace() + { + $lexer = new Lexer("'This is a string.'"); + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_STRING, $token['type']); + $this->assertEquals("'This is a string.'", $token['value']); + } + + public function testScannerRecognizesStringContainingSingleQuotes() + { + $lexer = new Lexer("'abc''defg'''"); + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_STRING, $token['type']); + $this->assertEquals("'abc''defg'''", $token['value']); + } + + public function testScannerRecognizesInputParameter() + { + $lexer = new Lexer('?1'); + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_INPUT_PARAMETER, $token['type']); + $this->assertEquals('?1', $token['value']); + } + + public function testScannerRecognizesNamedInputParameter() + { + $lexer = new Lexer(':name'); + $lexer->moveNext(); + $token = $lexer->lookahead; + $this->assertEquals(Lexer::T_INPUT_PARAMETER, $token['type']); + $this->assertEquals(':name', $token['value']); + } + + public function testScannerTokenizesASimpleQueryCorrectly() + { + $dql = "SELECT u FROM My\Namespace\User u WHERE u.name = 'Jack O''Neil'"; + $lexer = new Lexer($dql); + + $tokens = array( + array( + 'value' => 'SELECT', + 'type' => Lexer::T_SELECT, + 'position' => 0 + ), + array( + 'value' => 'u', + 'type' => Lexer::T_IDENTIFIER, + 'position' => 7 + ), + array( + 'value' => 'FROM', + 'type' => Lexer::T_FROM, + 'position' => 9 + ), + array( + 'value' => 'My\Namespace\User', + 'type' => Lexer::T_IDENTIFIER, + 'position' => 14 + ), + array( + 'value' => 'u', + 'type' => Lexer::T_IDENTIFIER, + 'position' => 32 + ), + array( + 'value' => 'WHERE', + 'type' => Lexer::T_WHERE, + 'position' => 34 + ), + array( + 'value' => 'u', + 'type' => Lexer::T_IDENTIFIER, + 'position' => 40 + ), + array( + 'value' => '.', + 'type' => Lexer::T_NONE, + 'position' => 41 + ), + array( + 'value' => 'name', + 'type' => Lexer::T_IDENTIFIER, + 'position' => 42 + ), + array( + 'value' => '=', + 'type' => Lexer::T_NONE, + 'position' => 47 + ), + array( + 'value' => "'Jack O''Neil'", + 'type' => Lexer::T_STRING, + 'position' => 49 + ) + ); + + foreach ($tokens as $expected) { + $lexer->moveNext(); + $actual = $lexer->lookahead; + $this->assertEquals($expected['value'], $actual['value']); + $this->assertEquals($expected['type'], $actual['type']); + $this->assertEquals($expected['position'], $actual['position']); + } + + $this->assertFalse($lexer->moveNext()); + } +} diff --git a/tests/Doctrine/Tests/ORM/Query/ScannerTest.php b/tests/Doctrine/Tests/ORM/Query/ScannerTest.php deleted file mode 100644 index b092a3539..000000000 --- a/tests/Doctrine/Tests/ORM/Query/ScannerTest.php +++ /dev/null @@ -1,280 +0,0 @@ -next(); - $this->assertEquals(Doctrine_Query_Token::T_IDENTIFIER, $token['type']); - $this->assertEquals('u', $token['value']); - } - - public function testScannerRecognizesIdentifierConsistingOfLetters() - { - $scanner = new Doctrine_Query_Scanner('someIdentifier'); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_IDENTIFIER, $token['type']); - $this->assertEquals('someIdentifier', $token['value']); - } - - public function testScannerRecognizesIdentifierIncludingDigits() - { - $scanner = new Doctrine_Query_Scanner('s0m31d3nt1f13r'); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_IDENTIFIER, $token['type']); - $this->assertEquals('s0m31d3nt1f13r', $token['value']); - } - - public function testScannerRecognizesIdentifierIncludingUnderscore() - { - $scanner = new Doctrine_Query_Scanner('some_identifier'); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_IDENTIFIER, $token['type']); - $this->assertEquals('some_identifier', $token['value']); - } - - public function testScannerRecognizesDecimalInteger() - { - $scanner = new Doctrine_Query_Scanner('1234'); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_INTEGER, $token['type']); - $this->assertEquals(1234, $token['value']); - } - - public function testScannerRecognizesFloat() - { - $scanner = new Doctrine_Query_Scanner('1.234'); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']); - $this->assertEquals(1.234, $token['value']); - } - - public function testScannerRecognizesFloatWithExponent() - { - $scanner = new Doctrine_Query_Scanner('1.2e3'); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']); - $this->assertEquals(1.2e3, $token['value']); - } - - public function testScannerRecognizesFloatWithExponent2() - { - $scanner = new Doctrine_Query_Scanner('0.2e3'); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']); - $this->assertEquals(.2e3, $token['value']); - } - - public function testScannerRecognizesFloatWithNegativeExponent() - { - $scanner = new Doctrine_Query_Scanner('7E-10'); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']); - $this->assertEquals(7E-10, $token['value']); - } - - public function testScannerRecognizesFloatBig() - { - $scanner = new Doctrine_Query_Scanner('1,234,567.89'); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']); - $this->assertEquals(1.23456789e6, $token['value']); - } - - public function testScannerRecognizesFloatBigWrongPoint() - { - $scanner = new Doctrine_Query_Scanner('12,34,56,7.89'); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']); - $this->assertEquals(1.23456789e6, $token['value']); - } - - public function testScannerRecognizesFloatLocaleSpecific() - { - $scanner = new Doctrine_Query_Scanner('1,234'); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']); - $this->assertEquals(1.234, $token['value']); - } - - public function testScannerRecognizesFloatLocaleSpecificBig() - { - $scanner = new Doctrine_Query_Scanner('1.234.567,89'); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']); - $this->assertEquals(1.23456789e6, $token['value']); - } - - public function testScannerRecognizesFloatLocaleSpecificBigWrongPoint() - { - $scanner = new Doctrine_Query_Scanner('12.34.56.7,89'); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']); - $this->assertEquals(1.23456789e6, $token['value']); - } - - public function testScannerRecognizesFloatLocaleSpecificExponent() - { - $scanner = new Doctrine_Query_Scanner('1,234e2'); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']); - $this->assertEquals(1.234e2, $token['value']); - } - - public function testScannerRecognizesFloatLocaleSpecificExponent2() - { - $scanner = new Doctrine_Query_Scanner('0,234e2'); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']); - $this->assertEquals(.234e2, $token['value']); - } - - public function testScannerRecognizesFloatContainingWhitespace() - { - $scanner = new Doctrine_Query_Scanner('- 1.234e2'); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_NONE, $token['type']); - $this->assertEquals('-', $token['value']); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']); - $this->assertNotEquals(-1.234e2, $token['value']); - $this->assertEquals(1.234e2, $token['value']); - } - - public function testScannerRecognizesStringContainingWhitespace() - { - $scanner = new Doctrine_Query_Scanner("'This is a string.'"); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_STRING, $token['type']); - $this->assertEquals("'This is a string.'", $token['value']); - } - - public function testScannerRecognizesStringContainingSingleQuotes() - { - $scanner = new Doctrine_Query_Scanner("'abc''defg'''"); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_STRING, $token['type']); - $this->assertEquals("'abc''defg'''", $token['value']); - } - - public function testScannerRecognizesInputParameter() - { - $scanner = new Doctrine_Query_Scanner('?'); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_INPUT_PARAMETER, $token['type']); - $this->assertEquals('?', $token['value']); - } - - public function testScannerRecognizesNamedInputParameter() - { - $scanner = new Doctrine_Query_Scanner(':name'); - - $token = $scanner->next(); - $this->assertEquals(Doctrine_Query_Token::T_INPUT_PARAMETER, $token['type']); - $this->assertEquals(':name', $token['value']); - } - - public function testScannerTokenizesASimpleQueryCorrectly() - { - $dql = "SELECT u.* FROM User u WHERE u.name = 'Jack O''Neil'"; - $scanner = new Doctrine_Query_Scanner($dql); - - $tokens = array( - array( - 'value' => 'SELECT', - 'type' => Doctrine_Query_Token::T_SELECT, - 'position' => 0 - ), - array( - 'value' => 'u', - 'type' => Doctrine_Query_Token::T_IDENTIFIER, - 'position' => 7 - ), - array( - 'value' => '.', - 'type' => Doctrine_Query_Token::T_NONE, - 'position' => 8 - ), - array( - 'value' => '*', - 'type' => Doctrine_Query_Token::T_NONE, - 'position' => 9 - ), - array( - 'value' => 'FROM', - 'type' => Doctrine_Query_Token::T_FROM, - 'position' => 11 - ), - array( - 'value' => 'User', - 'type' => Doctrine_Query_Token::T_IDENTIFIER, - 'position' => 16 - ), - array( - 'value' => 'u', - 'type' => Doctrine_Query_Token::T_IDENTIFIER, - 'position' => 21 - ), - array( - 'value' => 'WHERE', - 'type' => Doctrine_Query_Token::T_WHERE, - 'position' => 23 - ), - array( - 'value' => 'u', - 'type' => Doctrine_Query_Token::T_IDENTIFIER, - 'position' => 29 - ), - array( - 'value' => '.', - 'type' => Doctrine_Query_Token::T_NONE, - 'position' => 30 - ), - array( - 'value' => 'name', - 'type' => Doctrine_Query_Token::T_IDENTIFIER, - 'position' => 31 - ), - array( - 'value' => '=', - 'type' => Doctrine_Query_Token::T_NONE, - 'position' => 36 - ), - array( - 'value' => "'Jack O''Neil'", - 'type' => Doctrine_Query_Token::T_STRING, - 'position' => 38 - ) - ); - - foreach ($tokens as $expected) { - $actual = $scanner->next(); - $this->assertEquals($expected['value'], $actual['value']); - $this->assertEquals($expected['type'], $actual['type']); - $this->assertEquals($expected['position'], $actual['position']); - } - - $this->assertNull($scanner->next()); - } -}