[2.0] Refactored and reenabled Lexer tests.

lib/Doctrine/ORM/Query/Lexer.php

@@ -114,7 +114,7 @@ class Lexer
     public $lookahead;
 
     /**
-     * @var array The last matched token.
+     * @var array The last matched/seen token.
      */
     public $token;
 
@@ -151,17 +151,43 @@ class Lexer
      *
      * @return array|null the next token; null if there is no more tokens left
      */
-    public function next()
+    public function moveNext()
     {
         $this->token = $this->lookahead;
         $this->_peek = 0;
         if (isset($this->_tokens[$this->_position])) {
             $this->lookahead = $this->_tokens[$this->_position++];
+            return true;
         } else {
             $this->lookahead = null;
+            return false;
         }
     }
 
+    /**
+     * Attempts to match the given token with the current lookahead token.
+     *
+     * If they match, the lexer moves on to the next token, otherwise a syntax error
+     * is raised.
+     *
+     * @param int|string token type or value
+     * @return bool True, if tokens match; false otherwise.
+     */
+    /*public function match($token)
+    {
+        if (is_string($token)) {
+            $isMatch = ($this->lookahead['value'] === $token);
+        } else {
+            $isMatch = ($this->lookahead['type'] === $token);
+        }
+
+        if ( ! $isMatch) {
+            $this->syntaxError($this->getLiteral($token));
+        }
+
+        $this->moveNext();
+    }*/
+
     /**
      * Checks if an identifier is a keyword and returns its correct type.
      *

lib/Doctrine/ORM/Query/Parser.php

@@ -60,21 +60,21 @@ class Parser
      *
      * @var Doctrine_ORM_Query_Scanner
      */
-    protected $_lexer;
+    private $_lexer;
 
     /**
      * The Parser Result object.
      *
      * @var Doctrine_ORM_Query_ParserResult
      */
-    protected $_parserResult;
+    private $_parserResult;
     
     /**
      * The EntityManager.
      *
      * @var EnityManager
      */
-    protected $_em;
+    private $_em;
 
     /**
      * Creates a new query parser object.
@@ -131,8 +131,7 @@ class Parser
             $this->syntaxError($this->_lexer->getLiteral($token));
         }
 
-        $this->_lexer->next();
+        $this->_lexer->moveNext();
-        return true;
     }
 
     public function isA($value, $token)
@@ -322,7 +321,7 @@ class Parser
      */
     private function _QueryLanguage()
     {
-        $this->_lexer->next();
+        $this->_lexer->moveNext();
         switch ($this->_lexer->lookahead['type']) {
             case Lexer::T_SELECT:
                 return $this->_SelectStatement();

tests/Doctrine/Tests/ORM/Query/AllTests.php

@@ -22,9 +22,10 @@ class AllTests
         $suite->addTestSuite('Doctrine\Tests\ORM\Query\IdentifierRecognitionTest');
         $suite->addTestSuite('Doctrine\Tests\ORM\Query\SelectSqlGenerationTest');
         $suite->addTestSuite('Doctrine\Tests\ORM\Query\LanguageRecognitionTest');
+        $suite->addTestSuite('Doctrine\Tests\ORM\Query\LexerTest');
         
         /*
-        $suite->addTestSuite('Orm_Query_ScannerTest');
+        
         $suite->addTestSuite('Orm_Query_DqlGenerationTest');
         $suite->addTestSuite('Orm_Query_DeleteSqlGenerationTest');
         $suite->addTestSuite('Orm_Query_UpdateSqlGenerationTest');*/

tests/Doctrine/Tests/ORM/Query/LexerTest.php

@@ -0,0 +1,288 @@
+<?php
+
+namespace Doctrine\Tests\ORM\Query;
+
+use Doctrine\ORM\Query\Lexer;
+
+require_once __DIR__ . '/../../TestInit.php';
+
+class LexerTest extends \Doctrine\Tests\OrmTestCase
+{
+    //private $_lexer;
+
+    protected function setUp() {
+    }
+
+    public function testScannerRecognizesIdentifierWithLengthOfOneCharacter()
+    {
+        $lexer = new Lexer('u');
+        
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+
+        $this->assertEquals(Lexer::T_IDENTIFIER, $token['type']);
+        $this->assertEquals('u', $token['value']);
+    }
+
+    public function testScannerRecognizesIdentifierConsistingOfLetters()
+    {
+        $lexer = new Lexer('someIdentifier');
+
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_IDENTIFIER, $token['type']);
+        $this->assertEquals('someIdentifier', $token['value']);
+    }
+
+    public function testScannerRecognizesIdentifierIncludingDigits()
+    {
+        $lexer = new Lexer('s0m31d3nt1f13r');
+
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_IDENTIFIER, $token['type']);
+        $this->assertEquals('s0m31d3nt1f13r', $token['value']);
+    }
+
+    public function testScannerRecognizesIdentifierIncludingUnderscore()
+    {
+        $lexer = new Lexer('some_identifier');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_IDENTIFIER, $token['type']);
+        $this->assertEquals('some_identifier', $token['value']);
+    }
+
+    public function testScannerRecognizesDecimalInteger()
+    {
+        $lexer = new Lexer('1234');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_INTEGER, $token['type']);
+        $this->assertEquals(1234, $token['value']);
+    }
+
+    public function testScannerRecognizesFloat()
+    {
+        $lexer = new Lexer('1.234');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
+        $this->assertEquals(1.234, $token['value']);
+    }
+
+    public function testScannerRecognizesFloatWithExponent()
+    {
+        $lexer = new Lexer('1.2e3');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
+        $this->assertEquals(1.2e3, $token['value']);
+    }
+
+    public function testScannerRecognizesFloatWithExponent2()
+    {
+        $lexer = new Lexer('0.2e3');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
+        $this->assertEquals(.2e3, $token['value']);
+    }
+
+    public function testScannerRecognizesFloatWithNegativeExponent()
+    {
+        $lexer = new Lexer('7E-10');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
+        $this->assertEquals(7E-10, $token['value']);
+    }
+
+    public function testScannerRecognizesFloatBig()
+    {
+        $lexer = new Lexer('1,234,567.89');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
+        $this->assertEquals(1.23456789e6, $token['value']);
+    }
+
+    public function testScannerRecognizesFloatBigWrongPoint()
+    {
+        $lexer = new Lexer('12,34,56,7.89');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
+        $this->assertEquals(1.23456789e6, $token['value']);
+    }
+
+    public function testScannerRecognizesFloatLocaleSpecific()
+    {
+        $lexer = new Lexer('1,234');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
+        $this->assertEquals(1.234, $token['value']);
+    }
+
+    public function testScannerRecognizesFloatLocaleSpecificBig()
+    {
+        $lexer = new Lexer('1.234.567,89');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
+        $this->assertEquals(1.23456789e6, $token['value']);
+    }
+
+    public function testScannerRecognizesFloatLocaleSpecificBigWrongPoint()
+    {
+        $lexer = new Lexer('12.34.56.7,89');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
+        $this->assertEquals(1.23456789e6, $token['value']);
+    }
+
+    public function testScannerRecognizesFloatLocaleSpecificExponent()
+    {
+        $lexer = new Lexer('1,234e2');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
+        $this->assertEquals(1.234e2, $token['value']);
+    }
+
+    public function testScannerRecognizesFloatLocaleSpecificExponent2()
+    {
+        $lexer = new Lexer('0,234e2');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
+        $this->assertEquals(.234e2, $token['value']);
+    }
+
+    public function testScannerRecognizesFloatContainingWhitespace()
+    {
+        $lexer = new Lexer('-   1.234e2');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_NONE, $token['type']);
+        $this->assertEquals('-', $token['value']);
+
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_FLOAT, $token['type']);
+        $this->assertNotEquals(-1.234e2, $token['value']);
+        $this->assertEquals(1.234e2, $token['value']);
+    }
+
+    public function testScannerRecognizesStringContainingWhitespace()
+    {
+        $lexer = new Lexer("'This is a string.'");
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_STRING, $token['type']);
+        $this->assertEquals("'This is a string.'", $token['value']);
+    }
+
+    public function testScannerRecognizesStringContainingSingleQuotes()
+    {
+        $lexer = new Lexer("'abc''defg'''");
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_STRING, $token['type']);
+        $this->assertEquals("'abc''defg'''", $token['value']);
+    }
+
+    public function testScannerRecognizesInputParameter()
+    {
+        $lexer = new Lexer('?1');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_INPUT_PARAMETER, $token['type']);
+        $this->assertEquals('?1', $token['value']);
+    }
+
+    public function testScannerRecognizesNamedInputParameter()
+    {
+        $lexer = new Lexer(':name');
+        $lexer->moveNext();
+        $token = $lexer->lookahead;
+        $this->assertEquals(Lexer::T_INPUT_PARAMETER, $token['type']);
+        $this->assertEquals(':name', $token['value']);
+    }
+
+    public function testScannerTokenizesASimpleQueryCorrectly()
+    {
+        $dql = "SELECT u FROM My\Namespace\User u WHERE u.name = 'Jack O''Neil'";
+        $lexer = new Lexer($dql);
+
+        $tokens = array(
+            array(
+                'value' => 'SELECT',
+                'type'  => Lexer::T_SELECT,
+                'position' => 0
+            ),
+            array(
+                'value' => 'u',
+                'type'  => Lexer::T_IDENTIFIER,
+                'position' => 7
+            ),
+            array(
+                'value' => 'FROM',
+                'type'  => Lexer::T_FROM,
+                'position' => 9
+            ),
+            array(
+                'value' => 'My\Namespace\User',
+                'type'  => Lexer::T_IDENTIFIER,
+                'position' => 14
+            ),
+            array(
+                'value' => 'u',
+                'type'  => Lexer::T_IDENTIFIER,
+                'position' => 32
+            ),
+            array(
+                'value' => 'WHERE',
+                'type'  => Lexer::T_WHERE,
+                'position' => 34
+            ),
+            array(
+                'value' => 'u',
+                'type'  => Lexer::T_IDENTIFIER,
+                'position' => 40
+            ),
+            array(
+                'value' => '.',
+                'type'  => Lexer::T_NONE,
+                'position' => 41
+            ),
+            array(
+                'value' => 'name',
+                'type'  => Lexer::T_IDENTIFIER,
+                'position' => 42
+            ),
+            array(
+                'value' => '=',
+                'type'  => Lexer::T_NONE,
+                'position' => 47
+            ),
+            array(
+                'value' => "'Jack O''Neil'",
+                'type'  => Lexer::T_STRING,
+                'position' => 49
+            )
+        );
+
+        foreach ($tokens as $expected) {
+            $lexer->moveNext();
+            $actual = $lexer->lookahead;
+            $this->assertEquals($expected['value'], $actual['value']);
+            $this->assertEquals($expected['type'], $actual['type']);
+            $this->assertEquals($expected['position'], $actual['position']);
+        }
+
+        $this->assertFalse($lexer->moveNext());
+    }
+}

tests/Doctrine/Tests/ORM/Query/ScannerTest.php

@@ -1,280 +0,0 @@
-<?php
-class Orm_Query_ScannerTest extends Doctrine_OrmTestCase
-{
-    public function testScannerRecognizesIdentifierWithLengthOfOneCharacter()
-    {
-        $scanner = new Doctrine_Query_Scanner('u');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_IDENTIFIER, $token['type']);
-        $this->assertEquals('u', $token['value']);
-    }
-
-    public function testScannerRecognizesIdentifierConsistingOfLetters()
-    {
-        $scanner = new Doctrine_Query_Scanner('someIdentifier');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_IDENTIFIER, $token['type']);
-        $this->assertEquals('someIdentifier', $token['value']);
-    }
-
-    public function testScannerRecognizesIdentifierIncludingDigits()
-    {
-        $scanner = new Doctrine_Query_Scanner('s0m31d3nt1f13r');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_IDENTIFIER, $token['type']);
-        $this->assertEquals('s0m31d3nt1f13r', $token['value']);
-    }
-
-    public function testScannerRecognizesIdentifierIncludingUnderscore()
-    {
-        $scanner = new Doctrine_Query_Scanner('some_identifier');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_IDENTIFIER, $token['type']);
-        $this->assertEquals('some_identifier', $token['value']);
-    }
-
-    public function testScannerRecognizesDecimalInteger()
-    {
-        $scanner = new Doctrine_Query_Scanner('1234');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_INTEGER, $token['type']);
-        $this->assertEquals(1234, $token['value']);
-    }
-
-    public function testScannerRecognizesFloat()
-    {
-        $scanner = new Doctrine_Query_Scanner('1.234');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
-        $this->assertEquals(1.234, $token['value']);
-    }
-
-    public function testScannerRecognizesFloatWithExponent()
-    {
-        $scanner = new Doctrine_Query_Scanner('1.2e3');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
-        $this->assertEquals(1.2e3, $token['value']);
-    }
-
-    public function testScannerRecognizesFloatWithExponent2()
-    {
-        $scanner = new Doctrine_Query_Scanner('0.2e3');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
-        $this->assertEquals(.2e3, $token['value']);
-    }
-
-    public function testScannerRecognizesFloatWithNegativeExponent()
-    {
-        $scanner = new Doctrine_Query_Scanner('7E-10');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
-        $this->assertEquals(7E-10, $token['value']);
-    }
-
-    public function testScannerRecognizesFloatBig()
-    {
-        $scanner = new Doctrine_Query_Scanner('1,234,567.89');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
-        $this->assertEquals(1.23456789e6, $token['value']);
-    }
-
-    public function testScannerRecognizesFloatBigWrongPoint()
-    {
-        $scanner = new Doctrine_Query_Scanner('12,34,56,7.89');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
-        $this->assertEquals(1.23456789e6, $token['value']);
-    }
-
-    public function testScannerRecognizesFloatLocaleSpecific()
-    {
-        $scanner = new Doctrine_Query_Scanner('1,234');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
-        $this->assertEquals(1.234, $token['value']);
-    }
-
-    public function testScannerRecognizesFloatLocaleSpecificBig()
-    {
-        $scanner = new Doctrine_Query_Scanner('1.234.567,89');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
-        $this->assertEquals(1.23456789e6, $token['value']);
-    }
-
-    public function testScannerRecognizesFloatLocaleSpecificBigWrongPoint()
-    {
-        $scanner = new Doctrine_Query_Scanner('12.34.56.7,89');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
-        $this->assertEquals(1.23456789e6, $token['value']);
-    }
-
-    public function testScannerRecognizesFloatLocaleSpecificExponent()
-    {
-        $scanner = new Doctrine_Query_Scanner('1,234e2');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
-        $this->assertEquals(1.234e2, $token['value']);
-    }
-
-    public function testScannerRecognizesFloatLocaleSpecificExponent2()
-    {
-        $scanner = new Doctrine_Query_Scanner('0,234e2');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
-        $this->assertEquals(.234e2, $token['value']);
-    }
-
-    public function testScannerRecognizesFloatContainingWhitespace()
-    {
-        $scanner = new Doctrine_Query_Scanner('-   1.234e2');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_NONE, $token['type']);
-        $this->assertEquals('-', $token['value']);
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_FLOAT, $token['type']);
-        $this->assertNotEquals(-1.234e2, $token['value']);
-        $this->assertEquals(1.234e2, $token['value']);
-    }
-
-    public function testScannerRecognizesStringContainingWhitespace()
-    {
-        $scanner = new Doctrine_Query_Scanner("'This is a string.'");
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_STRING, $token['type']);
-        $this->assertEquals("'This is a string.'", $token['value']);
-    }
-
-    public function testScannerRecognizesStringContainingSingleQuotes()
-    {
-        $scanner = new Doctrine_Query_Scanner("'abc''defg'''");
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_STRING, $token['type']);
-        $this->assertEquals("'abc''defg'''", $token['value']);
-    }
-
-    public function testScannerRecognizesInputParameter()
-    {
-        $scanner = new Doctrine_Query_Scanner('?');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_INPUT_PARAMETER, $token['type']);
-        $this->assertEquals('?', $token['value']);
-    }
-
-    public function testScannerRecognizesNamedInputParameter()
-    {
-        $scanner = new Doctrine_Query_Scanner(':name');
-
-        $token = $scanner->next();
-        $this->assertEquals(Doctrine_Query_Token::T_INPUT_PARAMETER, $token['type']);
-        $this->assertEquals(':name', $token['value']);
-    }
-
-    public function testScannerTokenizesASimpleQueryCorrectly()
-    {
-        $dql = "SELECT u.* FROM User u WHERE u.name = 'Jack O''Neil'";
-        $scanner = new Doctrine_Query_Scanner($dql);
-
-        $tokens = array(
-            array(
-                'value' => 'SELECT',
-                'type'  => Doctrine_Query_Token::T_SELECT,
-                'position' => 0
-            ),
-            array(
-                'value' => 'u',
-                'type'  => Doctrine_Query_Token::T_IDENTIFIER,
-                'position' => 7
-            ),
-            array(
-                'value' => '.',
-                'type'  => Doctrine_Query_Token::T_NONE,
-                'position' => 8
-            ),
-            array(
-                'value' => '*',
-                'type'  => Doctrine_Query_Token::T_NONE,
-                'position' => 9
-            ),
-            array(
-                'value' => 'FROM',
-                'type'  => Doctrine_Query_Token::T_FROM,
-                'position' => 11
-            ),
-            array(
-                'value' => 'User',
-                'type'  => Doctrine_Query_Token::T_IDENTIFIER,
-                'position' => 16
-            ),
-            array(
-                'value' => 'u',
-                'type'  => Doctrine_Query_Token::T_IDENTIFIER,
-                'position' => 21
-            ),
-            array(
-                'value' => 'WHERE',
-                'type'  => Doctrine_Query_Token::T_WHERE,
-                'position' => 23
-            ),
-            array(
-                'value' => 'u',
-                'type'  => Doctrine_Query_Token::T_IDENTIFIER,
-                'position' => 29
-            ),
-            array(
-                'value' => '.',
-                'type'  => Doctrine_Query_Token::T_NONE,
-                'position' => 30
-            ),
-            array(
-                'value' => 'name',
-                'type'  => Doctrine_Query_Token::T_IDENTIFIER,
-                'position' => 31
-            ),
-            array(
-                'value' => '=',
-                'type'  => Doctrine_Query_Token::T_NONE,
-                'position' => 36
-            ),
-            array(
-                'value' => "'Jack O''Neil'",
-                'type'  => Doctrine_Query_Token::T_STRING,
-                'position' => 38
-            )
-        );
-
-        foreach ($tokens as $expected) {
-            $actual = $scanner->next();
-            $this->assertEquals($expected['value'], $actual['value']);
-            $this->assertEquals($expected['type'], $actual['type']);
-            $this->assertEquals($expected['position'], $actual['position']);
-        }
-
-        $this->assertNull($scanner->next());
-    }
-}