[2.0] Created common Lexer code for Annotations parser and Query parser. Added some docblocks
This commit is contained in:
parent
084add0af0
commit
88698c3a78
@ -1,11 +1,51 @@
|
||||
<?php
|
||||
/*
|
||||
* $Id$
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
* This software consists of voluntary contributions made by many individuals
|
||||
* and is licensed under the LGPL. For more information, see
|
||||
* <http://www.doctrine-project.org>.
|
||||
*/
|
||||
|
||||
namespace Doctrine\Common\Annotations;
|
||||
|
||||
/**
|
||||
* Annotations class
|
||||
*
|
||||
* @license http://www.opensource.org/licenses/lgpl-license.php LGPL
|
||||
* @link www.doctrine-project.org
|
||||
* @since 2.0
|
||||
* @version $Revision: 3938 $
|
||||
* @author Guilherme Blanco <guilhermeblanco@hotmail.com>
|
||||
* @author Jonathan Wage <jonwage@gmail.com>
|
||||
* @author Roman Borschel <roman@code-factory.org>
|
||||
*/
|
||||
class Annotation
|
||||
{
|
||||
/**
|
||||
* Value property. Common among all derived classes.
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
public $value;
|
||||
|
||||
/**
|
||||
* Constructor
|
||||
*
|
||||
* @param array $data Key-value for properties to be defined in this class
|
||||
*/
|
||||
public final function __construct(array $data)
|
||||
{
|
||||
foreach ($data as $key => $value) {
|
||||
|
@ -21,23 +21,49 @@
|
||||
|
||||
namespace Doctrine\Common\Annotations;
|
||||
|
||||
use \ReflectionClass, \ReflectionMethod, \ReflectionProperty;
|
||||
use Doctrine\Common\Cache\Cache;
|
||||
use \ReflectionClass,
|
||||
\ReflectionMethod,
|
||||
\ReflectionProperty,
|
||||
Doctrine\Common\Cache\Cache;
|
||||
|
||||
/**
|
||||
* A reader for docblock annotations.
|
||||
*
|
||||
* @author Roman Borschel <roman@code-factory.org>
|
||||
* @license http://www.opensource.org/licenses/lgpl-license.php LGPL
|
||||
* @link www.doctrine-project.org
|
||||
* @since 2.0
|
||||
* @version $Revision: 3938 $
|
||||
* @author Guilherme Blanco <guilhermeblanco@hotmail.com>
|
||||
* @author Jonathan Wage <jonwage@gmail.com>
|
||||
* @author Roman Borschel <roman@code-factory.org>
|
||||
*/
|
||||
class AnnotationReader
|
||||
{
|
||||
/**
|
||||
* Cache salt
|
||||
*
|
||||
* @var string
|
||||
* @static
|
||||
*/
|
||||
private static $CACHE_SALT = "@<Annot>";
|
||||
|
||||
/**
|
||||
* Annotations Parser
|
||||
*
|
||||
* @var Doctrine\Common\Annotations\Parser
|
||||
*/
|
||||
private $_parser;
|
||||
|
||||
/**
|
||||
* Cache machanism to store processed Annotations
|
||||
*
|
||||
* @var Doctrine\Common\Cache\Cache
|
||||
*/
|
||||
private $_cache;
|
||||
|
||||
/**
|
||||
* Initiaizes a new AnnotationReader that uses the given Cache provider to cache annotations.
|
||||
* Constructor. Initiaizes a new AnnotationReader that uses the given
|
||||
* Cache provider to cache annotations.
|
||||
*
|
||||
* @param Cache $cache The cache provider to use.
|
||||
*/
|
||||
|
@ -24,13 +24,15 @@ namespace Doctrine\Common\Annotations;
|
||||
/**
|
||||
* Simple lexer for docblock annotations.
|
||||
*
|
||||
* @author Roman Borschel <roman@code-factory.org>
|
||||
* @license http://www.opensource.org/licenses/lgpl-license.php LGPL
|
||||
* @link www.doctrine-project.org
|
||||
* @since 2.0
|
||||
* @version $Revision$
|
||||
* @version $Revision: 3938 $
|
||||
* @author Guilherme Blanco <guilhermeblanco@hotmail.com>
|
||||
* @author Jonathan Wage <jonwage@gmail.com>
|
||||
* @author Roman Borschel <roman@code-factory.org>
|
||||
*/
|
||||
class Lexer
|
||||
class Lexer extends \Doctrine\Common\Lexer
|
||||
{
|
||||
const T_NONE = 1;
|
||||
const T_FLOAT = 2;
|
||||
@ -41,138 +43,30 @@ class Lexer
|
||||
const T_FALSE = 7;
|
||||
|
||||
/**
|
||||
* Array of scanned tokens.
|
||||
*
|
||||
* @var array
|
||||
* @inheritdoc
|
||||
*/
|
||||
private $_tokens = array();
|
||||
private $_position = 0;
|
||||
private $_peek = 0;
|
||||
|
||||
/**
|
||||
* @var array The next token in the query string.
|
||||
*/
|
||||
public $lookahead;
|
||||
|
||||
/**
|
||||
* @var array The last matched/seen token.
|
||||
*/
|
||||
public $token;
|
||||
|
||||
public function setInput($input)
|
||||
protected function getCatchablePatterns()
|
||||
{
|
||||
$this->_tokens = array();
|
||||
$this->_scan($input);
|
||||
}
|
||||
|
||||
public function reset()
|
||||
{
|
||||
$this->lookahead = null;
|
||||
$this->token = null;
|
||||
$this->_peek = 0;
|
||||
$this->_position = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether a given token matches the current lookahead.
|
||||
*
|
||||
* @param integer|string $token
|
||||
* @return boolean
|
||||
*/
|
||||
public function isNextToken($token)
|
||||
{
|
||||
$la = $this->lookahead;
|
||||
return ($la['type'] === $token || $la['value'] === $token);
|
||||
}
|
||||
|
||||
/**
|
||||
* Moves to the next token in the input string.
|
||||
*
|
||||
* A token is an associative array containing three items:
|
||||
* - 'value' : the string value of the token in the input string
|
||||
* - 'type' : the type of the token (identifier, numeric, string, input
|
||||
* parameter, none)
|
||||
* - 'position' : the position of the token in the input string
|
||||
*
|
||||
* @return array|null the next token; null if there is no more tokens left
|
||||
*/
|
||||
public function moveNext()
|
||||
{
|
||||
$this->token = $this->lookahead;
|
||||
$this->_peek = 0;
|
||||
if (isset($this->_tokens[$this->_position])) {
|
||||
$this->lookahead = $this->_tokens[$this->_position++];
|
||||
return true;
|
||||
} else {
|
||||
$this->lookahead = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Tells the lexer to skip input tokens until it sees a token with the given value.
|
||||
*
|
||||
* @param $value The value to skip until.
|
||||
*/
|
||||
public function skipUntil($value)
|
||||
{
|
||||
while ($this->lookahead !== null && $this->lookahead['value'] !== $value) {
|
||||
$this->moveNext();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if an identifier is a keyword and returns its correct type.
|
||||
*
|
||||
* @param string $identifier identifier name
|
||||
* @return int token type
|
||||
*/
|
||||
private function _checkLiteral($identifier)
|
||||
{
|
||||
$name = 'Doctrine\Common\Annotations\Lexer::T_' . strtoupper($identifier);
|
||||
|
||||
if (defined($name)) {
|
||||
return constant($name);
|
||||
}
|
||||
|
||||
return self::T_IDENTIFIER;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scans the input string for tokens.
|
||||
*
|
||||
* @param string $input a query string
|
||||
*/
|
||||
private function _scan($input)
|
||||
{
|
||||
static $regex;
|
||||
|
||||
if ( ! isset($regex)) {
|
||||
$patterns = array(
|
||||
return array(
|
||||
'[a-z_][a-z0-9_\\\]*',
|
||||
'(?:[0-9]+(?:[\.][0-9]+)*)(?:e[+-]?[0-9]+)?',
|
||||
'"(?:[^"]|"")*"'
|
||||
);
|
||||
$regex = '/(' . implode(')|(', $patterns) . ')|\s+|\*+|(.)/i';
|
||||
}
|
||||
|
||||
$matches = preg_split($regex, $input, -1, PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE);
|
||||
|
||||
foreach ($matches as $match) {
|
||||
$type = $this->_getType($match);
|
||||
$this->_tokens[] = array(
|
||||
'value' => $match,
|
||||
'type' => $type
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @todo Doc
|
||||
* @inheritdoc
|
||||
*/
|
||||
private function _getType(&$value)
|
||||
protected function getNonCatchablePatterns()
|
||||
{
|
||||
return array('\s+', '\*+', '(.)');
|
||||
}
|
||||
|
||||
/**
|
||||
* @inheritdoc
|
||||
*/
|
||||
protected function _getType(&$value)
|
||||
{
|
||||
// $value is referenced because it can be changed if it is numeric.
|
||||
$type = self::T_NONE;
|
||||
|
||||
$newVal = $this->_getNumeric($value);
|
||||
@ -211,28 +105,19 @@ class Lexer
|
||||
}
|
||||
|
||||
/**
|
||||
* Moves the lookahead token forward.
|
||||
* Checks if an identifier is a keyword and returns its correct type.
|
||||
*
|
||||
* @return array|null The next token or NULL if there are no more tokens ahead.
|
||||
* @param string $identifier identifier name
|
||||
* @return int token type
|
||||
*/
|
||||
public function peek()
|
||||
private function _checkLiteral($identifier)
|
||||
{
|
||||
if (isset($this->_tokens[$this->_position + $this->_peek])) {
|
||||
return $this->_tokens[$this->_position + $this->_peek++];
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
$name = 'Doctrine\Common\Annotations\Lexer::T_' . strtoupper($identifier);
|
||||
|
||||
if (defined($name)) {
|
||||
return constant($name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Peeks at the next token, returns it and immediately resets the peek.
|
||||
*
|
||||
* @return array|null The next token or NULL if there are no more tokens ahead.
|
||||
*/
|
||||
public function glimpse()
|
||||
{
|
||||
$peek = $this->peek();
|
||||
$this->_peek = 0;
|
||||
return $peek;
|
||||
return self::T_IDENTIFIER;
|
||||
}
|
||||
}
|
240
lib/Doctrine/Common/Lexer.php
Normal file
240
lib/Doctrine/Common/Lexer.php
Normal file
@ -0,0 +1,240 @@
|
||||
<?php
|
||||
/*
|
||||
* $Id$
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
* This software consists of voluntary contributions made by many individuals
|
||||
* and is licensed under the LGPL. For more information, see
|
||||
* <http://www.doctrine-project.org>.
|
||||
*/
|
||||
|
||||
namespace Doctrine\Common;
|
||||
|
||||
/**
|
||||
* Simple generic lexical scanner
|
||||
*
|
||||
* @license http://www.opensource.org/licenses/lgpl-license.php LGPL
|
||||
* @link www.doctrine-project.org
|
||||
* @since 2.0
|
||||
* @version $Revision: 3938 $
|
||||
* @author Guilherme Blanco <guilhermeblanco@hotmail.com>
|
||||
* @author Jonathan Wage <jonwage@gmail.com>
|
||||
* @author Roman Borschel <roman@code-factory.org>
|
||||
*/
|
||||
abstract class Lexer
|
||||
{
|
||||
/**
|
||||
* Array of scanned tokens.
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
private $_tokens = array();
|
||||
|
||||
/**
|
||||
* @todo Doc
|
||||
*/
|
||||
private $_position = 0;
|
||||
|
||||
/**
|
||||
* @todo Doc
|
||||
*/
|
||||
private $_peek = 0;
|
||||
|
||||
/**
|
||||
* @var array The next token in the query string.
|
||||
*/
|
||||
public $lookahead;
|
||||
|
||||
/**
|
||||
* @var array The last matched/seen token.
|
||||
*/
|
||||
public $token;
|
||||
|
||||
/**
|
||||
* Inputs data to be tokenized
|
||||
*
|
||||
* @param string $input input to be tokenized
|
||||
*/
|
||||
public function setInput($input)
|
||||
{
|
||||
$this->_tokens = array();
|
||||
$this->reset();
|
||||
$this->_scan($input);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the scanner
|
||||
*
|
||||
*/
|
||||
public function reset()
|
||||
{
|
||||
$this->lookahead = null;
|
||||
$this->token = null;
|
||||
$this->_peek = 0;
|
||||
$this->_position = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the peek pointer to 0
|
||||
*
|
||||
*/
|
||||
public function resetPeek()
|
||||
{
|
||||
$this->_peek = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the lexer position on the input to the given position
|
||||
*
|
||||
* @param integer $position Position to place the lexical scanner
|
||||
*/
|
||||
public function resetPosition($position = 0)
|
||||
{
|
||||
$this->_position = $position;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether a given token matches the current lookahead.
|
||||
*
|
||||
* @param integer|string $token
|
||||
* @return boolean
|
||||
*/
|
||||
public function isNextToken($token)
|
||||
{
|
||||
$la = $this->lookahead;
|
||||
return ($la['type'] === $token || $la['value'] === $token);
|
||||
}
|
||||
|
||||
/**
|
||||
* Moves to the next token in the input string.
|
||||
*
|
||||
* A token is an associative array containing three items:
|
||||
* - 'value' : the string value of the token in the input string
|
||||
* - 'type' : the type of the token (identifier, numeric, string, input
|
||||
* parameter, none)
|
||||
* - 'position' : the position of the token in the input string
|
||||
*
|
||||
* @return array|null the next token; null if there is no more tokens left
|
||||
*/
|
||||
public function moveNext()
|
||||
{
|
||||
$this->token = $this->lookahead;
|
||||
$this->_peek = 0;
|
||||
if (isset($this->_tokens[$this->_position])) {
|
||||
$this->lookahead = $this->_tokens[$this->_position++];
|
||||
return true;
|
||||
} else {
|
||||
$this->lookahead = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Tells the lexer to skip input tokens until it sees a token with the given value.
|
||||
*
|
||||
* @param $value The value to skip until.
|
||||
*/
|
||||
public function skipUntil($value)
|
||||
{
|
||||
while ($this->lookahead !== null && $this->lookahead['value'] !== $value) {
|
||||
$this->moveNext();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @todo Doc
|
||||
*/
|
||||
public function isA($value, $token)
|
||||
{
|
||||
$type = $this->_getType($value);
|
||||
|
||||
return $type === $token;
|
||||
}
|
||||
|
||||
/**
|
||||
* Moves the lookahead token forward.
|
||||
*
|
||||
* @return array | null The next token or NULL if there are no more tokens ahead.
|
||||
*/
|
||||
public function peek()
|
||||
{
|
||||
if (isset($this->_tokens[$this->_position + $this->_peek])) {
|
||||
return $this->_tokens[$this->_position + $this->_peek++];
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Peeks at the next token, returns it and immediately resets the peek.
|
||||
*
|
||||
* @return array|null The next token or NULL if there are no more tokens ahead.
|
||||
*/
|
||||
public function glimpse()
|
||||
{
|
||||
$peek = $this->peek();
|
||||
$this->_peek = 0;
|
||||
return $peek;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scans the input string for tokens.
|
||||
*
|
||||
* @param string $input a query string
|
||||
*/
|
||||
protected function _scan($input)
|
||||
{
|
||||
static $regex;
|
||||
|
||||
if ( ! isset($regex)) {
|
||||
$regex = '/(' . implode(')|(', $this->getCatchablePatterns()) . ')|'
|
||||
. implode('|', $this->getNonCatchablePatterns()) . '/i';
|
||||
}
|
||||
|
||||
$flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE;
|
||||
$matches = preg_split($regex, $input, -1, $flags);
|
||||
|
||||
foreach ($matches as $match) {
|
||||
$value = $match[0];
|
||||
$type = $this->_getType($value);
|
||||
$this->_tokens[] = array(
|
||||
'value' => $value,
|
||||
'type' => $type,
|
||||
'position' => $match[1]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Lexical catchable patterns
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
abstract protected function getCatchablePatterns();
|
||||
|
||||
/**
|
||||
* Lexical non-catchable patterns
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
abstract protected function getNonCatchablePatterns();
|
||||
|
||||
/**
|
||||
* Retrieve token type. Also processes the token value if necessary.
|
||||
*
|
||||
* @param string $value
|
||||
* @return integer
|
||||
*/
|
||||
abstract protected function _getType(&$value);
|
||||
}
|
@ -32,7 +32,7 @@ namespace Doctrine\ORM\Query;
|
||||
* @since 2.0
|
||||
* @version $Revision$
|
||||
*/
|
||||
class Lexer
|
||||
class Lexer extends \Doctrine\Common\Lexer
|
||||
{
|
||||
const T_NONE = 1;
|
||||
const T_IDENTIFIER = 2;
|
||||
@ -94,33 +94,6 @@ class Lexer
|
||||
|
||||
private $_keywordsTable;
|
||||
|
||||
/**
|
||||
* Array of scanned tokens.
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
private $_tokens = array();
|
||||
|
||||
/**
|
||||
* @todo Doc
|
||||
*/
|
||||
private $_position = 0;
|
||||
|
||||
/**
|
||||
* @todo Doc
|
||||
*/
|
||||
private $_peek = 0;
|
||||
|
||||
/**
|
||||
* @var array The next token in the query string.
|
||||
*/
|
||||
public $lookahead;
|
||||
|
||||
/**
|
||||
* @var array The last matched/seen token.
|
||||
*/
|
||||
public $token;
|
||||
|
||||
/**
|
||||
* Creates a new query scanner object.
|
||||
*
|
||||
@ -128,105 +101,35 @@ class Lexer
|
||||
*/
|
||||
public function __construct($input)
|
||||
{
|
||||
$this->_scan($input);
|
||||
$this->setInput($input);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks whether a given token matches the current lookahead.
|
||||
*
|
||||
* @param integer|string $token
|
||||
* @return boolean
|
||||
* @inheritdoc
|
||||
*/
|
||||
public function isNextToken($token)
|
||||
protected function getCatchablePatterns()
|
||||
{
|
||||
$la = $this->lookahead;
|
||||
return ($la['type'] === $token || $la['value'] === $token);
|
||||
}
|
||||
|
||||
/**
|
||||
* Moves to the next token in the input string.
|
||||
*
|
||||
* A token is an associative array containing three items:
|
||||
* - 'value' : the string value of the token in the input string
|
||||
* - 'type' : the type of the token (identifier, numeric, string, input
|
||||
* parameter, none)
|
||||
* - 'position' : the position of the token in the input string
|
||||
*
|
||||
* @return array|null the next token; null if there is no more tokens left
|
||||
*/
|
||||
public function moveNext()
|
||||
{
|
||||
$this->token = $this->lookahead;
|
||||
$this->_peek = 0;
|
||||
if (isset($this->_tokens[$this->_position])) {
|
||||
$this->lookahead = $this->_tokens[$this->_position++];
|
||||
return true;
|
||||
} else {
|
||||
$this->lookahead = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if an identifier is a keyword and returns its correct type.
|
||||
*
|
||||
* @param string $identifier identifier name
|
||||
* @return int token type
|
||||
*/
|
||||
private function _checkLiteral($identifier)
|
||||
{
|
||||
$name = 'Doctrine\ORM\Query\Lexer::T_' . strtoupper($identifier);
|
||||
|
||||
if (defined($name)) {
|
||||
$type = constant($name);
|
||||
if ($type > 100) {
|
||||
return $type;
|
||||
}
|
||||
}
|
||||
|
||||
return self::T_IDENTIFIER;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scans the input string for tokens.
|
||||
*
|
||||
* @param string $input a query string
|
||||
*/
|
||||
private function _scan($input)
|
||||
{
|
||||
static $regex;
|
||||
|
||||
if ( ! isset($regex)) {
|
||||
$patterns = array(
|
||||
return array(
|
||||
'[a-z_][a-z0-9_\\\]*',
|
||||
'(?:[0-9]+(?:[,\.][0-9]+)*)(?:e[+-]?[0-9]+)?',
|
||||
"'(?:[^']|'')*'",
|
||||
'\?[1-9]+|:[a-z][a-z0-9_]+'
|
||||
);
|
||||
$regex = '/(' . implode(')|(', $patterns) . ')|\s+|(.)/i';
|
||||
}
|
||||
|
||||
$flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE;
|
||||
$matches = preg_split($regex, $input, -1, $flags);
|
||||
|
||||
foreach ($matches as $match) {
|
||||
$value = $match[0];
|
||||
$type = $this->_getType($value);
|
||||
$this->_tokens[] = array(
|
||||
'value' => $value,
|
||||
'type' => $type,
|
||||
'position' => $match[1]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @todo Doc
|
||||
* @inheritdoc
|
||||
*/
|
||||
private function _getType(&$value)
|
||||
protected function getNonCatchablePatterns()
|
||||
{
|
||||
return array('\s+', '(.)');
|
||||
}
|
||||
|
||||
/**
|
||||
* @inheritdoc
|
||||
*/
|
||||
protected function _getType(&$value)
|
||||
{
|
||||
// $value is referenced because it can be changed if it is numeric.
|
||||
// [TODO] Revisit the _isNumeric and _getNumeric methods to reduce overhead.
|
||||
$type = self::T_NONE;
|
||||
|
||||
$newVal = $this->_getNumeric($value);
|
||||
@ -263,72 +166,27 @@ class Lexer
|
||||
return $value;
|
||||
}
|
||||
|
||||
// World number: 1.000.000,02 or -1,234e-2
|
||||
$worldnum = strtr($value, array('.' => '', ',' => '.'));
|
||||
if (is_numeric($worldnum)) {
|
||||
return $worldnum;
|
||||
}
|
||||
|
||||
// American extensive number: 1,000,000.02
|
||||
$american_en = strtr($value, array(',' => ''));
|
||||
if (is_numeric($american_en)) {
|
||||
return $american_en;
|
||||
}
|
||||
|
||||
return false;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* @todo Doc
|
||||
*/
|
||||
public function isA($value, $token)
|
||||
{
|
||||
$type = $this->_getType($value);
|
||||
|
||||
return $type === $token;
|
||||
}
|
||||
|
||||
/**
|
||||
* Moves the lookahead token forward.
|
||||
* Checks if an identifier is a keyword and returns its correct type.
|
||||
*
|
||||
* @return array|null The next token or NULL if there are no more tokens ahead.
|
||||
* @param string $identifier identifier name
|
||||
* @return int token type
|
||||
*/
|
||||
public function peek()
|
||||
private function _checkLiteral($identifier)
|
||||
{
|
||||
if (isset($this->_tokens[$this->_position + $this->_peek])) {
|
||||
return $this->_tokens[$this->_position + $this->_peek++];
|
||||
} else {
|
||||
return null;
|
||||
$name = 'Doctrine\ORM\Query\Lexer::T_' . strtoupper($identifier);
|
||||
|
||||
if (defined($name)) {
|
||||
$type = constant($name);
|
||||
if ($type > 100) {
|
||||
return $type;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Peeks at the next token, returns it and immediately resets the peek.
|
||||
*
|
||||
* @return array|null The next token or NULL if there are no more tokens ahead.
|
||||
*/
|
||||
public function glimpse()
|
||||
{
|
||||
$peek = $this->peek();
|
||||
$this->_peek = 0;
|
||||
return $peek;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the peek pointer to 0.
|
||||
*/
|
||||
public function resetPeek()
|
||||
{
|
||||
$this->_peek = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the lexer position on the input to the given position.
|
||||
*/
|
||||
public function resetPosition($position = 0)
|
||||
{
|
||||
$this->_position = $position;
|
||||
return self::T_IDENTIFIER;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -100,65 +100,11 @@ class LexerTest extends \Doctrine\Tests\OrmTestCase
|
||||
|
||||
public function testScannerRecognizesFloatBig()
|
||||
{
|
||||
$lexer = new Lexer('1,234,567.89');
|
||||
$lexer = new Lexer('123456789.01');
|
||||
$lexer->moveNext();
|
||||
$token = $lexer->lookahead;
|
||||
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
|
||||
$this->assertEquals(1.23456789e6, $token['value']);
|
||||
}
|
||||
|
||||
public function testScannerRecognizesFloatBigWrongPoint()
|
||||
{
|
||||
$lexer = new Lexer('12,34,56,7.89');
|
||||
$lexer->moveNext();
|
||||
$token = $lexer->lookahead;
|
||||
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
|
||||
$this->assertEquals(1.23456789e6, $token['value']);
|
||||
}
|
||||
|
||||
public function testScannerRecognizesFloatLocaleSpecific()
|
||||
{
|
||||
$lexer = new Lexer('1,234');
|
||||
$lexer->moveNext();
|
||||
$token = $lexer->lookahead;
|
||||
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
|
||||
$this->assertEquals(1.234, $token['value']);
|
||||
}
|
||||
|
||||
public function testScannerRecognizesFloatLocaleSpecificBig()
|
||||
{
|
||||
$lexer = new Lexer('1.234.567,89');
|
||||
$lexer->moveNext();
|
||||
$token = $lexer->lookahead;
|
||||
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
|
||||
$this->assertEquals(1.23456789e6, $token['value']);
|
||||
}
|
||||
|
||||
public function testScannerRecognizesFloatLocaleSpecificBigWrongPoint()
|
||||
{
|
||||
$lexer = new Lexer('12.34.56.7,89');
|
||||
$lexer->moveNext();
|
||||
$token = $lexer->lookahead;
|
||||
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
|
||||
$this->assertEquals(1.23456789e6, $token['value']);
|
||||
}
|
||||
|
||||
public function testScannerRecognizesFloatLocaleSpecificExponent()
|
||||
{
|
||||
$lexer = new Lexer('1,234e2');
|
||||
$lexer->moveNext();
|
||||
$token = $lexer->lookahead;
|
||||
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
|
||||
$this->assertEquals(1.234e2, $token['value']);
|
||||
}
|
||||
|
||||
public function testScannerRecognizesFloatLocaleSpecificExponent2()
|
||||
{
|
||||
$lexer = new Lexer('0,234e2');
|
||||
$lexer->moveNext();
|
||||
$token = $lexer->lookahead;
|
||||
$this->assertEquals(Lexer::T_FLOAT, $token['type']);
|
||||
$this->assertEquals(.234e2, $token['value']);
|
||||
$this->assertEquals(1.2345678901e8, $token['value']);
|
||||
}
|
||||
|
||||
public function testScannerRecognizesFloatContainingWhitespace()
|
||||
|
Loading…
Reference in New Issue
Block a user