@ -38,7 +38,7 @@ class Tokenizer {
@@ -38,7 +38,7 @@ class Tokenizer {
int pos ;
int max ;
List < Token > tokens = new ArrayList < Token > ( ) ;
public Tokenizer ( String inputdata ) {
this . expressionString = inputdata ;
this . toProcess = ( inputdata + "\0" ) . toCharArray ( ) ;
@ -46,7 +46,7 @@ class Tokenizer {
@@ -46,7 +46,7 @@ class Tokenizer {
this . pos = 0 ;
process ( ) ;
}
public void process ( ) {
while ( pos < max ) {
char ch = toProcess [ pos ] ;
@ -128,6 +128,16 @@ class Tokenizer {
@@ -128,6 +128,16 @@ class Tokenizer {
pushCharToken ( TokenKind . ASSIGN ) ;
}
break ;
case '&' :
if ( isTwoCharToken ( TokenKind . SYMBOLIC_AND ) ) {
pushPairToken ( TokenKind . SYMBOLIC_AND ) ;
}
break ;
case '|' :
if ( isTwoCharToken ( TokenKind . SYMBOLIC_OR ) ) {
pushPairToken ( TokenKind . SYMBOLIC_OR ) ;
}
break ;
case '?' :
if ( isTwoCharToken ( TokenKind . SELECT ) ) {
pushPairToken ( TokenKind . SELECT ) ;
@ -195,12 +205,11 @@ class Tokenizer {
@@ -195,12 +205,11 @@ class Tokenizer {
}
}
}
public List < Token > getTokens ( ) {
return tokens ;
}
// STRING_LITERAL: '\''! (APOS|~'\'')* '\''!;
private void lexQuotedStringLiteral ( ) {
int start = pos ;
@ -223,7 +232,7 @@ class Tokenizer {
@@ -223,7 +232,7 @@ class Tokenizer {
pos + + ;
tokens . add ( new Token ( TokenKind . LITERAL_STRING , subarray ( start , pos ) , start , pos ) ) ;
}
// DQ_STRING_LITERAL: '"'! (~'"')* '"'!;
private void lexDoubleQuotedStringLiteral ( ) {
int start = pos ;
@ -241,8 +250,7 @@ class Tokenizer {
@@ -241,8 +250,7 @@ class Tokenizer {
pos + + ;
tokens . add ( new Token ( TokenKind . LITERAL_STRING , subarray ( start , pos ) , start , pos ) ) ;
}
// REAL_LITERAL :
// ('.' (DECIMAL_DIGIT)+ (EXPONENT_PART)? (REAL_TYPE_SUFFIX)?) |
// ((DECIMAL_DIGIT)+ '.' (DECIMAL_DIGIT)+ (EXPONENT_PART)? (REAL_TYPE_SUFFIX)?) |
@ -256,7 +264,7 @@ class Tokenizer {
@@ -256,7 +264,7 @@ class Tokenizer {
// fragment REAL_TYPE_SUFFIX : 'F' | 'f' | 'D' | 'd';
// INTEGER_LITERAL
// : (DECIMAL_DIGIT)+ (INTEGER_TYPE_SUFFIX)?;
private void lexNumericLiteral ( boolean firstCharIsZero ) {
boolean isReal = false ;
int start = pos ;
@ -353,10 +361,10 @@ class Tokenizer {
@@ -353,10 +361,10 @@ class Tokenizer {
}
}
}
// if this is changed, it must remain sorted
private static final String [ ] alternativeOperatorNames = { "DIV" , "EQ" , "GE" , "GT" , "LE" , "LT" , "MOD" , "NE" , "NOT" } ;
private void lexIdentifier ( ) {
int start = pos ;
do {
@ -375,7 +383,7 @@ class Tokenizer {
@@ -375,7 +383,7 @@ class Tokenizer {
}
tokens . add ( new Token ( TokenKind . IDENTIFIER , subarray , start , pos ) ) ;
}
private void pushIntToken ( char [ ] data , boolean isLong , int start , int end ) {
if ( isLong ) {
tokens . add ( new Token ( TokenKind . LITERAL_LONG , data , start , end ) ) ;
@ -398,7 +406,7 @@ class Tokenizer {
@@ -398,7 +406,7 @@ class Tokenizer {
tokens . add ( new Token ( TokenKind . LITERAL_HEXINT , data , start , end ) ) ;
}
}
private void pushRealToken ( char [ ] data , boolean isFloat , int start , int end ) {
if ( isFloat ) {
tokens . add ( new Token ( TokenKind . LITERAL_REAL_FLOAT , data , start , end ) ) ;
@ -406,13 +414,13 @@ class Tokenizer {
@@ -406,13 +414,13 @@ class Tokenizer {
tokens . add ( new Token ( TokenKind . LITERAL_REAL , data , start , end ) ) ;
}
}
private char [ ] subarray ( int start , int end ) {
char [ ] result = new char [ end - start ] ;
System . arraycopy ( toProcess , start , result , 0 , end - start ) ;
return result ;
}
/ * *
* Check if this might be a two character token .
* /
@ -421,7 +429,7 @@ class Tokenizer {
@@ -421,7 +429,7 @@ class Tokenizer {
Assert . isTrue ( toProcess [ pos ] = = kind . tokenChars [ 0 ] ) ;
return toProcess [ pos + 1 ] = = kind . tokenChars [ 1 ] ;
}
/ * *
* Push a token of just one character in length .
* /
@ -429,7 +437,7 @@ class Tokenizer {
@@ -429,7 +437,7 @@ class Tokenizer {
tokens . add ( new Token ( kind , pos , pos + 1 ) ) ;
pos + + ;
}
/ * *
* Push a token of two characters in length .
* /
@ -437,7 +445,7 @@ class Tokenizer {
@@ -437,7 +445,7 @@ class Tokenizer {
tokens . add ( new Token ( kind , pos , pos + 2 ) ) ;
pos + = 2 ;
}
private void pushOneCharOrTwoCharToken ( TokenKind kind , int pos , char [ ] data ) {
tokens . add ( new Token ( kind , data , pos , pos + kind . getLength ( ) ) ) ;
}
@ -446,7 +454,7 @@ class Tokenizer {
@@ -446,7 +454,7 @@ class Tokenizer {
private boolean isIdentifier ( char ch ) {
return isAlphabetic ( ch ) | | isDigit ( ch ) | | ch = = '_' | | ch = = '$' ;
}
private boolean isChar ( char a , char b ) {
char ch = toProcess [ pos ] ;
return ch = = a | | ch = = b ;
@ -467,7 +475,7 @@ class Tokenizer {
@@ -467,7 +475,7 @@ class Tokenizer {
private boolean isSign ( char ch ) {
return ch = = '+' | | ch = = '-' ;
}
private boolean isDigit ( char ch ) {
if ( ch > 255 ) {
return false ;
@ -481,14 +489,14 @@ class Tokenizer {
@@ -481,14 +489,14 @@ class Tokenizer {
}
return ( flags [ ch ] & IS_ALPHA ) ! = 0 ;
}
private boolean isHexadecimalDigit ( char ch ) {
if ( ch > 255 ) {
return false ;
}
return ( flags [ ch ] & IS_HEXDIGIT ) ! = 0 ;
}
private static final byte flags [ ] = new byte [ 256 ] ;
private static final byte IS_DIGIT = 0x01 ;
private static final byte IS_HEXDIGIT = 0x02 ;