[antlr-interest] Antlr v3 -- Lexing '..' and '1.'

[John B. Brodie]

Greetings!

I am trying to construct a lexer under Antlr v3 that will recognize a
RANGE (e.g. '..') operator, FLOAT (e.g. 1.) numbers, and some other stuff.

Here is what I have so far:

//------------------------- begin cut here ---------------------------

grammar Test;

test : .+ EOF ;

DOT : '.' ;
RANGE : '..' ;
ELLIPSIS : '...' ;

NATURAL : UINT ;

FLOAT :
        ( UINT ('.' ~'.') => ('.' UINT? EXPONENT?) ) /*1*/
    |   ( UINT EXPONENT )
    |   ( '.' UINT EXPONENT? )
    ;

fragment EXPONENT : ( 'e' | 'E' ) ( '+' | '-' )? UINT ;

fragment DIGIT_10 : '0'..'9' ;
fragment UINT : DIGIT_10+ ;



// Whitespace -- ignored
WS	:	(	' '
		|	'\t'
		|	'\f'
		|	( '\r' | '\n' ) // handle newlines
		)+
		{ channel=99; }
	;

//------------------------- end cut here -----------------------------

My problems center around (i think) the line I have labeled /*1*/ above.

I want to recognize a '.' inside a FLOAT if and only if that '.' is not
followed by another '.'

I think the syntactic predicate in /*1*/ does that.

Antlr v3 ignores the predicate.

Can you help me fix this?

(also attached below is a rather tooooo long Java test driver)

This is very similar to a problem in Pascal. Has the v2 Pascal grammar
been ported to v3 yet? If so, just point me to it.

Also I think I saw on the Mantra website that Mantra also is meant to have
floats and a RANGE operator, but I could not find any handling of RANGE in
the grammar contained in the Mantra source tar-ball.

I must have missed something here, please point me in the proper direction.

Thanks in advance!
   -jbb

TestDriver.java follows...
//------------------------- begin cut here ---------------------------
import java.io.*;
import java.util.*;

import org.antlr.runtime.*;

public class TestDriver {
   
   private static class Pair {
      public String input;
      public int[] expected;
      public Pair(String i, int[]e) {
         input = i;
         expected = e;
      }
   }

   private static final Pair[] x = {
      new Pair("1",new int[]{Test.NATURAL}),
      new Pair(".",new int[]{Test.DOT}),
      new Pair("..",new int[]{Test.RANGE}),
      new Pair("...",new int[]{Test.ELLIPSIS}),
      new Pair(".0",new int[]{Test.FLOAT}),
      new Pair(".0e+0",new int[]{Test.FLOAT}),
      new Pair("1.0",new int[]{Test.FLOAT}),
      new Pair("1.",new int[]{Test.FLOAT}),
      new Pair("1.0",new int[]{Test.FLOAT}),
      new Pair("1.0e+0",new int[]{Test.FLOAT}),
      new Pair("1e1",new int[]{Test.FLOAT}),
      new Pair("1e+1",new int[]{Test.FLOAT}),
      new Pair("1e-1",new int[]{Test.FLOAT}),
      new Pair("1 . 2",new int[]{Test.NATURAL,
                                 Test.WS,
                                 Test.DOT,
                                 Test.WS,
                                 Test.NATURAL}),
      new Pair("1 .. 2",new int[]{Test.NATURAL,
                                  Test.WS,
                                  Test.RANGE,
                                  Test.WS,
                                  Test.NATURAL}),
      new Pair("1 ... 2",new int[]{Test.NATURAL,
                                   Test.WS,
                                   Test.ELLIPSIS,
                                   Test.WS,
                                   Test.NATURAL}),
      new Pair("1. . 2.",new int[]{Test.FLOAT,
                                   Test.WS,
                                   Test.DOT,
                                   Test.WS,
                                   Test.FLOAT}),
      new Pair("1. .. 2.",new int[]{Test.FLOAT,
                                    Test.WS,
                                    Test.RANGE,
                                    Test.WS,
                                    Test.FLOAT}),
      new Pair("1. ... 2.",new int[]{Test.FLOAT,
                                     Test.WS,
                                     Test.ELLIPSIS,
                                     Test.WS,
                                     Test.FLOAT}),
      new Pair("1.1 . 2.2",new int[]{Test.FLOAT,
                                     Test.WS,
                                     Test.DOT,
                                     Test.WS,
                                     Test.FLOAT}),
      new Pair("1.1 .. 2.2",new int[]{Test.FLOAT,
                                      Test.WS,
                                      Test.RANGE,
                                      Test.WS,
                                      Test.FLOAT}),
      new Pair("1.1 ... 2.2",new int[]{Test.FLOAT,
                                       Test.WS,
                                       Test.ELLIPSIS,
                                       Test.WS,
                                       Test.FLOAT}),
      new Pair("1.",new int[]{Test.FLOAT}),
      new Pair("1..",new int[]{Test.NATURAL,Test.RANGE}),
      new Pair("1...",new int[]{Test.NATURAL,Test.ELLIPSIS}),
      new Pair("1.2",new int[]{Test.FLOAT}),
      new Pair("1..2",new int[]{Test.NATURAL,
                                Test.RANGE,
                                Test.NATURAL}),
      new Pair("1...2",new int[]{Test.NATURAL,
                                 Test.ELLIPSIS,
                                 Test.NATURAL})
   };

   public static void main(String[] args) {
      for(int i = 0; i < x.length; ++i) {
         try {
            System.out.format("testing `%s`: ",x[i].input);
            boolean success = true;

            CommonTokenStream tokens = new CommonTokenStream
               (new TestLexer
                (new ANTLRStringStream(x[i].input)));

            List l = tokens.getTokens();
            if( l.size() == x[i].expected.length ) {
               for(int j = 0; j < x[i].expected.length; ++j) {
                  Token t = (Token)l.get(j);
                  if (t.getType() != x[i].expected[j]) {
                     success = false;
                     System.out.format("token number %s, %s should be %s; ",
                                       j,
                                       Test.tokenNames[t.getType()],
                                       Test.tokenNames[x[i].expected[j]]);
                  }
               }
            } else {
               success = false;
               System.out.format("...wrong number of tokens:%s should be %s; ",
                                 l.size(),x[i].expected.length);
            }
            if (success) {
               System.out.format("OK%n");
            } else {
               System.out.format("FAIL: ");
               if( l.size() == 0 ) {
                  System.out.format("no tokens found!%n");
               } else {
                  Token t = (Token)l.get(0);
                  System.out.format("([%s,%s]",
                                    Test.tokenNames[t.getType()],
                                    t.getText());
                  for(int j = 1; j < l.size(); ++j) {
                     t = (Token)l.get(j);
                     System.out.format(", [%s,%s]",
                                       Test.tokenNames[t.getType()],
                                       t.getText());
                  }
                  System.out.format(")%n");
               }
            }
         } catch(Exception e) {
            System.out.println("exception: "+e);
         }
      }
   }
}
//------------------------- end cut here -----------------------------