[antlr-interest] Antlr v3 -- Lexing '..' and '1.'

John B. Brodie jbb at acm.org
Sun Jul 30 16:02:25 PDT 2006


Ter and Loring :-

Thanks for your replies!

I now have a working lexer fragment that handles float and '..'  tokens,
attached below.

I hope my implementation matches your intent for dealing with this....

(I also attach a TestDriver should anybody want to try to reproduce my result)

Thanks again!
   -jbb

Test.g follows...
//---------------------------begin cut here---------------------------
grammar Test;

//------------------------------------------------------------------------
// code to be incorporated into the generated lexer...

@lexer::header {
import java.util.Vector;
}

@lexer::members {
    // maximum number of emit() calls inside any rule action
    private static final int MAX_EMIT_COUNT = 2;

    // buffer (queue) to hold the emit()'d tokens
    private Token [] myToken = new Token[MAX_EMIT_COUNT];
    private int add_idx = 0; // deposit emit token here
    private int next_idx = 0; // next token to be delivered to parser

    public void emit(Token t) {
        token = t; // set flag to avoid automatic emit() at end of rule.
        myToken[add_idx++] = t;
    }

    public Token nextToken() {
        while (true) {
            if ( add_idx == next_idx ) {
                token = null;
                add_idx = 0;
                next_idx = 0;
                tokenStartCharIndex = getCharIndex();
                if ( input.LA(1)==CharStream.EOF ) {
                    return Token.EOF_TOKEN;
                }
                try {
                    mTokens();
                }
                catch (RecognitionException re) {
                    reportError(re);
                    recover(re);
                }
            } else {
                Token result = myToken[next_idx++];
                if ( result != Token.SKIP_TOKEN ) { // discard SKIP tokens
                    return result;
                }
            }
        }
    }
}


//------------------------------------------------------------------------
// next follows the (dummy) parser...

test : .+ EOF;

//------------------------------------------------------------------------
// and now, we specify the lexical analyzer...

DOT : '.' ;
RANGE : '..' ;
ELLIPSIS : '...' ;

NATURAL : UINT ;

FLOAT :
        ( i=UINT d=DOT
            ( ( UINT? EXPONENT? )
            | ( r=DOT
                    ( ( /*empty*/
                            { i.setType(NATURAL);
                              emit(i);
                              d.setType(RANGE);
                              d.setText("..");
                              emit(d); } )
                    | ( e=DOT
                            { i.setType(NATURAL);
                              emit(i);
                              d.setType(ELLIPSIS);
                              d.setText("...");
                              emit(d); } )
                    ) )
            ) )
    |   ( UINT EXPONENT )
    |   ( DOT UINT EXPONENT? )
    ;

fragment EXPONENT : ( 'e' | 'E' ) ( '+' | '-' )? UINT ;

fragment DIGIT_10 : '0'..'9' ;
fragment UINT : DIGIT_10+ ;



// Whitespace -- ignored
WS  :   (   ' '
        |   '\t'
        |   '\f'
        |   ( '\r' | '\n' ) // handle newlines
        )+
        { channel=99; }
    ;
//---------------------------end cut here-----------------------------

TestDriver.java follows:
//---------------------------begin cut here---------------------------
import java.io.*;
import java.util.*;

import org.antlr.runtime.*;

public class TestDriver {
   
   private static class Pair {
      public String input;
      public int[] expected;
      public Pair(String i, int[]e) {
         input = i;
         expected = e;
      }
   }

   private static final Pair[] x = {
      new Pair("1",new int[]{Test.NATURAL}),
      new Pair(".",new int[]{Test.DOT}),
      new Pair("..",new int[]{Test.RANGE}),
      new Pair("...",new int[]{Test.ELLIPSIS}),
      new Pair(".0",new int[]{Test.FLOAT}),
      new Pair(".0e+0",new int[]{Test.FLOAT}),
      new Pair("1.0",new int[]{Test.FLOAT}),
      new Pair("1.",new int[]{Test.FLOAT}),
      new Pair("1.0",new int[]{Test.FLOAT}),
      new Pair("1.0e+0",new int[]{Test.FLOAT}),
      new Pair("1e1",new int[]{Test.FLOAT}),
      new Pair("1e+1",new int[]{Test.FLOAT}),
      new Pair("1e-1",new int[]{Test.FLOAT}),
      new Pair("1 . 2",new int[]{Test.NATURAL,
                                 Test.WS,
                                 Test.DOT,
                                 Test.WS,
                                 Test.NATURAL}),
      new Pair("1 .. 2",new int[]{Test.NATURAL,
                                  Test.WS,
                                  Test.RANGE,
                                  Test.WS,
                                  Test.NATURAL}),
      new Pair("1 ... 2",new int[]{Test.NATURAL,
                                   Test.WS,
                                   Test.ELLIPSIS,
                                   Test.WS,
                                   Test.NATURAL}),
      new Pair("1. . 2.",new int[]{Test.FLOAT,
                                   Test.WS,
                                   Test.DOT,
                                   Test.WS,
                                   Test.FLOAT}),
      new Pair("1. .. 2.",new int[]{Test.FLOAT,
                                    Test.WS,
                                    Test.RANGE,
                                    Test.WS,
                                    Test.FLOAT}),
      new Pair("1. ... 2.",new int[]{Test.FLOAT,
                                     Test.WS,
                                     Test.ELLIPSIS,
                                     Test.WS,
                                     Test.FLOAT}),
      new Pair("1.1 . 2.2",new int[]{Test.FLOAT,
                                     Test.WS,
                                     Test.DOT,
                                     Test.WS,
                                     Test.FLOAT}),
      new Pair("1.1 .. 2.2",new int[]{Test.FLOAT,
                                      Test.WS,
                                      Test.RANGE,
                                      Test.WS,
                                      Test.FLOAT}),
      new Pair("1.1 ... 2.2",new int[]{Test.FLOAT,
                                       Test.WS,
                                       Test.ELLIPSIS,
                                       Test.WS,
                                       Test.FLOAT}),
      new Pair("1.",new int[]{Test.FLOAT}),
      new Pair("1..",new int[]{Test.NATURAL,Test.RANGE}),
      new Pair("1...",new int[]{Test.NATURAL,Test.ELLIPSIS}),
      new Pair("1.2",new int[]{Test.FLOAT}),
      new Pair("1..2",new int[]{Test.NATURAL,
                                Test.RANGE,
                                Test.NATURAL}),
      new Pair("1...2",new int[]{Test.NATURAL,
                                 Test.ELLIPSIS,
                                 Test.NATURAL})
   };

   private static void printTokens(List l) {

      if( l.size() == 0 ) {
         System.out.format("no tokens found!%n");

      } else {
         Token t = (Token)l.get(0);
         System.out.format("([%s,%s]",
                           Test.tokenNames[t.getType()],
                           t.getText());
         for(int j = 1; j < l.size(); ++j) {
            t = (Token)l.get(j);
            System.out.format(", [%s,%s]",
                              Test.tokenNames[t.getType()],
                              t.getText());
         }
         System.out.format(")");
      }
   }

   private static boolean checkTokens(List l, int [] p) {
      boolean success = true;

      if( l.size() == p.length ) {

         for(int j = 0; j < p.length; ++j) {
            Token t = (Token)l.get(j);
            if (t.getType() != p[j]) {
               success = false;
               System.out.format(" token number %s, %s should be %s; ",
                                 j,
                                 Test.tokenNames[t.getType()],
                                 Test.tokenNames[p[j]]);
            }
         }

      } else {
         success = false;
         System.out.format("...wrong number of tokens:%s should be %s; ",
                           l.size(),p.length);
      }

      return success;
   }

   public static void main(String[] args) {
      for(int i = 0; i < x.length; ++i) {
         try {
            System.out.format("testing `%s`: ",x[i].input);

            CommonTokenStream tokens = new CommonTokenStream
               (new TestLexer
                (new ANTLRStringStream(x[i].input)));

            List l = tokens.getTokens();
            printTokens(l);
            if (checkTokens(l, x[i].expected)) {
               System.out.format(" --- OK%n");
            } else {
               System.out.format(" --- FAIL%n");
            }

         } catch(Exception e) {
            System.out.println("exception: "+e);
         }
      }
   }
}
//---------------------------end cut here-----------------------------


More information about the antlr-interest mailing list