00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030
00031
00032
00033
00034
00035
00036
00037
00038
00039
00040
00041
00042
00043
00044
00045
00046
00047
00048
00049
00050
00051
00052
00053
00054
00055
00056
00057
00058
00059
00060
00061
00062
00063
00064
00065
00066
00067
00068
00069
00070
00071
00072
00073
00074
00075
00076
00077
00078
00079
00080
00081
00082
00083
00084
00085
00086
00087
00088
00089
00090
00091
00092
00093
00094
00095
00096
00097
00098
00099
00100
00101
00102
00103
00104
00105
00106
00107
00108
00109
00110
00111
00112
00113
00114
00115
00116
00117
00118
00119
00120
00121
00122
00123
00124
00125
00126
00127
00128
00129
00130
00131
00132
00133
00134
00135
00136
00137
00138
00139
00140
00141
00142 __version__ = '3.1.3'
00143
00144 def version_str_to_tuple(version_str):
00145 import re
00146 import sys
00147
00148 if version_str == 'HEAD':
00149 return (sys.maxint, sys.maxint, sys.maxint, sys.maxint)
00150
00151 m = re.match(r'(\d+)\.(\d+)(\.(\d+))?(b(\d+))?', version_str)
00152 if m is None:
00153 raise ValueError("Bad version string %r" % version_str)
00154
00155 major = int(m.group(1))
00156 minor = int(m.group(2))
00157 patch = int(m.group(4) or 0)
00158 beta = int(m.group(6) or sys.maxint)
00159
00160 return (major, minor, patch, beta)
00161
00162
00163 runtime_version_str = __version__
00164 runtime_version = version_str_to_tuple(runtime_version_str)
00165
00166
00167 from constants import *
00168 from dfa import *
00169 from exceptions import *
00170 from recognizers import *
00171 from streams import *
00172 from tokens import *
00173 """ANTLR3 exception hierarchy"""
00174
00175
00176 from antlr3.constants import INVALID_TOKEN_TYPE
00177
00178
00179
00180
00181 class BacktrackingFailed(Exception):
00182
00183 pass
00184
00185
00186
00187
00188
00189
00190
00191
00192
00193
00194
00195
00196
00197
00198
00199
00200
00201
00202
00203
00204
00205
00206
00207
00208
00209
00210
00211
00212
00213
00214
00215
00216
00217
00218 class RecognitionException(Exception):
00219
00220 def __init__(self, input=None):
00221 Exception.__init__(self)
00222
00223
00224 self.input = None
00225
00226
00227
00228 self.index = None
00229
00230
00231
00232
00233 self.token = None
00234
00235
00236
00237 self.node = None
00238
00239
00240 self.c = None
00241
00242
00243
00244
00245 self.line = None
00246
00247 self.charPositionInLine = None
00248
00249
00250
00251
00252
00253 self.approximateLineInfo = False
00254
00255
00256 if input is not None:
00257 self.input = input
00258 self.index = input.index()
00259
00260
00261 from antlr3.streams import TokenStream, CharStream
00262 from antlr3.tree import TreeNodeStream
00263
00264 if isinstance(self.input, TokenStream):
00265 self.token = self.input.LT(1)
00266 self.line = self.token.line
00267 self.charPositionInLine = self.token.charPositionInLine
00268
00269 if isinstance(self.input, TreeNodeStream):
00270 self.extractInformationFromTreeNodeStream(self.input)
00271
00272 else:
00273 if isinstance(self.input, CharStream):
00274 self.c = self.input.LT(1)
00275 self.line = self.input.line
00276 self.charPositionInLine = self.input.charPositionInLine
00277
00278 else:
00279 self.c = self.input.LA(1)
00280
00281 def extractInformationFromTreeNodeStream(self, nodes):
00282 from antlr3.tree import Tree, CommonTree
00283 from antlr3.tokens import CommonToken
00284
00285 self.node = nodes.LT(1)
00286 adaptor = nodes.adaptor
00287 payload = adaptor.getToken(self.node)
00288 if payload is not None:
00289 self.token = payload
00290 if payload.line <= 0:
00291
00292 i = -1
00293 priorNode = nodes.LT(i)
00294 while priorNode is not None:
00295 priorPayload = adaptor.getToken(priorNode)
00296 if priorPayload is not None and priorPayload.line > 0:
00297
00298 self.line = priorPayload.line
00299 self.charPositionInLine = priorPayload.charPositionInLine
00300 self.approximateLineInfo = True
00301 break
00302
00303 i -= 1
00304 priorNode = nodes.LT(i)
00305
00306 else:
00307 self.line = payload.line
00308 self.charPositionInLine = payload.charPositionInLine
00309
00310 elif isinstance(self.node, Tree):
00311 self.line = self.node.line
00312 self.charPositionInLine = self.node.charPositionInLine
00313 if isinstance(self.node, CommonTree):
00314 self.token = self.node.token
00315
00316 else:
00317 type = adaptor.getType(self.node)
00318 text = adaptor.getText(self.node)
00319 self.token = CommonToken(type=type, text=text)
00320
00321
00322
00323
00324 def getUnexpectedType(self):
00325
00326 from antlr3.streams import TokenStream
00327 from antlr3.tree import TreeNodeStream
00328
00329 if isinstance(self.input, TokenStream):
00330 return self.token.type
00331
00332 elif isinstance(self.input, TreeNodeStream):
00333 adaptor = self.input.treeAdaptor
00334 return adaptor.getType(self.node)
00335
00336 else:
00337 return self.c
00338
00339 unexpectedType = property(getUnexpectedType)
00340
00341
00342
00343
00344 class MismatchedTokenException(RecognitionException):
00345
00346 def __init__(self, expecting, input):
00347 RecognitionException.__init__(self, input)
00348 self.expecting = expecting
00349
00350
00351 def __str__(self):
00352
00353 return "MismatchedTokenException(%r!=%r)" % (
00354 self.getUnexpectedType(), self.expecting
00355 )
00356 __repr__ = __str__
00357
00358
00359
00360
00361 class UnwantedTokenException(MismatchedTokenException):
00362
00363 def getUnexpectedToken(self):
00364 return self.token
00365
00366
00367 def __str__(self):
00368 exp = ", expected %s" % self.expecting
00369 if self.expecting == INVALID_TOKEN_TYPE:
00370 exp = ""
00371
00372 if self.token is None:
00373 return "UnwantedTokenException(found=%s%s)" % (None, exp)
00374
00375 return "UnwantedTokenException(found=%s%s)" % (self.token.text, exp)
00376 __repr__ = __str__
00377
00378
00379
00380
00381
00382
00383
00384 class MissingTokenException(MismatchedTokenException):
00385
00386 def __init__(self, expecting, input, inserted):
00387 MismatchedTokenException.__init__(self, expecting, input)
00388
00389 self.inserted = inserted
00390
00391
00392 def getMissingType(self):
00393 return self.expecting
00394
00395
00396 def __str__(self):
00397 if self.inserted is not None and self.token is not None:
00398 return "MissingTokenException(inserted %r at %r)" % (
00399 self.inserted, self.token.text)
00400
00401 if self.token is not None:
00402 return "MissingTokenException(at %r)" % self.token.text
00403
00404 return "MissingTokenException"
00405 __repr__ = __str__
00406
00407
00408
00409
00410 class MismatchedRangeException(RecognitionException):
00411
00412 def __init__(self, a, b, input):
00413 RecognitionException.__init__(self, input)
00414
00415 self.a = a
00416 self.b = b
00417
00418
00419 def __str__(self):
00420 return "MismatchedRangeException(%r not in [%r..%r])" % (
00421 self.getUnexpectedType(), self.a, self.b
00422 )
00423 __repr__ = __str__
00424
00425
00426
00427
00428 class MismatchedSetException(RecognitionException):
00429
00430 def __init__(self, expecting, input):
00431 RecognitionException.__init__(self, input)
00432
00433 self.expecting = expecting
00434
00435
00436 def __str__(self):
00437 return "MismatchedSetException(%r not in %r)" % (
00438 self.getUnexpectedType(), self.expecting
00439 )
00440 __repr__ = __str__
00441
00442
00443
00444
00445 class MismatchedNotSetException(MismatchedSetException):
00446
00447 def __str__(self):
00448 return "MismatchedNotSetException(%r!=%r)" % (
00449 self.getUnexpectedType(), self.expecting
00450 )
00451 __repr__ = __str__
00452
00453
00454
00455
00456 class NoViableAltException(RecognitionException):
00457
00458 def __init__(
00459 self, grammarDecisionDescription, decisionNumber, stateNumber, input
00460 ):
00461 RecognitionException.__init__(self, input)
00462
00463 self.grammarDecisionDescription = grammarDecisionDescription
00464 self.decisionNumber = decisionNumber
00465 self.stateNumber = stateNumber
00466
00467
00468 def __str__(self):
00469 return "NoViableAltException(%r!=[%r])" % (
00470 self.unexpectedType, self.grammarDecisionDescription
00471 )
00472 __repr__ = __str__
00473
00474
00475
00476
00477 class EarlyExitException(RecognitionException):
00478
00479 def __init__(self, decisionNumber, input):
00480 RecognitionException.__init__(self, input)
00481
00482 self.decisionNumber = decisionNumber
00483
00484
00485
00486
00487
00488
00489
00490
00491
00492
00493 class FailedPredicateException(RecognitionException):
00494
00495 def __init__(self, input, ruleName, predicateText):
00496 RecognitionException.__init__(self, input)
00497
00498 self.ruleName = ruleName
00499 self.predicateText = predicateText
00500
00501
00502 def __str__(self):
00503 return "FailedPredicateException("+self.ruleName+",{"+self.predicateText+"}?)"
00504 __repr__ = __str__
00505
00506
00507
00508
00509 class MismatchedTreeNodeException(RecognitionException):
00510
00511 def __init__(self, expecting, input):
00512 RecognitionException.__init__(self, input)
00513
00514 self.expecting = expecting
00515
00516 def __str__(self):
00517 return "MismatchedTreeNodeException(%r!=%r)" % (
00518 self.getUnexpectedType(), self.expecting
00519 )
00520 __repr__ = __str__
00521 """ANTLR3 runtime package"""
00522
00523
00524 EOF = -1
00525
00526
00527
00528
00529 DEFAULT_CHANNEL = 0
00530
00531
00532
00533 HIDDEN_CHANNEL = 99
00534
00535
00536 EOR_TOKEN_TYPE = 1
00537
00538
00539
00540 DOWN = 2
00541
00542
00543 UP = 3
00544
00545 MIN_TOKEN_TYPE = UP+1
00546
00547 INVALID_TOKEN_TYPE = 0
00548
00549 """ANTLR3 runtime package"""
00550
00551 """ANTLR3 runtime package"""
00552
00553
00554 from antlr3.constants import EOF, DEFAULT_CHANNEL, INVALID_TOKEN_TYPE
00555
00556
00557
00558
00559
00560
00561
00562
00563
00564 class Token(object):
00565
00566
00567
00568
00569
00570
00571 def getText(self):
00572 raise NotImplementedError
00573
00574
00575
00576
00577
00578
00579 def setText(self, text):
00580 raise NotImplementedError
00581
00582
00583
00584
00585
00586
00587 def getType(self):
00588
00589 raise NotImplementedError
00590
00591
00592
00593
00594
00595 def setType(self, ttype):
00596
00597 raise NotImplementedError
00598
00599
00600
00601
00602
00603
00604
00605
00606 def getLine(self):
00607
00608 raise NotImplementedError
00609
00610
00611
00612
00613
00614 def setLine(self, line):
00615
00616 raise NotImplementedError
00617
00618
00619
00620
00621
00622
00623
00624
00625 def getCharPositionInLine(self):
00626
00627 raise NotImplementedError
00628
00629
00630
00631
00632
00633 def setCharPositionInLine(self, pos):
00634
00635 raise NotImplementedError
00636
00637
00638
00639
00640
00641
00642 def getChannel(self):
00643
00644 raise NotImplementedError
00645
00646
00647
00648
00649
00650 def setChannel(self, channel):
00651
00652 raise NotImplementedError
00653
00654
00655
00656
00657
00658
00659
00660
00661
00662 def getTokenIndex(self):
00663
00664 raise NotImplementedError
00665
00666
00667
00668
00669
00670 def setTokenIndex(self, index):
00671
00672 raise NotImplementedError
00673
00674
00675
00676
00677
00678
00679
00680 def getInputStream(self):
00681
00682 raise NotImplementedError
00683
00684
00685
00686
00687
00688
00689 def setInputStream(self, input):
00690
00691 raise NotImplementedError
00692
00693
00694
00695
00696
00697
00698
00699
00700
00701
00702
00703
00704
00705
00706
00707
00708
00709
00710
00711
00712 class CommonToken(Token):
00713
00714 def __init__(self, type=None, channel=DEFAULT_CHANNEL, text=None,
00715 input=None, start=None, stop=None, oldToken=None):
00716 Token.__init__(self)
00717
00718 if oldToken is not None:
00719 self.type = oldToken.type
00720 self.line = oldToken.line
00721 self.charPositionInLine = oldToken.charPositionInLine
00722 self.channel = oldToken.channel
00723 self.index = oldToken.index
00724 self._text = oldToken._text
00725 if isinstance(oldToken, CommonToken):
00726 self.input = oldToken.input
00727 self.start = oldToken.start
00728 self.stop = oldToken.stop
00729
00730 else:
00731 self.type = type
00732 self.input = input
00733 self.charPositionInLine = -1
00734 self.line = 0
00735 self.channel = channel
00736
00737
00738 self.index = -1
00739
00740
00741
00742
00743 self._text = text
00744
00745
00746 self.start = start
00747
00748
00749
00750 self.stop = stop
00751
00752
00753 def getText(self):
00754 if self._text is not None:
00755 return self._text
00756
00757 if self.input is None:
00758 return None
00759
00760 return self.input.substring(self.start, self.stop)
00761
00762
00763
00764
00765
00766
00767
00768
00769
00770 def setText(self, text):
00771 self._text = text
00772
00773 text = property(getText, setText)
00774
00775
00776 def getType(self):
00777 return self.type
00778
00779 def setType(self, ttype):
00780 self.type = ttype
00781
00782
00783 def getLine(self):
00784 return self.line
00785
00786 def setLine(self, line):
00787 self.line = line
00788
00789
00790 def getCharPositionInLine(self):
00791 return self.charPositionInLine
00792
00793 def setCharPositionInLine(self, pos):
00794 self.charPositionInLine = pos
00795
00796
00797 def getChannel(self):
00798 return self.channel
00799
00800 def setChannel(self, channel):
00801 self.channel = channel
00802
00803
00804 def getTokenIndex(self):
00805 return self.index
00806
00807 def setTokenIndex(self, index):
00808 self.index = index
00809
00810
00811 def getInputStream(self):
00812 return self.input
00813
00814 def setInputStream(self, input):
00815 self.input = input
00816
00817
00818 def __str__(self):
00819 if self.type == EOF:
00820 return "<EOF>"
00821
00822 channelStr = ""
00823 if self.channel > 0:
00824 channelStr = ",channel=" + str(self.channel)
00825
00826 txt = self.text
00827 if txt is not None:
00828 txt = txt.replace("\n","\\\\n")
00829 txt = txt.replace("\r","\\\\r")
00830 txt = txt.replace("\t","\\\\t")
00831 else:
00832 txt = "<no text>"
00833
00834 return "[@%d,%d:%d=%r,<%d>%s,%d:%d]" % (
00835 self.index,
00836 self.start, self.stop,
00837 txt,
00838 self.type, channelStr,
00839 self.line, self.charPositionInLine
00840 )
00841
00842
00843
00844
00845
00846
00847
00848
00849
00850
00851
00852
00853 class ClassicToken(Token):
00854
00855 def __init__(self, type=None, text=None, channel=DEFAULT_CHANNEL,
00856 oldToken=None
00857 ):
00858 Token.__init__(self)
00859
00860 if oldToken is not None:
00861 self.text = oldToken.text
00862 self.type = oldToken.type
00863 self.line = oldToken.line
00864 self.charPositionInLine = oldToken.charPositionInLine
00865 self.channel = oldToken.channel
00866
00867 self.text = text
00868 self.type = type
00869 self.line = None
00870 self.charPositionInLine = None
00871 self.channel = channel
00872 self.index = None
00873
00874
00875 def getText(self):
00876 return self.text
00877
00878 def setText(self, text):
00879 self.text = text
00880
00881
00882 def getType(self):
00883 return self.type
00884
00885 def setType(self, ttype):
00886 self.type = ttype
00887
00888
00889 def getLine(self):
00890 return self.line
00891
00892 def setLine(self, line):
00893 self.line = line
00894
00895
00896 def getCharPositionInLine(self):
00897 return self.charPositionInLine
00898
00899 def setCharPositionInLine(self, pos):
00900 self.charPositionInLine = pos
00901
00902
00903 def getChannel(self):
00904 return self.channel
00905
00906 def setChannel(self, channel):
00907 self.channel = channel
00908
00909
00910 def getTokenIndex(self):
00911 return self.index
00912
00913 def setTokenIndex(self, index):
00914 self.index = index
00915
00916
00917 def getInputStream(self):
00918 return None
00919
00920 def setInputStream(self, input):
00921 pass
00922
00923
00924 def toString(self):
00925 channelStr = ""
00926 if self.channel > 0:
00927 channelStr = ",channel=" + str(self.channel)
00928
00929 txt = self.text
00930 if txt is None:
00931 txt = "<no text>"
00932
00933 return "[@%r,%r,<%r>%s,%r:%r]" % (self.index,
00934 txt,
00935 self.type,
00936 channelStr,
00937 self.line,
00938 self.charPositionInLine
00939 )
00940
00941
00942 __str__ = toString
00943 __repr__ = toString
00944
00945
00946
00947 EOF_TOKEN = CommonToken(type=EOF)
00948
00949 INVALID_TOKEN = CommonToken(type=INVALID_TOKEN_TYPE)
00950
00951
00952
00953 SKIP_TOKEN = CommonToken(type=INVALID_TOKEN_TYPE)
00954
00955
00956 """ANTLR3 runtime package"""
00957
00958
00959 import codecs
00960 from StringIO import StringIO
00961
00962 from antlr3.constants import DEFAULT_CHANNEL, EOF
00963 from antlr3.tokens import Token, EOF_TOKEN
00964
00965
00966
00967
00968
00969
00970
00971
00972
00973
00974
00975
00976
00977
00978
00979
00980
00981
00982
00983
00984 class IntStream(object):
00985
00986 def consume(self):
00987 raise NotImplementedError
00988
00989
00990
00991
00992
00993
00994
00995
00996
00997 def LA(self, i):
00998
00999 raise NotImplementedError
01000
01001
01002
01003
01004
01005
01006
01007
01008
01009
01010
01011 def mark(self):
01012
01013 raise NotImplementedError
01014
01015
01016
01017
01018
01019
01020
01021
01022 def index(self):
01023
01024 raise NotImplementedError
01025
01026
01027
01028
01029
01030
01031
01032
01033
01034
01035
01036
01037
01038
01039
01040
01041
01042
01043
01044
01045
01046
01047 def rewind(self, marker=None):
01048
01049 raise NotImplementedError
01050
01051
01052
01053
01054
01055
01056
01057
01058
01059
01060
01061
01062 def release(self, marker=None):
01063
01064 raise NotImplementedError
01065
01066
01067
01068
01069
01070
01071
01072
01073
01074
01075
01076
01077
01078
01079
01080
01081
01082
01083
01084
01085
01086
01087
01088
01089
01090 def seek(self, index):
01091
01092 raise NotImplementedError
01093
01094
01095
01096
01097
01098
01099
01100
01101 def size(self):
01102
01103 raise NotImplementedError
01104
01105
01106
01107
01108
01109
01110
01111
01112 def getSourceName(self):
01113
01114 raise NotImplementedError
01115
01116
01117
01118
01119
01120
01121
01122
01123
01124 class CharStream(IntStream):
01125
01126
01127
01128
01129 EOF = -1
01130
01131
01132
01133
01134
01135
01136
01137
01138 def substring(self, start, stop):
01139
01140 raise NotImplementedError
01141
01142
01143
01144
01145
01146
01147
01148
01149
01150 def LT(self, i):
01151
01152 raise NotImplementedError
01153
01154
01155
01156
01157 def getLine(self):
01158
01159 raise NotImplementedError
01160
01161
01162
01163
01164
01165
01166 def setLine(self, line):
01167
01168 raise NotImplementedError
01169
01170
01171
01172
01173
01174
01175 def getCharPositionInLine(self):
01176
01177 raise NotImplementedError
01178
01179
01180 def setCharPositionInLine(self, pos):
01181 raise NotImplementedError
01182
01183
01184
01185
01186
01187
01188
01189
01190
01191
01192 class TokenStream(IntStream):
01193
01194
01195
01196
01197
01198
01199
01200
01201
01202
01203
01204
01205 def LT(self, k):
01206
01207 raise NotImplementedError
01208
01209
01210
01211
01212
01213
01214
01215
01216
01217
01218
01219 def get(self, i):
01220
01221 raise NotImplementedError
01222
01223
01224
01225
01226
01227
01228
01229 def getTokenSource(self):
01230
01231 raise NotImplementedError
01232
01233
01234
01235
01236
01237
01238
01239
01240
01241
01242
01243
01244
01245
01246
01247 def toString(self, start=None, stop=None):
01248
01249 raise NotImplementedError
01250
01251
01252
01253
01254
01255
01256
01257
01258
01259
01260
01261
01262
01263
01264
01265
01266
01267
01268
01269 class ANTLRStringStream(CharStream):
01270
01271
01272
01273
01274
01275
01276
01277
01278
01279 def __init__(self, data):
01280
01281 CharStream.__init__(self)
01282
01283
01284 self.strdata = unicode(data)
01285 self.data = [ord(c) for c in self.strdata]
01286
01287
01288 self.n = len(data)
01289
01290
01291 self.p = 0
01292
01293
01294 self.line = 1
01295
01296
01297
01298 self.charPositionInLine = 0
01299
01300
01301
01302
01303 self._markers = [ ]
01304 self.lastMarker = None
01305 self.markDepth = 0
01306
01307
01308 self.name = None
01309
01310
01311
01312
01313
01314
01315
01316
01317 def reset(self):
01318
01319 self.p = 0
01320 self.line = 1
01321 self.charPositionInLine = 0
01322 self._markers = [ ]
01323
01324
01325 def consume(self):
01326 try:
01327 if self.data[self.p] == 10:
01328 self.line += 1
01329 self.charPositionInLine = 0
01330 else:
01331 self.charPositionInLine += 1
01332
01333 self.p += 1
01334
01335 except IndexError:
01336
01337
01338 pass
01339
01340
01341
01342 def LA(self, i):
01343 if i == 0:
01344 return 0
01345
01346 if i < 0:
01347 i += 1
01348
01349 try:
01350 return self.data[self.p+i-1]
01351 except IndexError:
01352 return EOF
01353
01354
01355
01356 def LT(self, i):
01357 if i == 0:
01358 return 0
01359
01360 if i < 0:
01361 i += 1
01362
01363 try:
01364 return self.strdata[self.p+i-1]
01365 except IndexError:
01366 return EOF
01367
01368
01369
01370
01371
01372
01373
01374
01375 def index(self):
01376
01377 return self.p
01378
01379
01380 def size(self):
01381 return self.n
01382
01383
01384 def mark(self):
01385 state = (self.p, self.line, self.charPositionInLine)
01386 try:
01387 self._markers[self.markDepth] = state
01388 except IndexError:
01389 self._markers.append(state)
01390 self.markDepth += 1
01391
01392 self.lastMarker = self.markDepth
01393
01394 return self.lastMarker
01395
01396
01397 def rewind(self, marker=None):
01398 if marker is None:
01399 marker = self.lastMarker
01400
01401 p, line, charPositionInLine = self._markers[marker-1]
01402
01403 self.seek(p)
01404 self.line = line
01405 self.charPositionInLine = charPositionInLine
01406 self.release(marker)
01407
01408
01409 def release(self, marker=None):
01410 if marker is None:
01411 marker = self.lastMarker
01412
01413 self.markDepth = marker-1
01414
01415
01416
01417
01418
01419
01420
01421 def seek(self, index):
01422
01423 if index <= self.p:
01424 self.p = index
01425 return
01426
01427
01428 while self.p < index:
01429 self.consume()
01430
01431
01432 def substring(self, start, stop):
01433 return self.strdata[start:stop+1]
01434
01435
01436
01437
01438 def getLine(self):
01439 return self.line
01440
01441
01442
01443
01444
01445
01446
01447 def getCharPositionInLine(self):
01448 return self.charPositionInLine
01449
01450
01451
01452
01453 def setLine(self, line):
01454 self.line = line
01455
01456
01457
01458
01459
01460
01461
01462 def setCharPositionInLine(self, pos):
01463 self.charPositionInLine = pos
01464
01465
01466 def getSourceName(self):
01467 return self.name
01468
01469
01470
01471
01472
01473
01474
01475
01476
01477 class ANTLRFileStream(ANTLRStringStream):
01478
01479
01480
01481
01482
01483
01484
01485
01486
01487
01488 def __init__(self, fileName, encoding=None):
01489
01490 self.fileName = fileName
01491
01492 fp = codecs.open(fileName, 'rb', encoding)
01493 try:
01494 data = fp.read()
01495 finally:
01496 fp.close()
01497
01498 ANTLRStringStream.__init__(self, data)
01499
01500
01501
01502
01503 def getSourceName(self):
01504
01505 return self.fileName
01506
01507
01508
01509
01510
01511
01512
01513
01514
01515
01516
01517 class ANTLRInputStream(ANTLRStringStream):
01518
01519
01520
01521
01522
01523
01524
01525
01526
01527
01528 def __init__(self, file, encoding=None):
01529
01530 if encoding is not None:
01531
01532 reader = codecs.lookup(encoding)[2]
01533 file = reader(file)
01534
01535 data = file.read()
01536
01537 ANTLRStringStream.__init__(self, data)
01538
01539
01540
01541
01542
01543 StringStream = ANTLRStringStream
01544 FileStream = ANTLRFileStream
01545 InputStream = ANTLRInputStream
01546
01547
01548
01549
01550
01551
01552
01553
01554
01555
01556
01557
01558
01559
01560
01561
01562
01563
01564
01565
01566
01567 class CommonTokenStream(TokenStream):
01568
01569
01570
01571
01572
01573
01574
01575
01576
01577
01578 def __init__(self, tokenSource=None, channel=DEFAULT_CHANNEL):
01579
01580 TokenStream.__init__(self)
01581
01582 self.tokenSource = tokenSource
01583
01584
01585
01586 self.tokens = []
01587
01588
01589 self.channelOverrideMap = {}
01590
01591
01592 self.discardSet = set()
01593
01594
01595 self.channel = channel
01596
01597
01598 self.discardOffChannelTokens = False
01599
01600
01601
01602 self.p = -1
01603
01604
01605 self.lastMarker = None
01606
01607
01608
01609
01610 def setTokenSource(self, tokenSource):
01611
01612 self.tokenSource = tokenSource
01613 self.tokens = []
01614 self.p = -1
01615 self.channel = DEFAULT_CHANNEL
01616
01617
01618 def reset(self):
01619 self.p = 0
01620 self.lastMarker = None
01621
01622
01623
01624
01625
01626
01627
01628
01629 def fillBuffer(self):
01630
01631
01632 index = 0
01633 t = self.tokenSource.nextToken()
01634 while t is not None and t.type != EOF:
01635 discard = False
01636
01637 if self.discardSet is not None and t.type in self.discardSet:
01638 discard = True
01639
01640 elif self.discardOffChannelTokens and t.channel != self.channel:
01641 discard = True
01642
01643
01644 try:
01645 overrideChannel = self.channelOverrideMap[t.type]
01646
01647 except KeyError:
01648
01649 pass
01650
01651 else:
01652 if overrideChannel == self.channel:
01653 t.channel = overrideChannel
01654 else:
01655 discard = True
01656
01657 if not discard:
01658 t.index = index
01659 self.tokens.append(t)
01660 index += 1
01661
01662 t = self.tokenSource.nextToken()
01663
01664
01665 self.p = 0
01666 self.p = self.skipOffTokenChannels(self.p)
01667
01668
01669
01670
01671
01672
01673
01674
01675
01676
01677
01678 def consume(self):
01679
01680 if self.p < len(self.tokens):
01681 self.p += 1
01682
01683 self.p = self.skipOffTokenChannels(self.p)
01684
01685
01686
01687
01688
01689
01690
01691 def skipOffTokenChannels(self, i):
01692
01693 try:
01694 while self.tokens[i].channel != self.channel:
01695 i += 1
01696 except IndexError:
01697
01698 pass
01699
01700 return i
01701
01702
01703 def skipOffTokenChannelsReverse(self, i):
01704 while i >= 0 and self.tokens[i].channel != self.channel:
01705 i -= 1
01706
01707 return i
01708
01709
01710
01711
01712
01713
01714
01715
01716
01717
01718 def setTokenTypeChannel(self, ttype, channel):
01719
01720 self.channelOverrideMap[ttype] = channel
01721
01722
01723 def discardTokenType(self, ttype):
01724 self.discardSet.add(ttype)
01725
01726
01727
01728
01729
01730
01731
01732
01733 def getTokens(self, start=None, stop=None, types=None):
01734
01735 if self.p == -1:
01736 self.fillBuffer()
01737
01738 if stop is None or stop >= len(self.tokens):
01739 stop = len(self.tokens) - 1
01740
01741 if start is None or stop < 0:
01742 start = 0
01743
01744 if start > stop:
01745 return None
01746
01747 if isinstance(types, (int, long)):
01748
01749 types = set([types])
01750
01751 filteredTokens = [
01752 token for token in self.tokens[start:stop]
01753 if types is None or token.type in types
01754 ]
01755
01756 if len(filteredTokens) == 0:
01757 return None
01758
01759 return filteredTokens
01760
01761
01762
01763
01764
01765
01766
01767 def LT(self, k):
01768
01769 if self.p == -1:
01770 self.fillBuffer()
01771
01772 if k == 0:
01773 return None
01774
01775 if k < 0:
01776 return self.LB(-k)
01777
01778 i = self.p
01779 n = 1
01780
01781 while n < k:
01782
01783 i = self.skipOffTokenChannels(i+1)
01784 n += 1
01785
01786 try:
01787 return self.tokens[i]
01788 except IndexError:
01789 return EOF_TOKEN
01790
01791
01792
01793
01794 def LB(self, k):
01795
01796 if self.p == -1:
01797 self.fillBuffer()
01798
01799 if k == 0:
01800 return None
01801
01802 if self.p - k < 0:
01803 return None
01804
01805 i = self.p
01806 n = 1
01807
01808 while n <= k:
01809
01810 i = self.skipOffTokenChannelsReverse(i-1)
01811 n += 1
01812
01813 if i < 0:
01814 return None
01815
01816 return self.tokens[i]
01817
01818
01819
01820
01821
01822
01823
01824 def get(self, i):
01825
01826 return self.tokens[i]
01827
01828
01829 def LA(self, i):
01830 return self.LT(i).type
01831
01832
01833 def mark(self):
01834 self.lastMarker = self.index()
01835 return self.lastMarker
01836
01837
01838 def release(self, marker=None):
01839
01840 pass
01841
01842
01843 def size(self):
01844 return len(self.tokens)
01845
01846
01847 def index(self):
01848 return self.p
01849
01850
01851 def rewind(self, marker=None):
01852 if marker is None:
01853 marker = self.lastMarker
01854
01855 self.seek(marker)
01856
01857
01858 def seek(self, index):
01859 self.p = index
01860
01861
01862 def getTokenSource(self):
01863 return self.tokenSource
01864
01865
01866 def getSourceName(self):
01867 return self.tokenSource.getSourceName()
01868
01869
01870 def toString(self, start=None, stop=None):
01871 if self.p == -1:
01872 self.fillBuffer()
01873
01874 if start is None:
01875 start = 0
01876 elif not isinstance(start, int):
01877 start = start.index
01878
01879 if stop is None:
01880 stop = len(self.tokens) - 1
01881 elif not isinstance(stop, int):
01882 stop = stop.index
01883
01884 if stop >= len(self.tokens):
01885 stop = len(self.tokens) - 1
01886
01887 return ''.join([t.text for t in self.tokens[start:stop+1]])
01888
01889
01890
01891
01892 class RewriteOperation(object):
01893
01894 def __init__(self, stream, index, text):
01895 self.stream = stream
01896 self.index = index
01897 self.text = text
01898
01899
01900
01901
01902
01903 def execute(self, buf):
01904
01905 return self.index
01906
01907 def toString(self):
01908 opName = self.__class__.__name__
01909 return '<%s@%d:"%s">' % (opName, self.index, self.text)
01910
01911 __str__ = toString
01912 __repr__ = toString
01913
01914
01915
01916
01917 class InsertBeforeOp(RewriteOperation):
01918
01919 def execute(self, buf):
01920 buf.write(self.text)
01921 buf.write(self.stream.tokens[self.index].text)
01922 return self.index + 1
01923
01924
01925
01926
01927
01928
01929
01930
01931
01932 class ReplaceOp(RewriteOperation):
01933
01934 def __init__(self, stream, first, last, text):
01935 RewriteOperation.__init__(self, stream, first, text)
01936 self.lastIndex = last
01937
01938
01939 def execute(self, buf):
01940 if self.text is not None:
01941 buf.write(self.text)
01942
01943 return self.lastIndex + 1
01944
01945
01946 def toString(self):
01947 return '<ReplaceOp@%d..%d:"%s">' % (
01948 self.index, self.lastIndex, self.text)
01949
01950 __str__ = toString
01951 __repr__ = toString
01952
01953
01954
01955
01956
01957
01958 class DeleteOp(ReplaceOp):
01959
01960 def __init__(self, stream, first, last):
01961 ReplaceOp.__init__(self, stream, first, last, None)
01962
01963
01964 def toString(self):
01965 return '<DeleteOp@%d..%d>' % (self.index, self.lastIndex)
01966
01967 __str__ = toString
01968 __repr__ = toString
01969
01970
01971
01972
01973
01974
01975
01976
01977
01978
01979
01980
01981
01982
01983
01984
01985
01986
01987
01988
01989
01990
01991
01992
01993
01994
01995
01996
01997
01998
01999
02000
02001
02002
02003
02004
02005
02006
02007
02008
02009
02010
02011
02012
02013
02014
02015
02016
02017
02018
02019
02020
02021
02022
02023
02024
02025 class TokenRewriteStream(CommonTokenStream):
02026
02027 DEFAULT_PROGRAM_NAME = "default"
02028 MIN_TOKEN_INDEX = 0
02029
02030 def __init__(self, tokenSource=None, channel=DEFAULT_CHANNEL):
02031 CommonTokenStream.__init__(self, tokenSource, channel)
02032
02033
02034
02035
02036 self.programs = {}
02037 self.programs[self.DEFAULT_PROGRAM_NAME] = []
02038
02039
02040 self.lastRewriteTokenIndexes = {}
02041
02042
02043
02044
02045
02046
02047
02048
02049 def rollback(self, *args):
02050
02051 if len(args) == 2:
02052 programName = args[0]
02053 instructionIndex = args[1]
02054 elif len(args) == 1:
02055 programName = self.DEFAULT_PROGRAM_NAME
02056 instructionIndex = args[0]
02057 else:
02058 raise TypeError("Invalid arguments")
02059
02060 p = self.programs.get(programName, None)
02061 if p is not None:
02062 self.programs[programName] = (
02063 p[self.MIN_TOKEN_INDEX:instructionIndex])
02064
02065
02066
02067
02068 def deleteProgram(self, programName=DEFAULT_PROGRAM_NAME):
02069
02070 self.rollback(programName, self.MIN_TOKEN_INDEX)
02071
02072
02073 def insertAfter(self, *args):
02074 if len(args) == 2:
02075 programName = self.DEFAULT_PROGRAM_NAME
02076 index = args[0]
02077 text = args[1]
02078
02079 elif len(args) == 3:
02080 programName = args[0]
02081 index = args[1]
02082 text = args[2]
02083
02084 else:
02085 raise TypeError("Invalid arguments")
02086
02087 if isinstance(index, Token):
02088
02089 index = index.index
02090
02091
02092 self.insertBefore(programName, index+1, text)
02093
02094
02095 def insertBefore(self, *args):
02096 if len(args) == 2:
02097 programName = self.DEFAULT_PROGRAM_NAME
02098 index = args[0]
02099 text = args[1]
02100
02101 elif len(args) == 3:
02102 programName = args[0]
02103 index = args[1]
02104 text = args[2]
02105
02106 else:
02107 raise TypeError("Invalid arguments")
02108
02109 if isinstance(index, Token):
02110
02111 index = index.index
02112
02113 op = InsertBeforeOp(self, index, text)
02114 rewrites = self.getProgram(programName)
02115 rewrites.append(op)
02116
02117
02118 def replace(self, *args):
02119 if len(args) == 2:
02120 programName = self.DEFAULT_PROGRAM_NAME
02121 first = args[0]
02122 last = args[0]
02123 text = args[1]
02124
02125 elif len(args) == 3:
02126 programName = self.DEFAULT_PROGRAM_NAME
02127 first = args[0]
02128 last = args[1]
02129 text = args[2]
02130
02131 elif len(args) == 4:
02132 programName = args[0]
02133 first = args[1]
02134 last = args[2]
02135 text = args[3]
02136
02137 else:
02138 raise TypeError("Invalid arguments")
02139
02140 if isinstance(first, Token):
02141
02142 first = first.index
02143
02144 if isinstance(last, Token):
02145
02146 last = last.index
02147
02148 if first > last or first < 0 or last < 0 or last >= len(self.tokens):
02149 raise ValueError(
02150 "replace: range invalid: "+first+".."+last+
02151 "(size="+len(self.tokens)+")")
02152
02153 op = ReplaceOp(self, first, last, text)
02154 rewrites = self.getProgram(programName)
02155 rewrites.append(op)
02156
02157
02158 def delete(self, *args):
02159 self.replace(*(list(args) + [None]))
02160
02161
02162 def getLastRewriteTokenIndex(self, programName=DEFAULT_PROGRAM_NAME):
02163 return self.lastRewriteTokenIndexes.get(programName, -1)
02164
02165
02166 def setLastRewriteTokenIndex(self, programName, i):
02167 self.lastRewriteTokenIndexes[programName] = i
02168
02169
02170 def getProgram(self, name):
02171 p = self.programs.get(name, None)
02172 if p is None:
02173 p = self.initializeProgram(name)
02174
02175 return p
02176
02177
02178 def initializeProgram(self, name):
02179 p = []
02180 self.programs[name] = p
02181 return p
02182
02183
02184 def toOriginalString(self, start=None, end=None):
02185 if start is None:
02186 start = self.MIN_TOKEN_INDEX
02187 if end is None:
02188 end = self.size() - 1
02189
02190 buf = StringIO()
02191 i = start
02192 while i >= self.MIN_TOKEN_INDEX and i <= end and i < len(self.tokens):
02193 buf.write(self.get(i).text)
02194 i += 1
02195
02196 return buf.getvalue()
02197
02198
02199 def toString(self, *args):
02200 if len(args) == 0:
02201 programName = self.DEFAULT_PROGRAM_NAME
02202 start = self.MIN_TOKEN_INDEX
02203 end = self.size() - 1
02204
02205 elif len(args) == 1:
02206 programName = args[0]
02207 start = self.MIN_TOKEN_INDEX
02208 end = self.size() - 1
02209
02210 elif len(args) == 2:
02211 programName = self.DEFAULT_PROGRAM_NAME
02212 start = args[0]
02213 end = args[1]
02214
02215 if start is None:
02216 start = self.MIN_TOKEN_INDEX
02217 elif not isinstance(start, int):
02218 start = start.index
02219
02220 if end is None:
02221 end = len(self.tokens) - 1
02222 elif not isinstance(end, int):
02223 end = end.index
02224
02225
02226 if end >= len(self.tokens):
02227 end = len(self.tokens) - 1
02228
02229 if start < 0:
02230 start = 0
02231
02232 rewrites = self.programs.get(programName)
02233 if rewrites is None or len(rewrites) == 0:
02234
02235 return self.toOriginalString(start, end)
02236
02237 buf = StringIO()
02238
02239
02240 indexToOp = self.reduceToSingleOperationPerIndex(rewrites)
02241
02242
02243 i = start
02244 while i <= end and i < len(self.tokens):
02245 op = indexToOp.get(i)
02246
02247 try:
02248 del indexToOp[i]
02249 except KeyError:
02250 pass
02251
02252 t = self.tokens[i]
02253 if op is None:
02254
02255 buf.write(t.text)
02256 i += 1
02257
02258 else:
02259 i = op.execute(buf)
02260
02261
02262
02263
02264 if end == len(self.tokens) - 1:
02265
02266
02267 for i in sorted(indexToOp.keys()):
02268 op = indexToOp[i]
02269 if op.index >= len(self.tokens)-1:
02270 buf.write(op.text)
02271
02272 return buf.getvalue()
02273
02274 __str__ = toString
02275
02276
02277
02278
02279
02280
02281
02282
02283
02284
02285
02286
02287
02288
02289
02290
02291
02292
02293
02294
02295
02296
02297
02298
02299
02300
02301
02302
02303
02304
02305
02306
02307
02308
02309
02310
02311
02312
02313
02314
02315
02316
02317
02318
02319
02320
02321
02322
02323
02324 def reduceToSingleOperationPerIndex(self, rewrites):
02325
02326
02327 for i, rop in enumerate(rewrites):
02328 if rop is None:
02329 continue
02330
02331 if not isinstance(rop, ReplaceOp):
02332 continue
02333
02334
02335 for j, iop in self.getKindOfOps(rewrites, InsertBeforeOp, i):
02336 if iop.index >= rop.index and iop.index <= rop.lastIndex:
02337 rewrites[j] = None
02338
02339
02340 for j, prevRop in self.getKindOfOps(rewrites, ReplaceOp, i):
02341 if (prevRop.index >= rop.index
02342 and prevRop.lastIndex <= rop.lastIndex):
02343 rewrites[j] = None
02344 continue
02345
02346
02347 disjoint = (prevRop.lastIndex < rop.index
02348 or prevRop.index > rop.lastIndex)
02349 same = (prevRop.index == rop.index
02350 and prevRop.lastIndex == rop.lastIndex)
02351 if not disjoint and not same:
02352 raise ValueError(
02353 "replace op boundaries of %s overlap with previous %s"
02354 % (rop, prevRop))
02355
02356
02357 for i, iop in enumerate(rewrites):
02358 if iop is None:
02359 continue
02360
02361 if not isinstance(iop, InsertBeforeOp):
02362 continue
02363
02364
02365 for j, prevIop in self.getKindOfOps(rewrites, InsertBeforeOp, i):
02366 if prevIop.index == iop.index:
02367
02368
02369
02370 iop.text = self.catOpText(iop.text, prevIop.text)
02371 rewrites[j] = None
02372
02373
02374 for j, rop in self.getKindOfOps(rewrites, ReplaceOp, i):
02375 if iop.index == rop.index:
02376 rop.text = self.catOpText(iop.text, rop.text)
02377 rewrites[i] = None
02378 continue
02379
02380 if iop.index >= rop.index and iop.index <= rop.lastIndex:
02381 raise ValueError(
02382 "insert op %s within boundaries of previous %s"
02383 % (iop, rop))
02384
02385 m = {}
02386 for i, op in enumerate(rewrites):
02387 if op is None:
02388 continue
02389
02390 assert op.index not in m, "should only be one op per index"
02391 m[op.index] = op
02392
02393 return m
02394
02395
02396 def catOpText(self, a, b):
02397 x = ""
02398 y = ""
02399 if a is not None:
02400 x = a
02401 if b is not None:
02402 y = b
02403 return x + y
02404
02405
02406 def getKindOfOps(self, rewrites, kind, before=None):
02407 if before is None:
02408 before = len(rewrites)
02409 elif before > len(rewrites):
02410 before = len(rewrites)
02411
02412 for i, op in enumerate(rewrites[:before]):
02413 if op is None:
02414
02415 continue
02416 if op.__class__ == kind:
02417 yield i, op
02418
02419
02420 def toDebugString(self, start=None, end=None):
02421 if start is None:
02422 start = self.MIN_TOKEN_INDEX
02423 if end is None:
02424 end = self.size() - 1
02425
02426 buf = StringIO()
02427 i = start
02428 while i >= self.MIN_TOKEN_INDEX and i <= end and i < len(self.tokens):
02429 buf.write(self.get(i))
02430 i += 1
02431
02432 return buf.getvalue()
02433 """ANTLR3 runtime package"""
02434
02435
02436 import sys
02437 import inspect
02438
02439 from antlr3 import runtime_version, runtime_version_str
02440 from antlr3.constants import DEFAULT_CHANNEL, HIDDEN_CHANNEL, EOF, \
02441 EOR_TOKEN_TYPE, INVALID_TOKEN_TYPE
02442 from antlr3.exceptions import RecognitionException, MismatchedTokenException, \
02443 MismatchedRangeException, MismatchedTreeNodeException, \
02444 NoViableAltException, EarlyExitException, MismatchedSetException, \
02445 MismatchedNotSetException, FailedPredicateException, \
02446 BacktrackingFailed, UnwantedTokenException, MissingTokenException
02447 from antlr3.tokens import CommonToken, EOF_TOKEN, SKIP_TOKEN
02448 from antlr3.compat import set, frozenset, reversed
02449
02450
02451
02452
02453
02454
02455
02456
02457
02458
02459
02460 class RecognizerSharedState(object):
02461
02462 def __init__(self):
02463
02464
02465 self.following = []
02466
02467
02468
02469
02470 self.errorRecovery = False
02471
02472
02473
02474
02475
02476
02477 self.lastErrorIndex = -1
02478
02479
02480
02481 self.backtracking = 0
02482
02483
02484
02485
02486
02487
02488
02489 self.ruleMemo = None
02490
02491
02492 self.syntaxErrors = 0
02493
02494
02495
02496
02497
02498
02499
02500
02501
02502
02503
02504
02505
02506 self.token = None
02507
02508
02509
02510
02511 self.tokenStartCharIndex = -1
02512
02513
02514 self.tokenStartLine = None
02515
02516
02517 self.tokenStartCharPositionInLine = None
02518
02519
02520 self.channel = None
02521
02522
02523 self.type = None
02524
02525
02526
02527 self.text = None
02528
02529
02530
02531
02532
02533
02534
02535
02536
02537
02538
02539 class BaseRecognizer(object):
02540
02541 MEMO_RULE_FAILED = -2
02542 MEMO_RULE_UNKNOWN = -1
02543
02544
02545 DEFAULT_TOKEN_CHANNEL = DEFAULT_CHANNEL
02546
02547
02548 HIDDEN = HIDDEN_CHANNEL
02549
02550
02551 tokenNames = None
02552
02553
02554
02555 antlr_version = (3, 0, 1, 0)
02556 antlr_version_str = "3.0.1"
02557
02558 def __init__(self, state=None):
02559
02560 self.input = None
02561
02562
02563
02564
02565
02566
02567 if state is None:
02568 state = RecognizerSharedState()
02569 self._state = state
02570
02571 if self.antlr_version > runtime_version:
02572 raise RuntimeError(
02573 "ANTLR version mismatch: "
02574 "The recognizer has been generated by V%s, but this runtime "
02575 "is V%s. Please use the V%s runtime or higher."
02576 % (self.antlr_version_str,
02577 runtime_version_str,
02578 self.antlr_version_str))
02579 elif (self.antlr_version < (3, 1, 0, 0) and
02580 self.antlr_version != runtime_version):
02581
02582
02583 raise RuntimeError(
02584 "ANTLR version mismatch: "
02585 "The recognizer has been generated by V%s, but this runtime "
02586 "is V%s. Please use the V%s runtime."
02587 % (self.antlr_version_str,
02588 runtime_version_str,
02589 self.antlr_version_str))
02590
02591
02592 def setInput(self, input):
02593 self.input = input
02594
02595
02596
02597
02598
02599
02600 def reset(self):
02601
02602
02603 if self._state is None:
02604
02605 return
02606
02607 self._state.following = []
02608 self._state.errorRecovery = False
02609 self._state.lastErrorIndex = -1
02610 self._state.syntaxErrors = 0
02611
02612 self._state.backtracking = 0
02613 if self._state.ruleMemo is not None:
02614 self._state.ruleMemo = {}
02615
02616
02617
02618
02619
02620
02621
02622
02623
02624
02625
02626
02627
02628
02629
02630 def match(self, input, ttype, follow):
02631
02632 matchedSymbol = self.getCurrentInputSymbol(input)
02633 if self.input.LA(1) == ttype:
02634 self.input.consume()
02635 self._state.errorRecovery = False
02636 return matchedSymbol
02637
02638 if self._state.backtracking > 0:
02639
02640 raise BacktrackingFailed
02641
02642 matchedSymbol = self.recoverFromMismatchedToken(input, ttype, follow)
02643 return matchedSymbol
02644
02645
02646
02647
02648 def matchAny(self, input):
02649
02650 self._state.errorRecovery = False
02651 self.input.consume()
02652
02653
02654 def mismatchIsUnwantedToken(self, input, ttype):
02655 return input.LA(2) == ttype
02656
02657
02658 def mismatchIsMissingToken(self, input, follow):
02659 if follow is None:
02660
02661
02662 return False
02663
02664
02665 if EOR_TOKEN_TYPE in follow:
02666 viableTokensFollowingThisRule = self.computeContextSensitiveRuleFOLLOW()
02667 follow = follow | viableTokensFollowingThisRule
02668
02669 if len(self._state.following) > 0:
02670
02671 follow = follow - set([EOR_TOKEN_TYPE])
02672
02673
02674
02675
02676 if input.LA(1) in follow or EOR_TOKEN_TYPE in follow:
02677 return True
02678
02679 return False
02680
02681
02682
02683
02684
02685
02686
02687
02688
02689
02690
02691
02692
02693
02694
02695
02696
02697
02698
02699
02700 def reportError(self, e):
02701
02702
02703
02704 if self._state.errorRecovery:
02705 return
02706
02707 self._state.syntaxErrors += 1
02708 self._state.errorRecovery = True
02709
02710 self.displayRecognitionError(self.tokenNames, e)
02711
02712
02713 def displayRecognitionError(self, tokenNames, e):
02714 hdr = self.getErrorHeader(e)
02715 msg = self.getErrorMessage(e, tokenNames)
02716 self.emitErrorMessage(hdr+" "+msg)
02717
02718
02719
02720
02721
02722
02723
02724
02725
02726
02727
02728
02729
02730
02731
02732
02733
02734
02735
02736
02737
02738
02739
02740
02741
02742
02743 def getErrorMessage(self, e, tokenNames):
02744
02745 if isinstance(e, UnwantedTokenException):
02746 tokenName = "<unknown>"
02747 if e.expecting == EOF:
02748 tokenName = "EOF"
02749
02750 else:
02751 tokenName = self.tokenNames[e.expecting]
02752
02753 msg = "extraneous input %s expecting %s" % (
02754 self.getTokenErrorDisplay(e.getUnexpectedToken()),
02755 tokenName
02756 )
02757
02758 elif isinstance(e, MissingTokenException):
02759 tokenName = "<unknown>"
02760 if e.expecting == EOF:
02761 tokenName = "EOF"
02762
02763 else:
02764 tokenName = self.tokenNames[e.expecting]
02765
02766 msg = "missing %s at %s" % (
02767 tokenName, self.getTokenErrorDisplay(e.token)
02768 )
02769
02770 elif isinstance(e, MismatchedTokenException):
02771 tokenName = "<unknown>"
02772 if e.expecting == EOF:
02773 tokenName = "EOF"
02774 else:
02775 tokenName = self.tokenNames[e.expecting]
02776
02777 msg = "mismatched input " \
02778 + self.getTokenErrorDisplay(e.token) \
02779 + " expecting " \
02780 + tokenName
02781
02782 elif isinstance(e, MismatchedTreeNodeException):
02783 tokenName = "<unknown>"
02784 if e.expecting == EOF:
02785 tokenName = "EOF"
02786 else:
02787 tokenName = self.tokenNames[e.expecting]
02788
02789 msg = "mismatched tree node: %s expecting %s" \
02790 % (e.node, tokenName)
02791
02792 elif isinstance(e, NoViableAltException):
02793 msg = "no viable alternative at input " \
02794 + self.getTokenErrorDisplay(e.token)
02795
02796 elif isinstance(e, EarlyExitException):
02797 msg = "required (...)+ loop did not match anything at input " \
02798 + self.getTokenErrorDisplay(e.token)
02799
02800 elif isinstance(e, MismatchedSetException):
02801 msg = "mismatched input " \
02802 + self.getTokenErrorDisplay(e.token) \
02803 + " expecting set " \
02804 + repr(e.expecting)
02805
02806 elif isinstance(e, MismatchedNotSetException):
02807 msg = "mismatched input " \
02808 + self.getTokenErrorDisplay(e.token) \
02809 + " expecting set " \
02810 + repr(e.expecting)
02811
02812 elif isinstance(e, FailedPredicateException):
02813 msg = "rule " \
02814 + e.ruleName \
02815 + " failed predicate: {" \
02816 + e.predicateText \
02817 + "}?"
02818
02819 else:
02820 msg = str(e)
02821
02822 return msg
02823
02824
02825
02826
02827
02828
02829
02830
02831
02832
02833
02834 def getNumberOfSyntaxErrors(self):
02835 return self._state.syntaxErrors
02836
02837
02838
02839
02840
02841
02842 def getErrorHeader(self, e):
02843
02844 return "line %d:%d" % (e.line, e.charPositionInLine)
02845
02846
02847
02848
02849
02850
02851
02852
02853
02854
02855
02856
02857 def getTokenErrorDisplay(self, t):
02858
02859 s = t.text
02860 if s is None:
02861 if t.type == EOF:
02862 s = "<EOF>"
02863 else:
02864 s = "<"+t.type+">"
02865
02866 return repr(s)
02867
02868
02869
02870
02871 def emitErrorMessage(self, msg):
02872 sys.stderr.write(msg + '\n')
02873
02874
02875
02876
02877
02878
02879
02880
02881
02882
02883 def recover(self, input, re):
02884
02885
02886
02887 if self._state.lastErrorIndex == input.index():
02888
02889
02890
02891
02892 input.consume()
02893
02894 self._state.lastErrorIndex = input.index()
02895 followSet = self.computeErrorRecoverySet()
02896
02897 self.beginResync()
02898 self.consumeUntil(input, followSet)
02899 self.endResync()
02900
02901
02902
02903
02904
02905
02906
02907 def beginResync(self):
02908
02909 pass
02910
02911
02912
02913
02914
02915
02916
02917 def endResync(self):
02918
02919 pass
02920
02921
02922
02923
02924
02925
02926
02927
02928
02929
02930
02931
02932
02933
02934
02935
02936
02937
02938
02939
02940
02941
02942
02943
02944
02945
02946
02947
02948
02949
02950
02951
02952
02953
02954
02955
02956
02957
02958
02959
02960
02961
02962
02963
02964
02965
02966
02967
02968
02969
02970
02971
02972
02973
02974
02975
02976
02977
02978
02979
02980
02981
02982
02983
02984
02985
02986
02987
02988
02989
02990
02991
02992
02993
02994
02995
02996
02997
02998
02999
03000
03001
03002
03003
03004
03005
03006
03007
03008
03009
03010
03011
03012
03013
03014
03015 def computeErrorRecoverySet(self):
03016
03017 return self.combineFollows(False)
03018
03019
03020
03021
03022
03023
03024
03025
03026
03027
03028
03029
03030
03031
03032
03033
03034
03035
03036
03037
03038
03039
03040
03041
03042
03043
03044
03045
03046
03047
03048
03049
03050
03051
03052
03053
03054
03055
03056
03057
03058
03059
03060
03061
03062
03063
03064
03065
03066
03067
03068
03069
03070
03071
03072
03073
03074 def computeContextSensitiveRuleFOLLOW(self):
03075
03076 return self.combineFollows(True)
03077
03078
03079 def combineFollows(self, exact):
03080 followSet = set()
03081 for idx, localFollowSet in reversed(list(enumerate(self._state.following))):
03082 followSet |= localFollowSet
03083 if exact:
03084
03085 if EOR_TOKEN_TYPE in localFollowSet:
03086
03087
03088 if idx > 0:
03089 followSet.remove(EOR_TOKEN_TYPE)
03090
03091 else:
03092
03093 break
03094
03095 return followSet
03096
03097
03098
03099
03100
03101
03102
03103
03104
03105
03106
03107
03108
03109
03110
03111
03112
03113
03114
03115
03116
03117
03118
03119
03120
03121
03122
03123
03124
03125
03126
03127
03128 def recoverFromMismatchedToken(self, input, ttype, follow):
03129
03130 e = None
03131
03132
03133 if self.mismatchIsUnwantedToken(input, ttype):
03134 e = UnwantedTokenException(ttype, input)
03135
03136 self.beginResync()
03137 input.consume()
03138 self.endResync()
03139
03140
03141 self.reportError(e)
03142
03143
03144 matchedSymbol = self.getCurrentInputSymbol(input)
03145
03146
03147 input.consume()
03148 return matchedSymbol
03149
03150
03151 if self.mismatchIsMissingToken(input, follow):
03152 inserted = self.getMissingSymbol(input, e, ttype, follow)
03153 e = MissingTokenException(ttype, input, inserted)
03154
03155
03156 self.reportError(e)
03157 return inserted
03158
03159
03160 e = MismatchedTokenException(ttype, input)
03161 raise e
03162
03163
03164
03165
03166 def recoverFromMismatchedSet(self, input, e, follow):
03167
03168 if self.mismatchIsMissingToken(input, follow):
03169 self.reportError(e)
03170
03171 return self.getMissingSymbol(input, e, INVALID_TOKEN_TYPE, follow)
03172
03173
03174 raise e
03175
03176
03177
03178
03179
03180
03181
03182
03183
03184
03185
03186
03187
03188 def getCurrentInputSymbol(self, input):
03189
03190 return None
03191
03192
03193
03194
03195
03196
03197
03198
03199
03200
03201
03202
03203
03204
03205
03206
03207
03208
03209
03210
03211
03212
03213 def getMissingSymbol(self, input, e, expectedTokenType, follow):
03214
03215 return None
03216
03217
03218
03219
03220
03221
03222
03223
03224
03225
03226
03227
03228
03229
03230
03231
03232
03233
03234
03235
03236
03237
03238
03239
03240
03241 def consumeUntil(self, input, tokenTypes):
03242
03243 if not isinstance(tokenTypes, (set, frozenset)):
03244 tokenTypes = frozenset([tokenTypes])
03245
03246 ttype = input.LA(1)
03247 while ttype != EOF and ttype not in tokenTypes:
03248 input.consume()
03249 ttype = input.LA(1)
03250
03251
03252 def getRuleInvocationStack(self):
03253
03254
03255
03256
03257
03258
03259
03260
03261
03262
03263
03264
03265
03266
03267
03268
03269
03270
03271
03272
03273 def getRuleInvocationStack(self):
03274
03275 return self._getRuleInvocationStack(self.__module__)
03276
03277
03278
03279
03280
03281
03282
03283
03284
03285
03286
03287
03288 def _getRuleInvocationStack(cls, module):
03289
03290
03291
03292
03293
03294 rules = []
03295 for frame in reversed(inspect.stack()):
03296 code = frame[0].f_code
03297 codeMod = inspect.getmodule(code)
03298 if codeMod is None:
03299 continue
03300
03301
03302 if codeMod.__name__ != module:
03303 continue
03304
03305
03306 if code.co_name in ('nextToken', '<module>'):
03307 continue
03308
03309 rules.append(code.co_name)
03310
03311 return rules
03312
03313 _getRuleInvocationStack = classmethod(_getRuleInvocationStack)
03314
03315
03316 def getBacktrackingLevel(self):
03317 return self._state.backtracking
03318
03319 def setBacktrackingLevel(self, n):
03320 self._state.backtracking = n
03321
03322
03323
03324
03325 def failed(self):
03326
03327 return self._state.failed
03328
03329
03330
03331
03332
03333
03334
03335 def getGrammarFileName(self):
03336
03337 return self.grammarFileName
03338
03339
03340 def getSourceName(self):
03341 raise NotImplementedError
03342
03343
03344
03345
03346
03347
03348
03349 def toStrings(self, tokens):
03350
03351 if tokens is None:
03352 return None
03353
03354 return [token.text for token in tokens]
03355
03356
03357
03358
03359
03360
03361
03362
03363
03364
03365 def getRuleMemoization(self, ruleIndex, ruleStartIndex):
03366
03367 if ruleIndex not in self._state.ruleMemo:
03368 self._state.ruleMemo[ruleIndex] = {}
03369
03370 return self._state.ruleMemo[ruleIndex].get(
03371 ruleStartIndex, self.MEMO_RULE_UNKNOWN
03372 )
03373
03374
03375
03376
03377
03378
03379
03380
03381
03382
03383
03384
03385
03386 def alreadyParsedRule(self, input, ruleIndex):
03387
03388 stopIndex = self.getRuleMemoization(ruleIndex, input.index())
03389 if stopIndex == self.MEMO_RULE_UNKNOWN:
03390 return False
03391
03392 if stopIndex == self.MEMO_RULE_FAILED:
03393 raise BacktrackingFailed
03394
03395 else:
03396 input.seek(stopIndex + 1)
03397
03398 return True
03399
03400
03401
03402
03403
03404
03405
03406 def memoize(self, input, ruleIndex, ruleStartIndex, success):
03407
03408 if success:
03409 stopTokenIndex = input.index() - 1
03410 else:
03411 stopTokenIndex = self.MEMO_RULE_FAILED
03412
03413 if ruleIndex in self._state.ruleMemo:
03414 self._state.ruleMemo[ruleIndex][ruleStartIndex] = stopTokenIndex
03415
03416
03417 def traceIn(self, ruleName, ruleIndex, inputSymbol):
03418 sys.stdout.write("enter %s %s" % (ruleName, inputSymbol))
03419
03420 if self._state.backtracking > 0:
03421 sys.stdout.write(" backtracking=%s" % self._state.backtracking)
03422
03423 sys.stdout.write('\n')
03424
03425
03426 def traceOut(self, ruleName, ruleIndex, inputSymbol):
03427 sys.stdout.write("exit %s %s" % (ruleName, inputSymbol))
03428
03429 if self._state.backtracking > 0:
03430 sys.stdout.write(" backtracking=%s" % self._state.backtracking)
03431
03432 if self._state.failed:
03433 sys.stdout.write(" failed")
03434 else:
03435 sys.stdout.write(" succeeded")
03436
03437 sys.stdout.write('\n')
03438
03439
03440
03441
03442
03443
03444
03445
03446
03447
03448
03449
03450
03451
03452
03453
03454
03455
03456
03457 class TokenSource(object):
03458
03459
03460
03461
03462
03463
03464
03465 def nextToken(self):
03466
03467 raise NotImplementedError
03468
03469
03470
03471
03472
03473
03474
03475
03476
03477 def __iter__(self):
03478
03479 return self
03480
03481
03482
03483
03484
03485
03486
03487
03488
03489 def next(self):
03490
03491 token = self.nextToken()
03492 if token is None or token.type == EOF:
03493 raise StopIteration
03494 return token
03495
03496
03497
03498
03499
03500
03501
03502
03503
03504
03505
03506 class Lexer(BaseRecognizer, TokenSource):
03507
03508 def __init__(self, input, state=None):
03509 BaseRecognizer.__init__(self, state)
03510 TokenSource.__init__(self)
03511
03512
03513 self.input = input
03514
03515
03516 def reset(self):
03517 BaseRecognizer.reset(self)
03518
03519 if self.input is not None:
03520
03521 self.input.seek(0)
03522
03523 if self._state is None:
03524
03525 return
03526
03527
03528 self._state.token = None
03529 self._state.type = INVALID_TOKEN_TYPE
03530 self._state.channel = DEFAULT_CHANNEL
03531 self._state.tokenStartCharIndex = -1
03532 self._state.tokenStartLine = -1
03533 self._state.tokenStartCharPositionInLine = -1
03534 self._state.text = None
03535
03536
03537
03538
03539
03540
03541
03542 def nextToken(self):
03543
03544 while 1:
03545 self._state.token = None
03546 self._state.channel = DEFAULT_CHANNEL
03547 self._state.tokenStartCharIndex = self.input.index()
03548 self._state.tokenStartCharPositionInLine = self.input.charPositionInLine
03549 self._state.tokenStartLine = self.input.line
03550 self._state.text = None
03551 if self.input.LA(1) == EOF:
03552 return EOF_TOKEN
03553
03554 try:
03555 self.mTokens()
03556
03557 if self._state.token is None:
03558 self.emit()
03559
03560 elif self._state.token == SKIP_TOKEN:
03561 continue
03562
03563 return self._state.token
03564
03565 except NoViableAltException, re:
03566 self.reportError(re)
03567 self.recover(re)
03568
03569 except RecognitionException, re:
03570 self.reportError(re)
03571
03572
03573
03574
03575
03576
03577
03578
03579
03580
03581
03582 def skip(self):
03583
03584 self._state.token = SKIP_TOKEN
03585
03586
03587
03588
03589 def mTokens(self):
03590
03591
03592 raise NotImplementedError
03593
03594
03595
03596
03597 def setCharStream(self, input):
03598 self.input = None
03599 self.reset()
03600 self.input = input
03601
03602
03603 def getSourceName(self):
03604 return self.input.getSourceName()
03605
03606
03607
03608
03609
03610
03611
03612
03613
03614
03615
03616
03617
03618 def emit(self, token=None):
03619
03620 if token is None:
03621 token = CommonToken(
03622 input=self.input,
03623 type=self._state.type,
03624 channel=self._state.channel,
03625 start=self._state.tokenStartCharIndex,
03626 stop=self.getCharIndex()-1
03627 )
03628 token.line = self._state.tokenStartLine
03629 token.text = self._state.text
03630 token.charPositionInLine = self._state.tokenStartCharPositionInLine
03631
03632 self._state.token = token
03633
03634 return token
03635
03636
03637 def match(self, s):
03638 if isinstance(s, basestring):
03639 for c in s:
03640 if self.input.LA(1) != ord(c):
03641 if self._state.backtracking > 0:
03642 raise BacktrackingFailed
03643
03644 mte = MismatchedTokenException(c, self.input)
03645 self.recover(mte)
03646 raise mte
03647
03648 self.input.consume()
03649
03650 else:
03651 if self.input.LA(1) != s:
03652 if self._state.backtracking > 0:
03653 raise BacktrackingFailed
03654
03655 mte = MismatchedTokenException(unichr(s), self.input)
03656 self.recover(mte)
03657 raise mte
03658
03659 self.input.consume()
03660
03661
03662 def matchAny(self):
03663 self.input.consume()
03664
03665
03666 def matchRange(self, a, b):
03667 if self.input.LA(1) < a or self.input.LA(1) > b:
03668 if self._state.backtracking > 0:
03669 raise BacktrackingFailed
03670
03671 mre = MismatchedRangeException(unichr(a), unichr(b), self.input)
03672 self.recover(mre)
03673 raise mre
03674
03675 self.input.consume()
03676
03677
03678 def getLine(self):
03679 return self.input.line
03680
03681
03682 def getCharPositionInLine(self):
03683 return self.input.charPositionInLine
03684
03685
03686
03687
03688 def getCharIndex(self):
03689
03690 return self.input.index()
03691
03692
03693
03694
03695
03696
03697
03698 def getText(self):
03699 if self._state.text is not None:
03700 return self._state.text
03701
03702 return self.input.substring(
03703 self._state.tokenStartCharIndex,
03704 self.getCharIndex()-1
03705 )
03706
03707
03708
03709
03710
03711
03712
03713 def setText(self, text):
03714 self._state.text = text
03715
03716
03717 text = property(getText, setText)
03718
03719
03720 def reportError(self, e):
03721
03722
03723
03724
03725
03726
03727
03728
03729
03730
03731 self.displayRecognitionError(self.tokenNames, e)
03732
03733
03734 def getErrorMessage(self, e, tokenNames):
03735 msg = None
03736
03737 if isinstance(e, MismatchedTokenException):
03738 msg = "mismatched character " \
03739 + self.getCharErrorDisplay(e.c) \
03740 + " expecting " \
03741 + self.getCharErrorDisplay(e.expecting)
03742
03743 elif isinstance(e, NoViableAltException):
03744 msg = "no viable alternative at character " \
03745 + self.getCharErrorDisplay(e.c)
03746
03747 elif isinstance(e, EarlyExitException):
03748 msg = "required (...)+ loop did not match anything at character " \
03749 + self.getCharErrorDisplay(e.c)
03750
03751 elif isinstance(e, MismatchedNotSetException):
03752 msg = "mismatched character " \
03753 + self.getCharErrorDisplay(e.c) \
03754 + " expecting set " \
03755 + repr(e.expecting)
03756
03757 elif isinstance(e, MismatchedSetException):
03758 msg = "mismatched character " \
03759 + self.getCharErrorDisplay(e.c) \
03760 + " expecting set " \
03761 + repr(e.expecting)
03762
03763 elif isinstance(e, MismatchedRangeException):
03764 msg = "mismatched character " \
03765 + self.getCharErrorDisplay(e.c) \
03766 + " expecting set " \
03767 + self.getCharErrorDisplay(e.a) \
03768 + ".." \
03769 + self.getCharErrorDisplay(e.b)
03770
03771 else:
03772 msg = BaseRecognizer.getErrorMessage(self, e, tokenNames)
03773
03774 return msg
03775
03776
03777 def getCharErrorDisplay(self, c):
03778 if c == EOF:
03779 c = '<EOF>'
03780 return repr(c)
03781
03782
03783
03784
03785
03786
03787
03788
03789
03790 def recover(self, re):
03791
03792 self.input.consume()
03793
03794
03795 def traceIn(self, ruleName, ruleIndex):
03796 inputSymbol = "%s line=%d:%s" % (self.input.LT(1),
03797 self.getLine(),
03798 self.getCharPositionInLine()
03799 )
03800
03801 BaseRecognizer.traceIn(self, ruleName, ruleIndex, inputSymbol)
03802
03803
03804 def traceOut(self, ruleName, ruleIndex):
03805 inputSymbol = "%s line=%d:%s" % (self.input.LT(1),
03806 self.getLine(),
03807 self.getCharPositionInLine()
03808 )
03809
03810 BaseRecognizer.traceOut(self, ruleName, ruleIndex, inputSymbol)
03811
03812
03813
03814
03815
03816
03817
03818 class Parser(BaseRecognizer):
03819
03820 def __init__(self, lexer, state=None):
03821 BaseRecognizer.__init__(self, state)
03822
03823 self.setTokenStream(lexer)
03824
03825
03826 def reset(self):
03827 BaseRecognizer.reset(self)
03828 if self.input is not None:
03829 self.input.seek(0)
03830
03831
03832 def getCurrentInputSymbol(self, input):
03833 return input.LT(1)
03834
03835
03836 def getMissingSymbol(self, input, e, expectedTokenType, follow):
03837 if expectedTokenType == EOF:
03838 tokenText = "<missing EOF>"
03839 else:
03840 tokenText = "<missing " + self.tokenNames[expectedTokenType] + ">"
03841 t = CommonToken(type=expectedTokenType, text=tokenText)
03842 current = input.LT(1)
03843 if current.type == EOF:
03844 current = input.LT(-1)
03845
03846 if current is not None:
03847 t.line = current.line
03848 t.charPositionInLine = current.charPositionInLine
03849 t.channel = DEFAULT_CHANNEL
03850 return t
03851
03852
03853
03854
03855 def setTokenStream(self, input):
03856
03857 self.input = None
03858 self.reset()
03859 self.input = input
03860
03861
03862 def getTokenStream(self):
03863 return self.input
03864
03865
03866 def getSourceName(self):
03867 return self.input.getSourceName()
03868
03869
03870 def traceIn(self, ruleName, ruleIndex):
03871 BaseRecognizer.traceIn(self, ruleName, ruleIndex, self.input.LT(1))
03872
03873
03874 def traceOut(self, ruleName, ruleIndex):
03875 BaseRecognizer.traceOut(self, ruleName, ruleIndex, self.input.LT(1))
03876
03877
03878
03879
03880
03881
03882 class RuleReturnScope(object):
03883
03884
03885
03886 def getStart(self):
03887 return None
03888
03889
03890
03891
03892 def getStop(self):
03893 return None
03894
03895
03896
03897
03898 def getTree(self):
03899 return None
03900
03901
03902
03903
03904 def getTemplate(self):
03905 return None
03906
03907
03908
03909
03910
03911
03912
03913
03914
03915
03916
03917
03918
03919
03920
03921
03922
03923
03924
03925
03926
03927
03928 class ParserRuleReturnScope(RuleReturnScope):
03929
03930 def __init__(self):
03931 self.start = None
03932 self.stop = None
03933
03934
03935 def getStart(self):
03936 return self.start
03937
03938
03939 def getStop(self):
03940 return self.stop
03941
03942
03943