MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

JFlex: delete our patched 1.4.3 and complete IDEA-143060

Browse Source
This commit is contained in:
Gregory.Shrago
2016-05-12 00:27:10 +03:00
parent 9bfe6eef12
commit 02c7c658e8
151 changed files with 0 additions and 39844 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/lexer/jflex-1.4/.classpath
-9
View File
@@ -1,9 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="var" path="ECLIPSE_HOME/plugins/org.apache.ant_1.5.2/ant.jar"/>
<classpathentry kind="var"
path="ECLIPSE_HOME/plugins/org.junit_3.8.1/junit.jar" sourcepath="ECLIPSE_HOME/plugins/org.junit_3.8.1/src.jar"/>
<classpathentry kind="output" path="classes"/>
</classpath>
tools/lexer/jflex-1.4/.project
-17
View File
@@ -1,17 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>jflex</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>
tools/lexer/jflex-1.4/COPYRIGHT
-303
View File
@@ -1,303 +0,0 @@
JFlex - Copying, Warranty & License
JFlex is free software, published under the terms of the GNU General Public License.
There is absolutely NO WARRANTY for JFlex, its code and its documentation.
The code generated by JFlex inherits the copyright of the specification it
was produced from. If it was your specification, you may use the generated
code without restriction.
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0.
This License applies to any program or other work which contains
a notice placed by the copyright holder saying it may be distributed
under the terms of this General Public License. The "Program", below,
refers to any such program or work, and a "work based on the Program"
means either the Program or any derivative work under copyright law:
that is to say, a work containing the Program or a portion of it,
either verbatim or with modifications and/or translated into another
language. (Hereinafter, translation is included without limitation in
the term "modification".) Each licensee is addressed as "you".
Activities other than copying, distribution and modification are not
covered by this License; they are outside its scope. The act of
running the Program is not restricted, and the output from the Program
is covered only if its contents constitute a work based on the
Program (independent of having been made by running the Program).
Whether that is true depends on what the Program does.
1.
You may copy and distribute verbatim copies of the Program's
source code as you receive it, in any medium, provided that you
conspicuously and appropriately publish on each copy an appropriate
copyright notice and disclaimer of warranty; keep intact all the
notices that refer to this License and to the absence of any warranty;
and give any other recipients of the Program a copy of this License
along with the Program.
You may charge a fee for the physical act of transferring a copy, and
you may at your option offer warranty protection in exchange for a fee.
2.
You may modify your copy or copies of the Program or any portion
of it, thus forming a work based on the Program, and copy and
distribute such modifications or work under the terms of Section 1
above, provided that you also meet all of these conditions:
a)
You must cause the modified files to carry prominent notices
stating that you changed the files and the date of any change.
b)
You must cause any work that you distribute or publish, that in
whole or in part contains or is derived from the Program or any
part thereof, to be licensed as a whole at no charge to all third
parties under the terms of this License.
c)
If the modified program normally reads commands interactively
when run, you must cause it, when started running for such
interactive use in the most ordinary way, to print or display an
announcement including an appropriate copyright notice and a
notice that there is no warranty (or else, saying that you provide
a warranty) and that users may redistribute the program under
these conditions, and telling the user how to view a copy of this
License. (Exception: if the Program itself is interactive but
does not normally print such an announcement, your work based on
the Program is not required to print an announcement.)
These requirements apply to the modified work as a whole. If
identifiable sections of that work are not derived from the Program,
and can be reasonably considered independent and separate works in
themselves, then this License, and its terms, do not apply to those
sections when you distribute them as separate works. But when you
distribute the same sections as part of a whole which is a work based
on the Program, the distribution of the whole must be on the terms of
this License, whose permissions for other licensees extend to the
entire whole, and thus to each and every part regardless of who wrote it.
Thus, it is not the intent of this section to claim rights or contest
your rights to work written entirely by you; rather, the intent is to
exercise the right to control the distribution of derivative or
collective works based on the Program.
In addition, mere aggregation of another work not based on the Program
with the Program (or with a work based on the Program) on a volume of
a storage or distribution medium does not bring the other work under
the scope of this License.
3.
You may copy and distribute the Program (or a work based on it,
under Section 2) in object code or executable form under the terms of
Sections 1 and 2 above provided that you also do one of the following:
a)
Accompany it with the complete corresponding machine-readable
source code, which must be distributed under the terms of Sections
1 and 2 above on a medium customarily used for software interchange; or,
b)
Accompany it with a written offer, valid for at least three
years, to give any third party, for a charge no more than your
cost of physically performing source distribution, a complete
machine-readable copy of the corresponding source code, to be
distributed under the terms of Sections 1 and 2 above on a medium
customarily used for software interchange; or,
c)
Accompany it with the information you received as to the offer
to distribute corresponding source code. (This alternative is
allowed only for noncommercial distribution and only if you
received the program in object code or executable form with such
an offer, in accord with Subsection b above.)
The source code for a work means the preferred form of the work for
making modifications to it. For an executable work, complete source
code means all the source code for all modules it contains, plus any
associated interface definition files, plus the scripts used to
control compilation and installation of the executable. However, as a
special exception, the source code distributed need not include
anything that is normally distributed (in either source or binary
form) with the major components (compiler, kernel, and so on) of the
operating system on which the executable runs, unless that component
itself accompanies the executable.
If distribution of executable or object code is made by offering
access to copy from a designated place, then offering equivalent
access to copy the source code from the same place counts as
distribution of the source code, even though third parties are not
compelled to copy the source along with the object code.
4.
You may not copy, modify, sublicense, or distribute the Program
except as expressly provided under this License. Any attempt
otherwise to copy, modify, sublicense or distribute the Program is
void, and will automatically terminate your rights under this License.
However, parties who have received copies, or rights, from you under
this License will not have their licenses terminated so long as such
parties remain in full compliance.
5.
You are not required to accept this License, since you have not
signed it. However, nothing else grants you permission to modify or
distribute the Program or its derivative works. These actions are
prohibited by law if you do not accept this License. Therefore, by
modifying or distributing the Program (or any work based on the
Program), you indicate your acceptance of this License to do so, and
all its terms and conditions for copying, distributing or modifying
the Program or works based on it.
6.
Each time you redistribute the Program (or any work based on the
Program), the recipient automatically receives a license from the
original licensor to copy, distribute or modify the Program subject to
these terms and conditions. You may not impose any further
restrictions on the recipients' exercise of the rights granted herein.
You are not responsible for enforcing compliance by third parties to
this License.
7.
If, as a consequence of a court judgment or allegation of patent
infringement or for any other reason (not limited to patent issues),
conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot
distribute so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you
may not distribute the Program at all. For example, if a patent
license would not permit royalty-free redistribution of the Program by
all those who receive copies directly or indirectly through you, then
the only way you could satisfy both it and this License would be to
refrain entirely from distribution of the Program.
If any portion of this section is held invalid or unenforceable under
any particular circumstance, the balance of the section is intended to
apply and the section as a whole is intended to apply in other
circumstances.
It is not the purpose of this section to induce you to infringe any
patents or other property right claims or to contest validity of any
such claims; this section has the sole purpose of protecting the
integrity of the free software distribution system, which is
implemented by public license practices. Many people have made
generous contributions to the wide range of software distributed
through that system in reliance on consistent application of that
system; it is up to the author/donor to decide if he or she is willing
to distribute software through any other system and a licensee cannot
impose that choice.
This section is intended to make thoroughly clear what is believed to
be a consequence of the rest of this License.
8.
If the distribution and/or use of the Program is restricted in
certain countries either by patents or by copyrighted interfaces, the
original copyright holder who places the Program under this License
may add an explicit geographical distribution limitation excluding
those countries, so that distribution is permitted only in or among
countries not thus excluded. In such case, this License incorporates
the limitation as if written in the body of this License.
9.
The Free Software Foundation may publish revised and/or new versions
of the General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the Program
specifies a version number of this License which applies to it and "any
later version", you have the option of following the terms and conditions
either of that version or of any later version published by the Free
Software Foundation. If the Program does not specify a version number of
this License, you may choose any version ever published by the Free Software
Foundation.
10.
If you wish to incorporate parts of the Program into other free
programs whose distribution conditions are different, write to the author
to ask for permission. For software which is copyrighted by the Free
Software Foundation, write to the Free Software Foundation; we sometimes
make exceptions for this. Our decision will be guided by the two goals
of preserving the free status of all derivatives of our free software and
of promoting the sharing and reuse of software generally.
NO WARRANTY
11.
BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
REPAIR OR CORRECTION.
12.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
POSSIBILITY OF SUCH DAMAGES.
END OF TERMS AND CONDITIONS
tools/lexer/jflex-1.4/JFlex-Changes.txt
-21
View File
@@ -1,21 +0,0 @@
The following changes to JFlex 1.4.3 have been done by JetBrains developers:
July 26, 2005:
* JFlexTask.java: Added setCharAt() method for setting the -charat option from Ant
builds.
July 8, 2005:
* Main.java, Options.java, Emitter.java: Added new command line option -charat.
The option replaces array index access when accessing the zzBufferL array with
calls to the .charAt() method.
December 21, 2009
* Applied jflex-1.4.3-unicode-character-category.patch
October 1, 2012:
* Merged pull request :
Fix JFlex's Action.GENERAL_LOOK lookahead option to work with --charAt and --sliceandcharat.
https://github.com/gshakhn/intellij-community/commit/e537a27e2bfc8e12500f00fa33e8e51ccaf47243
tools/lexer/jflex-1.4/bin/jflex
-51
View File
@@ -1,51 +0,0 @@
#!/bin/bash
#
# JFlex start script $Revision$
#
# if Java is not in your binary path, you need to supply its
# location in this script. The script automatically finds
# JFLEX_HOME when called directly, via binary path, or symbolic
# link.
#
# Site wide installation: simply make a symlink from e.g.
# /usr/bin/jflex to this script at its original position
#
#===================================================================
#
# configurables:
# path to the java interpreter
JAVA=java
# end configurables
#
#===================================================================
#
# calculate true location
PRG=`type $0`
PRG="${PRG##* }"
# If PRG is a symlink, trace it to the real home directory
while [ -L "$PRG" ]
do
newprg=$(ls -l ${PRG})
newprg="${newprg##*-> }"
[ "${newprg}" = "${newprg#/}" ] && newprg="${PRG%/*}/${newprg}"
PRG="$newprg"
done
PRG="${PRG%/*}"
JFLEX_HOME="${PRG}/.."
# --------------------------------------------------------------------
export CLASSPATH
CLASSPATH=$JFLEX_HOME/lib/JFlex.jar
$JAVA JFlex.Main $@
#for more memory:
#$JAVA -Xmx128m JFlex.Main $@
tools/lexer/jflex-1.4/bin/jflex.bat
-19
View File
@@ -1,19 +0,0 @@
@echo off
REM Please adjust the paths JFLEX_HOME and JAVA_HOME to suit your needs
REM (please do not add a trailing backslash)
set JFLEX_HOME=C:\JFLEX
REM only needed for JDK 1.1.x:
set JAVA_HOME=C:\JAVA
REM -------------------------------------------------------------------
set CLPATH=%JAVA_HOME%\lib\classes.zip;%JFLEX_HOME%\lib\JFlex.jar
REM for JDK 1.1.x
%JAVA_HOME%\bin\java -classpath %CLPATH% JFlex.Main %1 %2 %3 %4 %5 %6 %7 %8 %9
REM for JDK 1.2
rem java -Xmx128m -jar %JFLEX_HOME%\lib\JFlex.jar %1 %2 %3 %4 %5 %6 %7 %8 %9
tools/lexer/jflex-1.4/doc/COPYRIGHT
-303
View File
@@ -1,303 +0,0 @@
JFlex - Copying, Warranty & License
JFlex is free software, published under the terms of the GNU General Public License.
There is absolutely NO WARRANTY for JFlex, its code and its documentation.
The code generated by JFlex inherits the copyright of the specification it
was produced from. If it was your specification, you may use the generated
code without restriction.
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0.
This License applies to any program or other work which contains
a notice placed by the copyright holder saying it may be distributed
under the terms of this General Public License. The "Program", below,
refers to any such program or work, and a "work based on the Program"
means either the Program or any derivative work under copyright law:
that is to say, a work containing the Program or a portion of it,
either verbatim or with modifications and/or translated into another
language. (Hereinafter, translation is included without limitation in
the term "modification".) Each licensee is addressed as "you".
Activities other than copying, distribution and modification are not
covered by this License; they are outside its scope. The act of
running the Program is not restricted, and the output from the Program
is covered only if its contents constitute a work based on the
Program (independent of having been made by running the Program).
Whether that is true depends on what the Program does.
1.
You may copy and distribute verbatim copies of the Program's
source code as you receive it, in any medium, provided that you
conspicuously and appropriately publish on each copy an appropriate
copyright notice and disclaimer of warranty; keep intact all the
notices that refer to this License and to the absence of any warranty;
and give any other recipients of the Program a copy of this License
along with the Program.
You may charge a fee for the physical act of transferring a copy, and
you may at your option offer warranty protection in exchange for a fee.
2.
You may modify your copy or copies of the Program or any portion
of it, thus forming a work based on the Program, and copy and
distribute such modifications or work under the terms of Section 1
above, provided that you also meet all of these conditions:
a)
You must cause the modified files to carry prominent notices
stating that you changed the files and the date of any change.
b)
You must cause any work that you distribute or publish, that in
whole or in part contains or is derived from the Program or any
part thereof, to be licensed as a whole at no charge to all third
parties under the terms of this License.
c)
If the modified program normally reads commands interactively
when run, you must cause it, when started running for such
interactive use in the most ordinary way, to print or display an
announcement including an appropriate copyright notice and a
notice that there is no warranty (or else, saying that you provide
a warranty) and that users may redistribute the program under
these conditions, and telling the user how to view a copy of this
License. (Exception: if the Program itself is interactive but
does not normally print such an announcement, your work based on
the Program is not required to print an announcement.)
These requirements apply to the modified work as a whole. If
identifiable sections of that work are not derived from the Program,
and can be reasonably considered independent and separate works in
themselves, then this License, and its terms, do not apply to those
sections when you distribute them as separate works. But when you
distribute the same sections as part of a whole which is a work based
on the Program, the distribution of the whole must be on the terms of
this License, whose permissions for other licensees extend to the
entire whole, and thus to each and every part regardless of who wrote it.
Thus, it is not the intent of this section to claim rights or contest
your rights to work written entirely by you; rather, the intent is to
exercise the right to control the distribution of derivative or
collective works based on the Program.
In addition, mere aggregation of another work not based on the Program
with the Program (or with a work based on the Program) on a volume of
a storage or distribution medium does not bring the other work under
the scope of this License.
3.
You may copy and distribute the Program (or a work based on it,
under Section 2) in object code or executable form under the terms of
Sections 1 and 2 above provided that you also do one of the following:
a)
Accompany it with the complete corresponding machine-readable
source code, which must be distributed under the terms of Sections
1 and 2 above on a medium customarily used for software interchange; or,
b)
Accompany it with a written offer, valid for at least three
years, to give any third party, for a charge no more than your
cost of physically performing source distribution, a complete
machine-readable copy of the corresponding source code, to be
distributed under the terms of Sections 1 and 2 above on a medium
customarily used for software interchange; or,
c)
Accompany it with the information you received as to the offer
to distribute corresponding source code. (This alternative is
allowed only for noncommercial distribution and only if you
received the program in object code or executable form with such
an offer, in accord with Subsection b above.)
The source code for a work means the preferred form of the work for
making modifications to it. For an executable work, complete source
code means all the source code for all modules it contains, plus any
associated interface definition files, plus the scripts used to
control compilation and installation of the executable. However, as a
special exception, the source code distributed need not include
anything that is normally distributed (in either source or binary
form) with the major components (compiler, kernel, and so on) of the
operating system on which the executable runs, unless that component
itself accompanies the executable.
If distribution of executable or object code is made by offering
access to copy from a designated place, then offering equivalent
access to copy the source code from the same place counts as
distribution of the source code, even though third parties are not
compelled to copy the source along with the object code.
4.
You may not copy, modify, sublicense, or distribute the Program
except as expressly provided under this License. Any attempt
otherwise to copy, modify, sublicense or distribute the Program is
void, and will automatically terminate your rights under this License.
However, parties who have received copies, or rights, from you under
this License will not have their licenses terminated so long as such
parties remain in full compliance.
5.
You are not required to accept this License, since you have not
signed it. However, nothing else grants you permission to modify or
distribute the Program or its derivative works. These actions are
prohibited by law if you do not accept this License. Therefore, by
modifying or distributing the Program (or any work based on the
Program), you indicate your acceptance of this License to do so, and
all its terms and conditions for copying, distributing or modifying
the Program or works based on it.
6.
Each time you redistribute the Program (or any work based on the
Program), the recipient automatically receives a license from the
original licensor to copy, distribute or modify the Program subject to
these terms and conditions. You may not impose any further
restrictions on the recipients' exercise of the rights granted herein.
You are not responsible for enforcing compliance by third parties to
this License.
7.
If, as a consequence of a court judgment or allegation of patent
infringement or for any other reason (not limited to patent issues),
conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot
distribute so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you
may not distribute the Program at all. For example, if a patent
license would not permit royalty-free redistribution of the Program by
all those who receive copies directly or indirectly through you, then
the only way you could satisfy both it and this License would be to
refrain entirely from distribution of the Program.
If any portion of this section is held invalid or unenforceable under
any particular circumstance, the balance of the section is intended to
apply and the section as a whole is intended to apply in other
circumstances.
It is not the purpose of this section to induce you to infringe any
patents or other property right claims or to contest validity of any
such claims; this section has the sole purpose of protecting the
integrity of the free software distribution system, which is
implemented by public license practices. Many people have made
generous contributions to the wide range of software distributed
through that system in reliance on consistent application of that
system; it is up to the author/donor to decide if he or she is willing
to distribute software through any other system and a licensee cannot
impose that choice.
This section is intended to make thoroughly clear what is believed to
be a consequence of the rest of this License.
8.
If the distribution and/or use of the Program is restricted in
certain countries either by patents or by copyrighted interfaces, the
original copyright holder who places the Program under this License
may add an explicit geographical distribution limitation excluding
those countries, so that distribution is permitted only in or among
countries not thus excluded. In such case, this License incorporates
the limitation as if written in the body of this License.
9.
The Free Software Foundation may publish revised and/or new versions
of the General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the Program
specifies a version number of this License which applies to it and "any
later version", you have the option of following the terms and conditions
either of that version or of any later version published by the Free
Software Foundation. If the Program does not specify a version number of
this License, you may choose any version ever published by the Free Software
Foundation.
10.
If you wish to incorporate parts of the Program into other free
programs whose distribution conditions are different, write to the author
to ask for permission. For software which is copyrighted by the Free
Software Foundation, write to the Free Software Foundation; we sometimes
make exceptions for this. Our decision will be guided by the two goals
of preserving the free status of all derivatives of our free software and
of promoting the sharing and reuse of software generally.
NO WARRANTY
11.
BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
REPAIR OR CORRECTION.
12.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
POSSIBILITY OF SUCH DAMAGES.
END OF TERMS AND CONDITIONS
tools/lexer/jflex-1.4/doc/crossref.png
BIN
View File
Binary file not shown.
tools/lexer/jflex-1.4/doc/footnote.png
BIN
View File
Binary file not shown.
tools/lexer/jflex-1.4/doc/jflex_anttask.html
-165
View File
@@ -1,165 +0,0 @@
<html>
<head>
<meta http-equiv="Content-Language" content="en-us">
<title>JFlex Ant Task</title>
</head>
<body>
<h2><a name="JFlex">JFlex Ant Task</a></h2>
<p>
JFlex can be easily integrated with <a HREF="http://ant.apache.org/"
target="_top">Ant</a> build tool.
To use JFlex with Ant, simply copy JFlex.jar to <i>$ANT_HOME/lib/</i> directory.
</p>
<h3>Description</h3>
<p>
The JFlex Ant Task invokes the <a HREF="http://jflex.de/" target="_top">JFlex</a> lexical
analyzer generator on a grammar file.
</p>
<p>
To use the JFlex task, the following line must be placed in the Ant build file:
</p>
<pre>&lt;taskdef classname=&quot;JFlex.anttask.JFlexTask&quot; name=&quot;jflex&quot; /&gt;</pre>
<p>
The JFlex task requires the <i>file</i> attribute to be set to the source grammar file (*.flex).
Unless the target directory is specified with the <i>destdir</i> option,
the generated class will be saved to the same directory where grammar file resides.
Like javac, the JFlex task creates subdirectories in <i>destdir</i> according to the
generated class package.
</p>
<p>
This task only invokes JFlex if the grammar file is newer than the generated
files.
</p>
<h3>Parameters</h3>
<table border="1" cellpadding="2" cellspacing="0" width="80%">
<tr>
<td valign="top"><b>Attribute</b></td>
<td valign="top"><b>Description</b></td>
<td align="center" valign="top"><b>Required</b></td>
<td align="center" valign="top"><b>Default</b></td>
</tr>
<tr>
<td valign="top">file="file"</td>
<td valign="top">The grammar file to process.</td>
<td valign="top" align="center">Yes</td>
<td></td>
</tr>
<tr>
<td valign="top">destdir="dir"</td>
<td valign="top">
The directory to write the generated files to. If not set, the files
are written to the directory containing the grammar file. Note that unlike
JFlex's "-d" command line option, <i>destdir</i> causes the generated file to be written to
{destdir}/<b>{package name}</b>. This behaviour is similar to <i>javac -d dir</i>.
</td>
<td valign="top" align="center">No</td>
<td></td>
</tr>
<tr>
<td valign="top">outdir="dir"</td>
<td valign="top">
The directory to write the generated files to. If not set, the files
are written to the directory containing the grammar file. This options works
exactly like JFlex's "-d" command line option, it causes the output file to
be written to <i>dir</i> regardless of the package name.
</td>
<td valign="top" align="center">No</td>
<td></td>
</tr>
<tr>
<td valign="top">verbose</td>
<td valign="top">Display generation process messages.</td>
<td align="center" valign="top">No</td>
<td align="center" valign="top">&quot;off&quot;</td>
</tr>
<tr>
<td valign="top">dump</td>
<td valign="top">Dump character classes, NFA and DFA tables.</td>
<td align="center" valign="top">No</td>
<td align="center" valign="top">&quot;off&quot;</td>
</tr>
<tr>
<td valign="top">time or<p>timeStatistics</td>
<td valign="top">Display generation time statistics.</td>
<td align="center" valign="top">No</td>
<td align="center" valign="top">&quot;off&quot;</td>
</tr>
<tr>
<td valign="top">nomin or<p>skipMinimization</td>
<td valign="top">Skip minimization step.</td>
<td align="center" valign="top">No</td>
<td align="center" valign="top">&quot;off&quot;</td>
</tr>
<tr>
<td valign="top">skel="file" or<p>skeleton="file"</td>
<td valign="top">Use external skeleton file.</td>
<td align="center" valign="top">No</td>
<td align="center" valign="top">&quot;off&quot;</td>
</tr>
<tr>
<td valign="top">dot or<p>generateDot</td>
<td valign="top">Write graphviz .dot files for the generated automata (alpha).</td>
<td align="center" valign="top">No</td>
<td align="center" valign="top">&quot;off&quot;</td>
</tr>
<tr>
<td valign="top">nobak</td>
<td valign="top">Do not make a backup if the generated file exists.</td>
<td align="center" valign="top">No</td>
<td align="center" valign="top">&quot;off&quot;</td>
</tr>
<tr>
<td valign="top">switch</td>
<td valign="top">Use code generation method <i>switch</t>.</td>
<td align="center" valign="top">No</td>
<td align="center" valign="top">&quot;off&quot;</td>
</tr>
<tr>
<td valign="top">table</td>
<td valign="top">Use code generation method <i>table</t>.</td>
<td align="center" valign="top">No</td>
<td align="center" valign="top">&quot;off&quot;</td>
</tr>
<tr>
<td valign="top">pack</td>
<td valign="top">Use code generation method <i>pack</t>.</td>
<td align="center" valign="top">No</td>
<td align="center" valign="top">&quot;on&quot;</td>
</tr>
</table>
<h3>Example</h3>
<blockquote><pre>
&lt;jflex
file=&quot;src/parser/Parser.flex&quot;
destdir=&quot;build/generated/&quot;
/&gt;
</pre></blockquote>
<p>
JFlex generates the lexical analyzer for <tt>src/parser/Parser.flex</tt> and saves the result
to <tt>build/generated/parser/</tt>, providing <tt>Parser.flex</tt> declares to be in package <tt>parser</tt>.
</p>
<blockquote><pre>
&lt;jflex
file=&quot;src/parser/Parser.flex&quot;
destdir=&quot;build/generated/&quot;
/&gt;
&lt;javac
srcdir=&quot;build/generated/&quot;
destdir=&quot;build/classes/&quot;
/&gt;
</pre></blockquote>
<p>
The same as above plus compile generated classes to <i>build/classes</i>
</p>
<hr>
</body>
</html>
tools/lexer/jflex-1.4/doc/logo.gif
BIN
View File
Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 31 KiB

tools/lexer/jflex-1.4/doc/manual.css
-19
View File
@@ -1,19 +0,0 @@
body { background-color:#FFFFFF; }
table { background-color:#F0F0F0; }
h1 { font-size:16pt; font-style:italic; font-weight:bold; font-family:arial,helvetica;
border-top-width:medium; border-top-style:solid; padding-top:5mm; margin-top:1cm; }
h2 { font-size:14pt; font-style:italic; font-weight:bold; font-family:arial,helvetica;
border-top-width:medium; border-top-style:solid; padding-top:5mm; margin-top:1cm; }
h3 { font-size:14pt; font-weight:bold; font-family:arial,helvetica; margin-top:1cm; }
h4 { font-size:12pt; font-family:arial,helvetica; }
SMALL.TINY { font-size:xx-small }
SMALL.SCRIPTSIZE { font-size:xx-small }
SMALL.FOOTNOTESIZE { font-size:x-small }
SMALL.SMALL { font-size:small }
BIG.LARGE { font-size:large }
BIG.XLARGE { font-size:x-large }
BIG.XXLARGE { font-size:xx-large }
BIG.HUGE { font-size:xx-large }
BIG.XHUGE { font-size:xx-large }
tools/lexer/jflex-1.4/doc/manual.html
-3920
View File
File diff suppressed because it is too large Load Diff
tools/lexer/jflex-1.4/doc/manual.pdf
BIN
View File
Binary file not shown.
tools/lexer/jflex-1.4/doc/manual.ps.gz
BIN
View File
Binary file not shown.
tools/lexer/jflex-1.4/examples/binary/Makefile
-22
View File
@@ -1,22 +0,0 @@
JAVA = java
JAVAC = javac
JFLEX = jflex
# -------------------------------------------------
all: run
run: compile
$(JAVA) BinaryLexer *
compile: BinaryLexer.class StraightStreamReader.class
clean:
rm -f *~ *.class BinaryLexer.java
%.class: %.java
$(JAVAC) $^
BinaryLexer.java: binary.flex
$(JFLEX) $^
tools/lexer/jflex-1.4/examples/binary/README
-18
View File
@@ -1,18 +0,0 @@
This directory contains a small example of a scanner for binary files.
The example tries to detect if an input file is a java class file by
checking if the first 4 bytes contain the magic number 0xCAFEBABE.
It uses a custom Reader class that copies bytes to characters one to
one. The characters the scanner gets to read are the bytes from the
input stream without any conversion. Many thanks to Stephen Ostermiller
for providing the custom Reader class, the original version of that
class is available from http://www.smo.f2s.com/utils/
See the JFlex manual for details on how to scan binaries and how
this is different from scanning text files.
Files:
binary.flex the scanner spec
StraightStreamReader.java the custom reader class (by Stephen Ostermiller)
Makefile rules for make to compile and run the example
README this file
tools/lexer/jflex-1.4/examples/binary/StraightStreamReader.java
-287
View File
@@ -1,287 +0,0 @@
/*
* An InputStreamReader that does no character encoding translations.
* Copyright (C) 2001 Stephen Ostermiller <utils@Ostermiller.com>
*
* Changes: (2001 by Gerwin Klein <lsf@jflex.de>)
* - commented out package declaration for the example
* - original version at http://www.smo.f2s.com/utils/
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* See COPYING.TXT for details.
*/
// package com.Ostermiller.util;
import java.io.*;
/**
* A StraightStreamReader is a bridge from byte streams to character streams: It reads bytes
* and translates them into characters without using a character encoding. The characters
* that a StraightStreamReader returns may not be valid unicode characters but they are
* guaranteed to be in the 0x00 to 0xFF range.
* <P>
* Most of the time you want to do character encoding translation when translating bytes to
* characters. If you are planning on displaying the text, you should always do this and should
* use an InputStreamReader for the purpose. Sometimes it is useful to treat characters as bytes
* with some extra bits. In these cases you would want to use a StraightStreamReader.
* <P>
* For top efficiency, consider wrapping an StraightStreamReader within a BufferedReader. For example:<br>
* <code>BufferedReader in = new BufferedReader(new StraightStreamReader(System.in));</code>
*/
public class StraightStreamReader extends Reader{
/**
* The input stream from which all methods in this class read.
*/
private InputStream in;
/**
* A byte array to be used for calls to the InputStream. This
* is cached as a class variable to avoid object creation and
* deletion each time a read is called. This buffer may be
* null and may not be large enough. Make sure to check it
* before using it.
*/
private byte[] buffer;
/**
* Create a StraightStreamReader from an InputStream
*
* @param in InputStream to wrap a Reader around.
*/
public StraightStreamReader(InputStream in) {
this.in = in;
}
/**
* Close the stream.
*
* @throws IOException If an I/O error occurs
*/
public void close() throws IOException {
in.close();
}
/**
* Mark the present position in the stream. Subsequent calls to reset()
* will attempt to reposition the stream to this point. Not all
* character-input streams support the mark() operation.
*
* @param readAheadLimit Limit on the number of characters that may be read
* while still preserving the mark. After reading this many characters,
* attempting to reset the stream may fail.
* @throws IOException If the stream does not support mark(), or if some other I/O error occurs
*/
public void mark(int readAheadLimit) throws IOException {
in.mark(readAheadLimit);
}
/**
* Tell whether this stream supports the mark() operation.
*
* @return true if and only if this stream supports the mark operation.
*/
public boolean markSupported(){
return in.markSupported();
}
/**
* Read a single character. This method will block until a character is available, an
* I/O error occurs, or the end of the stream is reached.
*
* @return The character read, as an integer in the range 0 to 256 (0x00-0xff), or -1 if
* the end of the stream has been reached
* @throws IOException If an I/O error occurs
*/
public int read() throws IOException {
return in.read();
}
/**
* Read characters into an array. This method will block until some input is available,
* an I/O error occurs, or the end of the stream is reached.
*
* @param cbuf Destination buffer
* @return The number of bytes read, or -1 if the end of the stream has been reached
* @throws IOException If an I/O error occurs
*/
public int read(char[] cbuf) throws IOException {
return read(cbuf, 0, cbuf.length);
}
/**
* Read characters into an array. This method will block until some input is available,
* an I/O error occurs, or the end of the stream is reached.
*
* @param cbuf Destination buffer
* @param off Offset at which to start storing characters
* @param len Maximum number of characters to read
* @return The number of bytes read, or -1 if the end of the stream has been reached
* @throws IOException If an I/O error occurs
*/
public int read(char[] cbuf, int off, int len) throws IOException {
// ensure the capacity of the buffer that we will be using
// to read from the input stream
if (buffer == null || buffer.length < len){
buffer = new byte[len];
}
// read from the input stream and copy it to the character array
int length = in.read(buffer, 0, len);
for (int i=0; i<length; i++){
cbuf[off+i] = (char)(0xFF & buffer[i]);
}
return length;
}
/**
* Tell whether this stream is ready to be read.
*
* @return True if the next read() is guaranteed not to block for input, false otherwise.
* Note that returning false does not guarantee that the next read will block.
* @throws IOException If an I/O error occurs
*/
public boolean ready() throws IOException {
return (in.available() > 0);
}
/**
* Reset the stream. If the stream has been marked, then attempt to reposition it at the mark.
* If the stream has not been marked, then attempt to reset it in some way appropriate to the
* particular stream, for example by repositioning it to its starting point. Not all
* character-input streams support the reset() operation, and some support reset()
* without supporting mark().
*
* @throws IOException If the stream has not been marked, or if the mark has been invalidated,
* or if the stream does not support reset(), or if some other I/O error occurs
*/
public void reset() throws IOException {
in.reset();
}
/**
* Skip characters. This method will block until some characters are available,
* an I/O error occurs, or the end of the stream is reached.
*
* @param n The number of characters to skip
* @return The number of characters actually skipped
* @throws IllegalArgumentException If n is negative
* @throws IOException If an I/O error occurs
*/
public long skip(long n) throws IOException {
return in.skip(n);
}
/**
* Regression test for this class. If this class is working, this should
* run and print no errors.
* <P>
* This method creates a tempory file in the working directory called "test.txt".
* This file should not exist before hand, and the program should have create,
* read, write, and delete access to this file.
*
* @param args command line arguments (ignored)
*/
private static void main(String[] args){
try {
File f = new File("test.txt");
if (f.exists()){
throw new IOException(f + " already exists. I don't want to overwrite it.");
}
StraightStreamReader in;
char[] cbuf = new char[0x1000];
int read;
int totRead;
// write a file with all possible values of bytes
FileOutputStream out = new FileOutputStream(f);
for (int i=0x00; i<0x100; i++){
out.write(i);
}
out.close();
// read it back using the read single character method
in = new StraightStreamReader(new FileInputStream(f));
for (int i=0x00; i<0x100; i++){
read = in.read();
if (read != i){
System.err.println("Error: " + i + " read as " + read);
}
}
in.close();
// read as much of it back as possible with one simple buffer read.
in = new StraightStreamReader(new FileInputStream(f));
totRead = in.read(cbuf);
if (totRead != 0x100){
System.err.println("Simple buffered read did not read the full amount: 0x" + Integer.toHexString(totRead));
}
for (int i=0x00; i<totRead; i++){
if (cbuf[i] != i){
System.err.println("Error: 0x" + i + " read as 0x" + cbuf[i]);
}
}
in.close();
// read it back using buffer read method.
in = new StraightStreamReader(new FileInputStream(f));
totRead = 0;
while (totRead <= 0x100 && (read = in.read(cbuf, totRead, 0x100 - totRead)) > 0){
totRead += read;
}
if (totRead != 0x100){
System.err.println("Not enough read. Bytes read: " + Integer.toHexString(totRead));
}
for (int i=0x00; i<totRead; i++){
if (cbuf[i] != i){
System.err.println("Error: 0x" + i + " read as 0x" + cbuf[i]);
}
}
in.close();
// read it back using an offset buffer read method.
in = new StraightStreamReader(new FileInputStream(f));
totRead = 0;
while (totRead <= 0x100 && (read = in.read(cbuf, totRead+0x123, 0x100 - totRead)) > 0){
totRead += read;
}
if (totRead != 0x100){
System.err.println("Not enough read. Bytes read: " + Integer.toHexString(totRead));
}
for (int i=0x00; i<totRead; i++){
if (cbuf[i+0x123] != i){
System.err.println("Error: 0x" + i + " read as 0x" + cbuf[i+0x123]);
}
}
in.close();
// read it back using a partial offset buffer read method.
in = new StraightStreamReader(new FileInputStream(f));
totRead = 0;
while (totRead <= 0x100 && (read = in.read(cbuf, totRead+0x123, 7)) > 0){
totRead += read;
}
if (totRead != 0x100){
System.err.println("Not enough read. Bytes read: " + Integer.toHexString(totRead));
}
for (int i=0x00; i<totRead; i++){
if (cbuf[i+0x123] != i){
System.err.println("Error: 0x" + i + " read as 0x" + cbuf[i+0x123]);
}
}
in.close();
f.delete();
} catch (IOException x){
System.err.println(x.getMessage());
}
}
}
tools/lexer/jflex-1.4/examples/binary/binary.flex
-52
View File
@@ -1,52 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 1998-2004 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
import java.io.*;
%%
%class BinaryLexer
%8bit
%int
%{
public static void main(String [] argv) {
for (int i = 0; i < argv.length; i++) {
try {
System.out.print("["+argv[i]+"] is ");
BinaryLexer l = new BinaryLexer(new StraightStreamReader(new FileInputStream(argv[i])));
l.yylex();
}
catch (Exception e) {
e.printStackTrace(System.out);
System.exit(1);
}
}
}
%}
magic = \xCA \xFE \xBA \xBE
%%
{magic} [^]+ { System.out.println("a class file"); }
[^]+ { System.out.println("not a class file"); }
tools/lexer/jflex-1.4/examples/byaccj/Makefile
-27
View File
@@ -1,27 +0,0 @@
# only works with the Java extension of yacc:
# byacc/j from http://troi.lincom-asg.com/~rjamison/byacc/
JFLEX = jflex
BYACCJ = yacc -J
JAVAC = javac
# targets:
all: Parser.class
run: Parser.class
java Parser
build: clean Parser.class
clean:
rm -f *~ *.class *.java
Parser.class: Yylex.java Parser.java
$(JAVAC) Parser.java
Yylex.java: calc.flex
$(JFLEX) calc.flex
Parser.java: calc.y
$(BYACCJ) calc.y
tools/lexer/jflex-1.4/examples/byaccj/README
-22
View File
@@ -1,22 +0,0 @@
This directory contains an interoperability example for BYacc/J and JFlex
(a small calculator)
You need the parser generator byacc/j from
http://troi.lincom-asg.com/~rjamison/byacc/
for this example to work (version byacc/j 1.1 has been tested).
Use the Makefile to generate the lexer and parser.
The example can then be started with
java Parser
files:
calc.flex
JFlex specification for the lexical part of the arithmetic expressions
calc.y
BYacc/J specification and main program for the calculator
tools/lexer/jflex-1.4/examples/byaccj/calc.flex
-62
View File
@@ -1,62 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2000 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* Thanks to Larry Bell and Bob Jamison for suggestions and comments. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
%%
%byaccj
%{
private Parser yyparser;
public Yylex(java.io.Reader r, Parser yyparser) {
this(r);
this.yyparser = yyparser;
}
%}
NUM = [0-9]+ ("." [0-9]+)?
NL = \n | \r | \r\n
%%
/* operators */
"+" |
"-" |
"*" |
"/" |
"^" |
"(" |
")" { return (int) yycharat(0); }
/* newline */
{NL} { return Parser.NL; }
/* float */
{NUM} { yyparser.yylval = new ParserVal(Double.parseDouble(yytext()));
return Parser.NUM; }
/* whitespace */
[ \t]+ { }
\b { System.err.println("Sorry, backspace doesn't work"); }
/* error fallback */
[^] { System.err.println("Error: unexpected character '"+yytext()+"'"); return -1; }
tools/lexer/jflex-1.4/examples/byaccj/calc.y
-112
View File
@@ -1,112 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This is a modified version of the example from *
* http://www.lincom-asg.com/~rjamison/byacc/ *
* *
* Thanks to Larry Bell and Bob Jamison for suggestions and comments. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
%{
import java.io.*;
%}
%token NL /* newline */
%token <dval> NUM /* a number */
%type <dval> exp
%left '-' '+'
%left '*' '/'
%left NEG /* negation--unary minus */
%right '^' /* exponentiation */
%%
input: /* empty string */
| input line
;
line: NL { if (interactive) System.out.print("Expression: "); }
| exp NL { System.out.println(" = " + $1);
if (interactive) System.out.print("Expression: "); }
;
exp: NUM { $$ = $1; }
| exp '+' exp { $$ = $1 + $3; }
| exp '-' exp { $$ = $1 - $3; }
| exp '*' exp { $$ = $1 * $3; }
| exp '/' exp { $$ = $1 / $3; }
| '-' exp %prec NEG { $$ = -$2; }
| exp '^' exp { $$ = Math.pow($1, $3); }
| '(' exp ')' { $$ = $2; }
;
%%
private Yylex lexer;
private int yylex () {
int yyl_return = -1;
try {
yylval = new ParserVal(0);
yyl_return = lexer.yylex();
}
catch (IOException e) {
System.err.println("IO error :"+e);
}
return yyl_return;
}
public void yyerror (String error) {
System.err.println ("Error: " + error);
}
public Parser(Reader r) {
lexer = new Yylex(r, this);
}
static boolean interactive;
public static void main(String args[]) throws IOException {
System.out.println("BYACC/Java with JFlex Calculator Demo");
Parser yyparser;
if ( args.length > 0 ) {
// parse a file
yyparser = new Parser(new FileReader(args[0]));
}
else {
// interactive mode
System.out.println("[Quit with CTRL-D]");
System.out.print("Expression: ");
interactive = true;
yyparser = new Parser(new InputStreamReader(System.in));
}
yyparser.yyparse();
if (interactive) {
System.out.println();
System.out.println("Have a nice day");
}
}
tools/lexer/jflex-1.4/examples/cup/Main.java
-39
View File
@@ -1,39 +0,0 @@
/*
This example comes from a short article series in the Linux
Gazette by Richard A. Sevenich and Christopher Lopes, titled
"Compiler Construction Tools". The article series starts at
http://www.linuxgazette.com/issue39/sevenich.html
Small changes and updates to newest JFlex+Cup versions
by Gerwin Klein
*/
/*
Commented By: Christopher Lopes
File Name: Main.java
To Create:
After the scanner, lcalc.flex, and the parser, ycalc.cup, have been created.
> javac Main.java
To Run:
> java Main test.txt
where test.txt is an test input file for the calculator.
*/
import java.io.FileReader;
public class Main {
static public void main(String argv[]) {
/* Start the parser */
try {
parser p = new parser(new Lexer(new FileReader(argv[0])));
Object result = p.parse().value;
} catch (Exception e) {
/* do cleanup here -- possibly rethrow e */
e.printStackTrace();
}
}
}
tools/lexer/jflex-1.4/examples/cup/Makefile
-26
View File
@@ -1,26 +0,0 @@
JAVA=java
JAVAC=javac
JFLEX=jflex
CUP=$(JAVA) java_cup.Main <
all: test
test: output.txt
@(diff output.txt output.good && echo "Test OK!") || echo "Test failed!"
output.txt: Main.class test.txt
$(JAVA) Main test.txt > output.txt
Main.class: Main.java Lexer.java parser.java
%.class: %.java
$(JAVAC) $^
Lexer.java: lcalc.flex
$(JFLEX) lcalc.flex
parser.java: ycalc.cup
$(CUP) ycalc.cup
clean:
rm -f parser.java Lexer.java sym.java output.txt *.class *~
tools/lexer/jflex-1.4/examples/cup/README
-30
View File
@@ -1,30 +0,0 @@
A small JFlex+Cup example
It comes from a short article series in the Linux Gazette by Richard
A. Sevenich and Christopher Lopes, titled "Compiler Construction
Tools". The article series starts at
http://www.linuxgazette.com/issue39/sevenich.html
Small changes and updates to newest JFlex+Cup versions by Gerwin Klein
To compile:
>jflex lcalc.flex
>java java_cup.Main < ycalc.cup
>javac Main.java
To run:
>java Main test.txt
Files:
Main.java demo of a main program
Makefile makefile to compile and test the example
README this file
lcalc.flex the lexer spec
output.good how the output should look like for the test
ycalc.cup the parser spec
test.txt sample input for testing
tools/lexer/jflex-1.4/examples/cup/lcalc.flex
-142
View File
@@ -1,142 +0,0 @@
/*
This example comes from a short article series in the Linux
Gazette by Richard A. Sevenich and Christopher Lopes, titled
"Compiler Construction Tools". The article series starts at
http://www.linuxgazette.com/issue39/sevenich.html
Small changes and updates to newest JFlex+Cup versions
by Gerwin Klein
*/
/*
Commented By: Christopher Lopes
File Name: lcalc.flex
To Create: > jflex lcalc.flex
and then after the parser is created
> javac Lexer.java
*/
/* --------------------------Usercode Section------------------------ */
import java_cup.runtime.*;
%%
/* -----------------Options and Declarations Section----------------- */
/*
The name of the class JFlex will create will be Lexer.
Will write the code to the file Lexer.java.
*/
%class Lexer
/*
The current line number can be accessed with the variable yyline
and the current column number with the variable yycolumn.
*/
%line
%column
/*
Will switch to a CUP compatibility mode to interface with a CUP
generated parser.
*/
%cup
/*
Declarations
Code between %{ and %}, both of which must be at the beginning of a
line, will be copied letter to letter into the lexer class source.
Here you declare member variables and functions that are used inside
scanner actions.
*/
%{
/* To create a new java_cup.runtime.Symbol with information about
the current token, the token will have no value in this
case. */
private Symbol symbol(int type) {
return new Symbol(type, yyline, yycolumn);
}
/* Also creates a new java_cup.runtime.Symbol with information
about the current token, but this object has a value. */
private Symbol symbol(int type, Object value) {
return new Symbol(type, yyline, yycolumn, value);
}
%}
/*
Macro Declarations
These declarations are regular expressions that will be used latter
in the Lexical Rules Section.
*/
/* A line terminator is a \r (carriage return), \n (line feed), or
\r\n. */
LineTerminator = \r|\n|\r\n
/* White space is a line terminator, space, tab, or line feed. */
WhiteSpace = {LineTerminator} | [ \t\f]
/* A literal integer is is a number beginning with a number between
one and nine followed by zero or more numbers between zero and nine
or just a zero. */
dec_int_lit = 0 | [1-9][0-9]*
/* A identifier integer is a word beginning a letter between A and
Z, a and z, or an underscore followed by zero or more letters
between A and Z, a and z, zero and nine, or an underscore. */
dec_int_id = [A-Za-z_][A-Za-z_0-9]*
%%
/* ------------------------Lexical Rules Section---------------------- */
/*
This section contains regular expressions and actions, i.e. Java
code, that will be executed when the scanner matches the associated
regular expression. */
/* YYINITIAL is the state at which the lexer begins scanning. So
these regular expressions will only be matched if the scanner is in
the start state YYINITIAL. */
<YYINITIAL> {
/* Return the token SEMI declared in the class sym that was found. */
";" { return symbol(sym.SEMI); }
/* Print the token found that was declared in the class sym and then
return it. */
"+" { System.out.print(" + "); return symbol(sym.PLUS); }
"-" { System.out.print(" - "); return symbol(sym.MINUS); }
"*" { System.out.print(" * "); return symbol(sym.TIMES); }
"/" { System.out.print(" / "); return symbol(sym.DIVIDE); }
"(" { System.out.print(" ( "); return symbol(sym.LPAREN); }
")" { System.out.print(" ) "); return symbol(sym.RPAREN); }
/* If an integer is found print it out, return the token NUMBER
that represents an integer and the value of the integer that is
held in the string yytext which will get turned into an integer
before returning */
{dec_int_lit} { System.out.print(yytext());
return symbol(sym.NUMBER, new Integer(yytext())); }
/* If an identifier is found print it out, return the token ID
that represents an identifier and the default value one that is
given to all identifiers. */
{dec_int_id} { System.out.print(yytext());
return symbol(sym.ID, new Integer(1));}
/* Don't do anything if whitespace is found */
{WhiteSpace} { /* just skip what was found, do nothing */ }
}
/* No token was found for the input so through an error. Print out an
Illegal character message with the illegal character that was found. */
[^] { throw new Error("Illegal character <"+yytext()+">"); }
tools/lexer/jflex-1.4/examples/cup/output.good
-4
View File
@@ -1,4 +0,0 @@
2 + 4 = 6
5 * ( 6 - 3 ) + 1 = 16
6 / 3 * 5 + 20 = 30
4 * 76 / 31 = 9
tools/lexer/jflex-1.4/examples/cup/test.txt
-4
View File
@@ -1,4 +0,0 @@
2+4;
5*(6-3)+1;
6/3*5+20;
4*76/31;
tools/lexer/jflex-1.4/examples/cup/ycalc.cup
-234
View File
@@ -1,234 +0,0 @@
/*
This example comes from a short article series in the Linux
Gazette by Richard A. Sevenich and Christopher Lopes, titled
"Compiler Construction Tools". The article series starts at
http://www.linuxgazette.com/issue39/sevenich.html
Small changes and updates to newest JFlex+Cup versions
by Gerwin Klein
*/
/*
Commented By: Christopher Lopes
File Name: ycalc.cup
To Create: > java java_cup.Main < ycalc.cup
*/
/* ----------------------Preliminary Declarations Section--------------------*/
/* Import the class java_cup.runtime.* */
import java_cup.runtime.*;
/* Parser code to change the way the parser reports errors (include
line and column number of the error). */
parser code {:
/* Change the method report_error so it will display the line and
column of where the error occurred in the input as well as the
reason for the error which is passed into the method in the
String 'message'. */
public void report_error(String message, Object info) {
/* Create a StringBuffer called 'm' with the string 'Error' in it. */
StringBuffer m = new StringBuffer("Error");
/* Check if the information passed to the method is the same
type as the type java_cup.runtime.Symbol. */
if (info instanceof java_cup.runtime.Symbol) {
/* Declare a java_cup.runtime.Symbol object 's' with the
information in the object info that is being typecasted
as a java_cup.runtime.Symbol object. */
java_cup.runtime.Symbol s = ((java_cup.runtime.Symbol) info);
/* Check if the line number in the input is greater or
equal to zero. */
if (s.left >= 0) {
/* Add to the end of the StringBuffer error message
the line number of the error in the input. */
m.append(" in line "+(s.left+1));
/* Check if the column number in the input is greater
or equal to zero. */
if (s.right >= 0)
/* Add to the end of the StringBuffer error message
the column number of the error in the input. */
m.append(", column "+(s.right+1));
}
}
/* Add to the end of the StringBuffer error message created in
this method the message that was passed into this method. */
m.append(" : "+message);
/* Print the contents of the StringBuffer 'm', which contains
an error message, out on a line. */
System.err.println(m);
}
/* Change the method report_fatal_error so when it reports a fatal
error it will display the line and column number of where the
fatal error occurred in the input as well as the reason for the
fatal error which is passed into the method in the object
'message' and then exit.*/
public void report_fatal_error(String message, Object info) {
report_error(message, info);
System.exit(1);
}
:};
/* ------------Declaration of Terminals and Non Terminals Section----------- */
/* Terminals (tokens returned by the scanner).
Terminals that have no value are listed first and then terminals
that do have an value, in this case an integer value, are listed on
the next line down. */
terminal SEMI, PLUS, MINUS, TIMES, DIVIDE, LPAREN, RPAREN;
terminal Integer NUMBER, ID;
/* Non terminals used in the grammar section.
Non terminals that have an object value are listed first and then
non terminals that have an integer value are listed. An object
value means that it can be any type, it isn't set to a specific
type. So it could be an Integer or a String or whatever. */
non terminal Object expr_list, expr_part;
non terminal Integer expr, factor, term;
/* -------------Precedence and Associatively of Terminals Section----------- */
/*
Precedence of non terminals could be defined here. If you do define
precedence here you won't need to worry about precedence in the
Grammar Section, i.e. that TIMES should have a higher precedence
than PLUS.
The precedence defined here would look something like this where the
lower line always will have higher precedence than the line before it.
precedence left PLUS, MINUS;
precedence left TIMES, DIVIDE;
*/
/* ----------------------------Grammar Section-------------------- */
/* The grammar for our parser.
expr_list ::= expr_list expr_part
| expr_part
expr_part ::= expr SEMI
expr ::= factor PLUS expr
| factor MINUS expr
| factor
factor ::= factor TIMES term
| factor DIVIDE term
| term
term ::= LPAREN expr RPAREN
| NUMBER
| ID
*/
/* 'expr_list' is the start of our grammar. It can lead to another
'expr_list' followed by an 'expr_part' or it can just lead to an
'expr_part'. The lhs of the non terminals 'expr_list' and
'expr_part' that are in the rhs side of the production below need
to be found. Then the rhs sides of those non terminals need to be
followed in a similar manner, i.e. if there are any non terminals
in the rhs of those productions then the productions with those non
terminals need to be found and those rhs's followed. This process
keeps continuing until only terminals are found in the rhs of a
production. Then we can work our way back up the grammar bringing
any values that might have been assigned from a terminal. */
expr_list ::= expr_list expr_part
|
expr_part;
/* 'expr_part' is an 'expr' followed by the terminal 'SEMI'. The ':e'
after the non terminal 'expr' is a label an is used to access the
value of 'expr' which will be an integer. The action for the
production lies between {: and :}. This action will print out the
line " = + e" where e is the value of 'expr'. Before the action
takes places we need to go deeper into the grammar since 'expr' is
a non terminal. Whenever a non terminal is encountered on the rhs
of a production we need to find the rhs of that non terminal until
there are no more non terminals in the rhs. So when we finish
going through the grammar and don't encounter any more non
terminals in the rhs productions will return until we get back to
'expr' and at that point 'expr' will contain an integer value. */
expr_part ::= expr:e
{: System.out.println(" = " + e); :}
SEMI
;
/* 'expr' can lead to 'factor PLUS expr', 'factor MINUS expr', or
'factor'. The 'TIMES' and 'DIVIDE' productions are not at this
level. They are at a lower level in the grammar which in affect
makes them have higher precedence. Actions for the rhs of the non
terminal 'expr' return a value to 'expr'. This value that is
created is an integer and gets stored in 'RESULT' in the action.
RESULT is the label that is assigned automatically to the rhs, in
this case 'expr'. If the rhs is just 'factor' then 'f' refers to
the non terminal 'factor'. The value of 'f' is retrieved with the
function 'intValue()' and will be stored in 'RESULT'. In the other
two cases 'f' and 'e' refers to the non terminals 'factor' and
'expr' respectively with a terminal between them, either 'PLUS' or
'MINUS'. The value of each is retrieved with the same function
'intValue'. The values will be added or subtracted and then the
new integer will be stored in 'RESULT'.*/
expr ::= factor:f PLUS expr:e
{: RESULT = new Integer(f.intValue() + e.intValue()); :}
|
factor:f MINUS expr:e
{: RESULT = new Integer(f.intValue() - e.intValue()); :}
|
factor:f
{: RESULT = new Integer(f.intValue()); :}
;
/* 'factor' can lead to 'factor TIMES term', 'factor DIVIDE term', or
'term'. Since the productions for TIMES and DIVIDE are lower in
the grammar than 'PLUS' and 'MINUS' they will have higher
precedence. The same sort of actions take place in the rhs of
'factor' as in 'expr'. The only difference is the operations that
takes place on the values retrieved with 'intValue()', 'TIMES' and
'DIVIDE' here instead of 'PLUS' and 'MINUS'. */
factor ::= factor:f TIMES term:t
{: RESULT = new Integer(f.intValue() * t.intValue()); :}
|
factor:f DIVIDE term:t
{: RESULT = new Integer(f.intValue() / t.intValue()); :}
|
term:t
{: RESULT = new Integer(t.intValue()); :}
;
/* 'term' can lead to 'LPAREN expr RPAREN', 'NUMBER', or 'ID'. The
first production has the non terminal 'expr' in it so the
production with its lhs side needs to be found and followed. The
next rhs has no non terminals. So the grammar ends here and can go
back up. When it goes back up it will bring the value that was
retrieved when the scanner encounter the token 'NUMBER'. 'RESULT'
is assigned 'n', which refers to 'NUMBER', as the action for this
production. The same action occurs for 'ID', except the 'i' is
used to refer to 'ID'. 'ID' is also the only thing on the rhs of
the production. And since 'ID' is a terminal the grammar will end
here and go back up. */
term ::= LPAREN expr:e RPAREN
{: RESULT = e; :}
|
NUMBER:n
{: RESULT = n; :}
|
ID:i
{: RESULT = i; :}
;
tools/lexer/jflex-1.4/examples/interpreter/AST.java
-30
View File
@@ -1,30 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/**
* General interface for nodes in the abstract syntax tree. Contains
* only the method toString which is already inherited from Object,
* so the interface doesn't add any functionality. It only provides
* a common super type for all elements in the AST.
*/
interface AST {
public String toString(); // already inherited from Object
}
tools/lexer/jflex-1.4/examples/interpreter/Main.java
-88
View File
@@ -1,88 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
import java.io.File;
import java.io.FileReader;
import java.io.InputStreamReader;
import java.io.Reader;
/**
* Main program of the interpreter for the AS programming language.
* Based on JFlex/CUP.
*
* Steps:
* - scanning (Yylex)
* - context free parsing and AST building (yyparse)
* - build up symbol table (setSymtabs)
* - check context conditions (checkcontext)
* - prepare interpretation (prepInterp)
* - start interpretation (interpret)
*/
public class Main {
public static void main(String [] args) throws Exception {
Reader reader = null;
if (args.length == 1) {
File input = new File(args[0]);
if (!input.canRead()) {
System.out.println("Error: could not read ["+input+"]");
}
reader = new FileReader(input);
}
else {
reader = new InputStreamReader(System.in);
}
Yylex scanner = new Yylex(reader); // create scanner
SymTab symtab = new SymTab(); // set global symbol table
scanner.setSymtab(symtab);
parser parser = new parser(scanner); // create parser
Tprogram syntaxbaum = null;
try {
syntaxbaum = (Tprogram) parser.parse().value; // parse
}
catch (Exception e) {
e.printStackTrace();
}
// System.out.println(symtab);
System.out.println(syntaxbaum);
syntaxbaum.setSymtabs(); // set symbol table
// syntaxbaum.printSymtabs(); // print symbol table
syntaxbaum.checkcontext(); // CoCo (DefVar, DefFun, Arity)
if(contexterror>0) return;
syntaxbaum.prepInterp(); // var. indices and function pointers
// im Syntaxbaum setzen
syntaxbaum.interpret(); // interpretation
}
static int contexterror = 0; // number of errors in context conditions
public static void error(String s) {
System.out.println((contexterror++)+". "+s);
}
}
tools/lexer/jflex-1.4/examples/interpreter/Makefile
-42
View File
@@ -1,42 +0,0 @@
# uses JFlex >= 1.3.2, and CUP >= 0.10j
#
# targets:
#
# make all
# generates lexer, and parser, and compiles all *.java files
#
# make run (or just: make)
# starts the program on a test example
#
.SUFFIXES: $(SUFFIXES) .class .java
.java.class:
javac $*.java
FILE= Yylex.java parser.java sym.java \
SymTab.java AST.java \
Tnumber.java Tident.java Texp.java \
Tfun.java Texpinfix.java Tuminus.java \
Tboolexp.java Tifthenelse.java \
Tdekl.java Tdekllist.java \
Tparlist.java Texplist.java Tprogram.java \
Main.java \
SymtabEntry.java STEfun.java STEvar.java
run: all
java Main < example.as
all: Yylex.java parser.java $(FILE:java=class)
clean:
rm -f *.class *~ *.bak Yylex.java parser.java sym.java
Yylex.java: scanner.flex
jflex scanner.flex
parser.java: parser.cup
java java_cup.Main -interface < parser.cup
tools/lexer/jflex-1.4/examples/interpreter/README
-31
View File
@@ -1,31 +0,0 @@
This directory contains an interpreter for a small functional
programming language (called "AS"). It uses JFlex and CUP.
Directory contents:
- Main.java is the main program. It reads an AS program from
std in and interprets it.
- Makefile runs jflex, cup, javac and the compiled interpreter
- Classes beginning with "T" implement the abstract syntax tree.
They also contain context condition checking and the interpreter.
- Symtab.java, SymtabEntry.java, STEfun.java, STEvar.java implement
the symbol table.
- scanner.lex and parser.cup contain the scanner and parser.
- example.as is an example program in AS
The language is described in:
Manfred Broy: Einfuehrung in die Informatik, Springer Verlag
The files are a solution to excercise 3.40 in the book:
Manfred Broy,Bernhard Rumpe:
Uebungen zur Einfuehrung in die Informatik --
Strukturierte Aufgabensammlung mit Musterloesungen,
Springer Verlag, 2001
Both books are only available in German. Sorry.
tools/lexer/jflex-1.4/examples/interpreter/STEfun.java
-54
View File
@@ -1,54 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/**
* Symbol table entry for functions.
*
* Contains arity and reference to location of definition
*/
class STEfun extends SymtabEntry {
int arity;
Tdekl dekl; // location of definition
public STEfun(String f, Tdekl d, int a) {
super(f);
dekl=d;
arity=a;
}
public int kind() {
return SymtabEntry.FUN;
}
public String toString() {
return "function "+name+", arity "+arity;
}
public int arity() {
return arity;
}
public Tdekl getDekl() {
return dekl;
}
}
tools/lexer/jflex-1.4/examples/interpreter/STEvar.java
-57
View File
@@ -1,57 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/**
* Symbol table entry for variables.
*
* Contains index in the parameter list and a flag if it
* is an input variable.
*/
class STEvar extends SymtabEntry {
boolean is_input;
int index;
public STEvar(String v, boolean ii, int ind) {
super(v);
is_input=ii;
index=ind;
}
public int kind() {
return SymtabEntry.VAR;
}
public String toString() {
if (is_input)
return "input var "+name+" ("+index+")";
else
return "parameter "+name+" ("+index+")";
}
public int getIndex() {
return index;
}
public boolean isInput() {
return is_input;
}
}
tools/lexer/jflex-1.4/examples/interpreter/SymTab.java
-79
View File
@@ -1,79 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
import java.util.Enumeration;
import java.util.Hashtable;
/**
* Symbol table for the interpreter, contains information about
* variables and functions.
*
* For every binding location of a name a symbol will be created.
* The symbol tables are connected hierarchically by pointers to
* the predecessor. Lookup takes predecessors into account.
*/
public class SymTab {
Hashtable h; // contains the liste of words
// key: String, value: SymtabEntry
SymTab pred; // predecessor symbol table (if exists)
public SymTab() {
this(null);
}
public SymTab(SymTab p) {
h = new Hashtable();
pred = p;
}
public boolean enter(String s, SymtabEntry e) {
Object value = lookup(s);
h.put(s, e);
return(value==null);
}
public SymtabEntry lookup(String s) {
Object value = h.get(s);
if (value==null && pred!=null)
value = pred.lookup(s);
return ((SymtabEntry)value);
}
public String toString() { // for output with print
String res = "symbol table\n=============\n";
Enumeration e = h.keys();
String key;
while(e.hasMoreElements()) {
key = (String)e.nextElement();
res += key+" \t"+h.get(key)+"\n";
}
if (pred!=null) res+="++ predecessor!\n";
return(res);
}
public int size() {
return(h.size());
}
}
tools/lexer/jflex-1.4/examples/interpreter/SymtabEntry.java
-48
View File
@@ -1,48 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/**
* Symbol table entry for names, there are subclasses for
* variables and functions.
*
* Defines constants UNKNOWN, VAR und FUN as kinds of
* symbol table entries.
*/
class SymtabEntry {
String name;
public SymtabEntry(String v) {
name=v;
}
public int kind() {
return UNKNOWN;
}
public String toString() {
return("unknown "+name);
}
static final int UNKNOWN = 12;
static final int VAR = 13;
static final int FUN = 14;
}
tools/lexer/jflex-1.4/examples/interpreter/Tboolexp.java
-64
View File
@@ -1,64 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/**
* AST node for bool expressions
*/
class Tboolexp implements AST {
Texp exp1, exp2; // left and right subexpression
char kind; // '=', '<' and '!' for "<="
public Tboolexp(Texp e1, char k, Texp e2) {
exp1=e1;
kind=k;
exp2=e2;
}
public String toString() {
if (kind!='!')
return(""+exp1+kind+exp2);
else
return(exp1+"<="+exp2);
}
public void checkcontext(SymTab st) { // context conditions
exp1.checkcontext(st);
exp2.checkcontext(st);
}
public void prepInterp(SymTab st) { // set pointers and indices
exp1.prepInterp(st);
exp2.prepInterp(st);
}
public boolean interpret(int[] in, int[] par) {
int e1 = exp1.interpret(in,par);
int e2 = exp2.interpret(in,par);
switch(kind) {
case '=': return(e1==e2);
case '<': return(e1<e2);
case '!': return(e1<=e2);
}
return(false); // error
}
}
tools/lexer/jflex-1.4/examples/interpreter/Tdekl.java
-75
View File
@@ -1,75 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/**
* AST node for function declarations.
*
* Also contains a reference to the symbol table of
* the paramaters and their arity.
*/
class Tdekl implements AST {
Tident ident; // identifier
Tparlist parlist; // liste of parameter
Texp exp; // function body
public Tdekl(Tident i, Tparlist p, Texp e) {
parlist=p;
ident=i;
exp=e;
}
public String toString() {
return(ident+"("+parlist+") = \n "+exp);
}
SymTab params; // symbol table of the parameters
int arity;
public void setSymtab(SymTab st) {
params = new SymTab(st);
parlist.setSymtab(params,false,0);
arity = params.size();
boolean isNew = st.enter(ident.toString(),
new STEfun(ident.toString(),this,arity));
// CoCo (Fun)
if(!isNew) Main.error("funktion "+ident+" defined twice!");
}
public void printSymtabs() {
System.out.print("funktion "+ident.toString()+"\n"+params);
}
public void checkcontext() {
exp.checkcontext(params); // CoCo (DefFun,DefVar,Arity)
}
public void prepInterp(SymTab st) { // set pointers and indices
exp.prepInterp(params);
}
public int interpret(int[] in, int[] par) {
return(exp.interpret(in,par));
}
public int arity() { return(arity); }
}
tools/lexer/jflex-1.4/examples/interpreter/Tdekllist.java
-69
View File
@@ -1,69 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/**
* AST node for declaration lists of functions
*/
class Tdekllist implements AST {
Tdekllist dekllist; // rest list (optional null)
Tdekl dekl; // declaration
public Tdekllist(Tdekllist p, Tdekl e) {
dekllist=p;
dekl=e;
}
public Tdekllist(Tdekl e) {
dekllist=null;
dekl=e;
}
public String toString() {
if (dekllist!=null)
return(dekllist+",\n"+dekl);
else
return(dekl.toString());
}
public void setSymtab(SymTab st) {
if (dekllist!=null)
dekllist.setSymtab(st);
dekl.setSymtab(st);
}
public void printSymtabs() {
if (dekllist!=null)
dekllist.printSymtabs();
dekl.printSymtabs();
}
public void checkcontext() {
if (dekllist!=null)
dekllist.checkcontext();
dekl.checkcontext(); // CoCo (DefFun,DefVar,Arity)
} // in function body
public void prepInterp(SymTab st) { // set pointers and indices
dekl.prepInterp(st);
if (dekllist!=null) dekllist.prepInterp(st);
}
}
tools/lexer/jflex-1.4/examples/interpreter/Texp.java
-43
View File
@@ -1,43 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/**
* AST node for an integer expression.
*
* The non terminal exp is the sum of multiple variants and
* therefore modeled as an abstract class.
*
* The interpretation function <tt>interpret</tt> is called with
* valuations of input variables <tt>in</tt> and parameters
* <tt>par</tt>. Before interpret can be called, pointers
* and variable indices must be set with <tt>prepInterp</tt>.
*/
abstract class Texp implements AST {
// test context conditions (DefFun,DefVar,Arity)
abstract public void checkcontext(SymTab st);
// set pointers and indices for variables and functions
abstract public void prepInterp(SymTab st);
// interpretation
abstract public int interpret(int[] in, int[] par);
}
tools/lexer/jflex-1.4/examples/interpreter/Texpinfix.java
-64
View File
@@ -1,64 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/**
* AST node for infix expressions
*/
class Texpinfix extends Texp implements AST {
Texp exp1, exp2; // left and right sub expression
char kind; // kind ('+', '-', '*', '/')
public Texpinfix(Texp e1, char k, Texp e2) {
exp1=e1;
kind=k;
exp2=e2;
}
public String toString() {
return("("+exp1+kind+exp2+")");
}
public void checkcontext(SymTab st) { // context conditions
exp1.checkcontext(st);
exp2.checkcontext(st);
}
public void prepInterp(SymTab st) { // set pointers und indices
exp1.prepInterp(st);
exp2.prepInterp(st);
}
public int interpret(int[] in, int[] par) {
int e1 = exp1.interpret(in,par);
int e2 = exp2.interpret(in,par);
switch (kind) {
case '+': return(e1+e2);
case '-': return(e1-e2);
case '*': return(e1*e2);
case '/': return(e1/e2);
}
return -1; // error
}
}
tools/lexer/jflex-1.4/examples/interpreter/Texplist.java
-73
View File
@@ -1,73 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/**
* AST node for a list of expressions.
*
* The interpretation of a list of expressions stores the
* results of the expressions in an array that can be used
* as parameter list for function calls.
*/
class Texplist implements AST {
Texplist explist; // next list element (optional null)
Texp exp; // expression of this list node
public Texplist(Texplist p, Texp e) {
explist=p;
exp=e;
}
public Texplist(Texp e) {
explist=null;
exp=e;
}
public String toString() {
if (explist!=null)
return explist+","+exp;
else
return exp.toString();
}
public void checkcontext(SymTab st) {
if (explist!=null)
explist.checkcontext(st);
exp.checkcontext(st); // CoCo (DefFun,DefVar,Arity)
} // in expression
public int length() {
if (explist!=null)
return 1+explist.length();
else
return 1;
}
public void prepInterp(SymTab st) { // set pointers and indices
exp.prepInterp(st);
if (explist!=null) explist.prepInterp(st);
}
public void interpret(int[] in, int[] par, int[] res, int index) {
res[index] = exp.interpret(in,par);
if (explist!=null) explist.interpret(in,par,res,index+1);
}
}
tools/lexer/jflex-1.4/examples/interpreter/Tfun.java
-66
View File
@@ -1,66 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/**
* AST node for function application.
*
* Also contains pointer to declaration location of the function.
*/
class Tfun extends Texp implements AST {
Tident ident; // name of the function
Texplist explist; // parameter list
public Tfun(Tident i, Texplist e) {
ident=i;
explist=e;
}
public String toString() {
return ident+"("+explist+")";
}
public void checkcontext(SymTab st) { // CoCo (DefFun,Arity)
explist.checkcontext(st);
SymtabEntry ste = st.lookup(ident.toString());
if (ste==null)
Main.error("function not defined: "+ident);
else if (ste.kind() != SymtabEntry.FUN)
Main.error("variable used as funktion: "+ident);
else if (((STEfun)ste).arity() != explist.length())
Main.error("wrong arity at function call: "+ident);
}
Tdekl fundekl; // pointer to location of function declaration
// set pointers and indices
public void prepInterp(SymTab st) {
fundekl = ((STEfun)st.lookup(ident.toString())).getDekl();
explist.prepInterp(st);
}
public int interpret(int[] in, int[] par) {
int[] newparams = new int[fundekl.arity()];
explist.interpret(in,par,newparams,0);
return fundekl.interpret(in,newparams);
}
}
tools/lexer/jflex-1.4/examples/interpreter/Tident.java
-62
View File
@@ -1,62 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/**
* AST node for an identifier
*/
class Tident extends Texp implements AST {
String name;
public Tident(String s) {
name = s;
}
public String toString() {
return name;
}
public void checkcontext(SymTab st) { // CoCo (DefVar)
SymtabEntry ste = st.lookup(name);
if (ste==null)
Main.error("variable not defined: "+name);
else if (ste.kind() != SymtabEntry.VAR)
Main.error("function used as variable: "+name);
}
int index; // number of ident in environment
boolean is_input; // is it an input variable?
public void prepInterp(SymTab st) { // set index for environment
STEvar ste = (STEvar)st.lookup(name);
index = ste.getIndex();
is_input = ste.isInput();
}
public int interpret(int[] in, int[] par) {
if (is_input)
return(in[index]);
else
return(par[index]);
}
}
tools/lexer/jflex-1.4/examples/interpreter/Tifthenelse.java
-59
View File
@@ -1,59 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/**
* AST node for if-then-else expressions
*/
class Tifthenelse extends Texp implements AST {
Tboolexp boolexp; // condition
Texp exp1, exp2; // then and else branch
public Tifthenelse(Tboolexp b, Texp e1, Texp e2) {
boolexp=b;
exp1=e1;
exp2=e2;
}
public String toString() {
return "if "+boolexp+" then "+exp1+" else "+exp2+" fi";
}
public void checkcontext(SymTab st) {
boolexp.checkcontext(st);
exp1.checkcontext(st);
exp2.checkcontext(st);
}
public void prepInterp(SymTab st) {
boolexp.prepInterp(st);
exp1.prepInterp(st);
exp2.prepInterp(st);
}
public int interpret(int[] in, int[] par) {
boolean b = boolexp.interpret(in,par);
if (b)
return exp1.interpret(in,par);
else
return exp2.interpret(in,par);
}
}
tools/lexer/jflex-1.4/examples/interpreter/Tnumber.java
-47
View File
@@ -1,47 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/**
* AST node for a number
*/
class Tnumber extends Texp implements AST {
int n; // value of the number
public Tnumber(String s) {
try { n = Integer.parseInt(s); }
catch (NumberFormatException e) { n=-1; };
}
public String toString() {
return(""+n);
}
public void checkcontext(SymTab st) {
}
public void prepInterp(SymTab st) {
}
public int interpret(int[] in, int[] par) {
return(n);
}
}
tools/lexer/jflex-1.4/examples/interpreter/Tparlist.java
-54
View File
@@ -1,54 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/**
* AST node for a parameter list.
*/
class Tparlist implements AST {
Tparlist parlist; // rest of the liste (optional null)
Tident ident; // identifier
public Tparlist(Tparlist p, Tident i) {
parlist=p;
ident=i;
}
public Tparlist(Tident i) {
parlist=null;
ident=i;
}
public String toString() {
if (parlist!=null)
return parlist+","+ident;
else
return ident.toString();
}
public void setSymtab(SymTab st, boolean isInput, int index) {
boolean isNew = st.enter(ident.toString(),
new STEvar(ident.toString(), isInput, index));
if (!isNew) Main.error("Variable "+ident+" defined twice!");
if (parlist!=null) parlist.setSymtab(st, isInput, index+1);
}
}
tools/lexer/jflex-1.4/examples/interpreter/Tprogram.java
-99
View File
@@ -1,99 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/**
* AST node for the whole program (top node).
*
* Also contains two symbol tables, one for input variables,
* one for function names.
*
* All operations like context check, symbol table build up
* etc. start here.
*/
class Tprogram implements AST {
Tparlist parlist; // input variables
Tdekllist dekllist; // function declarations
Texplist explist; // result expressions
Texplist arguments; // input values
public Tprogram(Tparlist p, Tdekllist d, Texplist e, Texplist a) {
parlist=p;
dekllist=d;
explist=e;
arguments=a;
}
public String toString() {
return("Program:\n=============\ninput "+parlist+
"\nfunctions\n"+dekllist+"\noutput "+explist+
"\narguments "+arguments+"\nend");
}
SymTab inputs; // table of input variables
SymTab functions; // table of functions
public void setSymtabs() { // calculate symbol table entries
inputs = new SymTab(); // set input variables
parlist.setSymtab(inputs, true, 0);
functions = new SymTab(inputs);
dekllist.setSymtab(functions);
}
public void printSymtabs() {
System.out.print("Input variables-\n"+inputs);
System.out.print("Functions-\n"+functions);
dekllist.printSymtabs();
}
public void checkcontext() {
dekllist.checkcontext(); // CoCo (DefFun,DefVar,Arity)
// in function bodies
explist.checkcontext(functions); // CoCo (DefFun,DefVar,Arity)
// in result expressions
arguments.checkcontext(new SymTab()); // CoCo (constants)
// in arguments
if (arguments.length()!=inputs.size())
Main.error("Argument list and input variables list differ!");
}
public void prepInterp() { // set pointers and indices
dekllist.prepInterp(functions);
explist.prepInterp(functions);
}
public void interpret() {
int[] inputEnv = new int[inputs.size()]; // set input
arguments.interpret(null,null,inputEnv,0);
System.out.println("Result:\n=============");
int[] ergebnis = new int[explist.length()];
explist.interpret(inputEnv,null,ergebnis,0); // calculate result
int i;
for (i=explist.length()-1; i > 0; i--)
System.out.print(ergebnis[i]+", ");
System.out.println(ergebnis[i]);
}
}
tools/lexer/jflex-1.4/examples/interpreter/Tuminus.java
-49
View File
@@ -1,49 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/**
* AST node for unary minus expressions.
*/
class Tuminus extends Texp implements AST {
Texp exp; // the negated expression
public Tuminus(Texp e) {
exp=e;
}
public String toString() {
return "-"+exp;
}
public void checkcontext(SymTab st) {
exp.checkcontext(st);
}
public void prepInterp(SymTab st) {
exp.prepInterp(st);
}
public int interpret(int[] in, int[] par) {
return -(exp.interpret(in,par));
}
}
tools/lexer/jflex-1.4/examples/interpreter/build.xml
-38
View File
@@ -1,38 +0,0 @@
<project name="interpreter" default="run">
<taskdef classname="JFlex.anttask.JFlexTask" name="jflex" />
<taskdef classname="java_cup.AntTask" name="cup" />
<target name="build" depends="realclean,compile" />
<target name="run" depends="compile">
<java classname="Main"> <arg line="example.as"/> </java>
</target>
<target name="compile" depends="jflex,cup">
<javac srcdir="." destdir="." />
</target>
<target name="jflex">
<jflex file="scanner.flex" />
</target>
<target name="cup">
<cup file="parser.cup" interface="yes" dir="."/>
</target>
<target name="genclean">
<delete file="Yylex.java"/>
<delete file="sym.java" />
<delete file="parser.java" />
</target>
<target name="clean">
<delete>
<fileset dir="." includes="**/*~"/>
<fileset dir="." includes="**/*.class"/>
</delete>
</target>
<target name="realclean" depends="clean,genclean" />
</project>
tools/lexer/jflex-1.4/examples/interpreter/example.as
-16
View File
@@ -1,16 +0,0 @@
// example program for interpreter testing
// contains division and modulo functins
input a,b
functions div(x,y) = if x < y
then 0
else div(x-y,y)+1
fi,
mod(x,y) = if x < y
then x
else mod(x-y,y)
fi
output div(a,b), mod(a,b)
arguments 324, 17
end
tools/lexer/jflex-1.4/examples/interpreter/parser.cup
-112
View File
@@ -1,112 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
// definition of tokens, if applicable with token type
terminal INPUT, FUNCTIONS, OUTPUT, END, ARGUMENTS;
terminal IF, THEN, ELSE, FI, ERROR;
terminal COMMA, LPAR, RPAR;
terminal EQ, LE, LEQ, MINUS, PLUS, TIMES, DIV, UMINUS;
terminal String ID, NUMBER;
non terminal Tprogram program;
non terminal Tparlist parlist;
non terminal Texplist explist;
non terminal Tdekllist dekllist;
non terminal Tdekl dekl;
non terminal Texp exp;
non terminal Tboolexp boolexp;
non terminal Tident ident;
non terminal Tnumber number;
// precedences, left associativity
precedence left EQ, LE, LEQ;
precedence left MINUS, PLUS;
precedence left TIMES, DIV;
precedence left UMINUS;
// here the rules start
program ::= INPUT parlist:p FUNCTIONS dekllist:d OUTPUT explist:o
ARGUMENTS explist:a END
{: RESULT = new Tprogram(p,d,o,a); :}
;
parlist ::= ident:i
{: RESULT = new Tparlist(i); :}
| parlist:p COMMA ident:i
{: RESULT = new Tparlist(p,i); :}
;
explist ::= exp:e
{: RESULT = new Texplist(e); :}
| explist:l COMMA exp:e
{: RESULT = new Texplist(l,e); :}
;
dekllist ::= dekl:d
{: RESULT = new Tdekllist(d);:}
| dekllist:l COMMA dekl:d
{: RESULT = new Tdekllist(l,d); :}
;
dekl ::= ident:i LPAR parlist:p RPAR EQ exp:e
{: RESULT = new Tdekl(i,p,e); :}
;
exp ::= number:n
{: RESULT = n; :}
| ident:i
{: RESULT = i; :}
| ident:i LPAR explist:e RPAR
{: RESULT = new Tfun(i,e); :}
| LPAR exp:e RPAR
{: RESULT = e; :}
| MINUS exp:e
{: RESULT = new Tuminus(e); :} %prec UMINUS
| exp:l PLUS exp:r
{: RESULT = new Texpinfix(l,'+',r); :}
| exp:l TIMES exp:r
{: RESULT = new Texpinfix(l,'*',r); :}
| exp:l DIV exp:r
{: RESULT = new Texpinfix(l,'/',r); :}
| exp:l MINUS exp:r
{: RESULT = new Texpinfix(l,'-',r); :}
| IF boolexp:b THEN exp:t ELSE exp:e FI
{: RESULT = new Tifthenelse(b,t,e); :}
;
boolexp ::= exp:l EQ exp:r
{: RESULT = new Tboolexp(l,'=',r); :}
| exp:l LE exp:r
{: RESULT = new Tboolexp(l,'<',r); :}
| exp:l LEQ exp:r
{: RESULT = new Tboolexp(l,'!',r); :}
;
ident ::= ID:n
{: RESULT = new Tident(n); :}
;
number ::= NUMBER:z
{: RESULT = new Tnumber(z); :}
;
tools/lexer/jflex-1.4/examples/interpreter/scanner.flex
-71
View File
@@ -1,71 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2001 Gerwin Klein <lsf@jflex.de> *
* Copyright (C) 2001 Bernhard Rumpe <rumpe@in.tum.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
import java_cup.runtime.Symbol;
%%
%cup
%implements sym
%{
SymTab symtab; // externe symbol table
public void setSymtab(SymTab symtab) {
this.symtab = symtab;
}
private Symbol sym(int sym) {
return new Symbol(sym);
}
private Symbol sym(int sym, Object val) {
return new Symbol(sym, val);
}
%}
%%
"arguments" { return sym(ARGUMENTS); }
"input" { return sym(INPUT); }
"functions" { return sym(FUNCTIONS); }
"output" { return sym(OUTPUT); }
"end" { return sym(END); }
"if" { return sym(IF); }
"then" { return sym(THEN); }
"else" { return sym(ELSE); }
"fi" { return sym(FI); }
[a-z]+ { symtab.enter(yytext(),new SymtabEntry(yytext()));
return sym(ID,yytext()); }
[0-9]+ { return sym(NUMBER,yytext()); }
"," { return sym(COMMA); }
"(" { return sym(LPAR); }
")" { return sym(RPAR); }
"=" { return sym(EQ); }
"-" { return sym(MINUS); }
"+" { return sym(PLUS); }
"*" { return sym(TIMES); }
"/" { return sym(DIV); }
"<" { return sym(LE); }
"<=" { return sym(LEQ); }
[\ \t\b\f\r\n]+ { /* eat whitespace */ }
"//"[^\n]* { /* one-line comment */ }
. { throw new Error("Unexpected character ["+yytext()+"]"); }
tools/lexer/jflex-1.4/examples/java/JavaParser.java
-47
View File
@@ -1,47 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 1998-2004 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
import java.io.FileReader;
/**
* Simple test driver for the java parser. Just runs it on some
* input files, gives no useful output.
*/
public class JavaParser {
public static void main(String argv[]) {
for (int i = 0; i < argv.length; i++) {
try {
System.out.println("Parsing ["+argv[i]+"]");
Scanner s = new Scanner(new UnicodeEscapes(new FileReader(argv[i])));
parser p = new parser(s);
p.parse();
System.out.println("No errors.");
}
catch (Exception e) {
e.printStackTrace(System.out);
System.exit(1);
}
}
}
}
tools/lexer/jflex-1.4/examples/java/JavaSymbol.java
-31
View File
@@ -1,31 +0,0 @@
public class JavaSymbol extends java_cup.runtime.Symbol {
private int line;
private int column;
public JavaSymbol(int type, int line, int column) {
this(type, line, column, -1, -1, null);
}
public JavaSymbol(int type, int line, int column, Object value) {
this(type, line, column, -1, -1, value);
}
public JavaSymbol(int type, int line, int column, int left, int right, Object value) {
super(type, left, right, value);
this.line = line;
this.column = column;
}
public int getLine() {
return line;
}
public int getColumn() {
return column;
}
public String toString() {
return "line "+line+", column "+column+", sym: "+sym+(value == null ? "" : (", value: '"+value+"'"));
}
}
tools/lexer/jflex-1.4/examples/java/Makefile
-48
View File
@@ -1,48 +0,0 @@
# You need CUP v0.10j (or newer) for this makefile to work (for java12.cup)
#
# CUP classes should be included in CLASSPATH
CUP = java java_cup.Main -interface <
JFLEX = jflex
JAVA = java
JAVAC = javac
#JAVAC = jikes
JAVACFLAGS =
# --------------------------------------------------
all: test
test: lexer-output.txt
@(diff lexer-output.txt lexer-output.good && echo "Test OK!") || echo "Test failed!"
lexer-output.txt: compile
$(JAVA) TestLexer TestLexer.java > lexer-output.txt
compile: scanner parser unicode
$(JAVAC) $(JAVACFLAGS) JavaParser.java TestLexer.java
parser: parser.java
parser.java: java12.cup
$(CUP) java12.cup
scanner: Scanner.java
Scanner.java: java.flex
$(JFLEX) java.flex
unicode: UnicodeEscapes.java
UnicodeEscapes.java: unicode.flex
$(JFLEX) unicode.flex
clean:
rm -f *.class
rm -f *~
rm -f Scanner.java
rm -f parser.java
rm -f sym.java
rm -f UnicodeEscapes.java
rm -f lexer-output.txt
tools/lexer/jflex-1.4/examples/java/README
-41
View File
@@ -1,41 +0,0 @@
This directory contains a scanner and parser for the Java programming language.
You need the parser generator CUP v0.10j (or newer) for the parser to work.
Use the Makefile to generate the lexer and parser, or type:
jflex unicode.flex
jflex java.flex
java java_cup.Main -interface < java12.cup
javac JavaParser.java TestLexer.java
The parser can be tested with:
java JavaParser <inputfiles>
The scanner (without parser attached) can be test with:
java TestLexer <inputfiles>
files:
unicode.flex
JFlex specification for the Unicode preprocessing phase
(see section 3.3 of the Java Language Specification).
Demonstrates how to implement a FilterReader with JFlex.
java.flex
JFlex specification for the "real" Java 1.2 lexer.
java12.cup
CUP specification for the Java 1.2 parser
Copyright (C) 1998 by C. Scott Ananian <cananian@alumni.princeton.edu>
(with small modifications to interface with the Lexer)
JavaParser.java
a simple main class to run the parser (no other useful output though)
TestLexer.java
a simple test driver for the scanner, produces debug output
lexer-output.good
lexer output for input file TestLexer.java for testing
tools/lexer/jflex-1.4/examples/java/TestLexer.java
-62
View File
@@ -1,62 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 2004 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
import java_cup.runtime.Symbol;
import java.io.FileReader;
/**
* Simple test driver for the java lexer. Just runs it on some
* input files and produces debug output. Needs symbol class from
* parser.
*/
public class TestLexer {
/** some numerals to for lexer testing */
int intDec = 37;
long longDec = 37l;
int intHex = 0x0001;
long longHex = 0xFFFFl;
int intOct = 0377;
long longOc = 007l;
public static void main(String argv[]) {
for (int i = 0; i < argv.length; i++) {
try {
System.out.println("Lexing ["+argv[i]+"]");
Scanner scanner = new Scanner(new UnicodeEscapes(new FileReader(argv[i])));
Symbol s;
do {
s = scanner.debug_next_token();
System.out.println("token: "+s);
} while (s.sym != sym.EOF);
System.out.println("No errors.");
}
catch (Exception e) {
e.printStackTrace(System.out);
System.exit(1);
}
}
}
}
tools/lexer/jflex-1.4/examples/java/java.flex
-312
View File
@@ -1,312 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 1998-2004 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* Java 1.2 language lexer specification */
/* Use together with unicode.flex for Unicode preprocesssing */
/* and java12.cup for a Java 1.2 parser */
/* Note that this lexer specification is not tuned for speed.
It is in fact quite slow on integer and floating point literals,
because the input is read twice and the methods used to parse
the numbers are not very fast.
For a production quality application (e.g. a Java compiler)
this could be optimized */
import java_cup.runtime.*;
%%
%public
%class Scanner
%implements sym
%unicode
%line
%column
%cup
%cupdebug
%{
StringBuffer string = new StringBuffer();
private Symbol symbol(int type) {
return new JavaSymbol(type, yyline+1, yycolumn+1);
}
private Symbol symbol(int type, Object value) {
return new JavaSymbol(type, yyline+1, yycolumn+1, value);
}
/**
* assumes correct representation of a long value for
* specified radix in scanner buffer from <code>start</code>
* to <code>end</code>
*/
private long parseLong(int start, int end, int radix) {
long result = 0;
long digit;
for (int i = start; i < end; i++) {
digit = Character.digit(yycharat(i),radix);
result*= radix;
result+= digit;
}
return result;
}
%}
/* main character classes */
LineTerminator = \r|\n|\r\n
InputCharacter = [^\r\n]
WhiteSpace = {LineTerminator} | [ \t\f]
/* comments */
Comment = {TraditionalComment} | {EndOfLineComment} |
{DocumentationComment}
TraditionalComment = "/*" [^*] ~"*/" | "/*" "*"+ "/"
EndOfLineComment = "//" {InputCharacter}* {LineTerminator}?
DocumentationComment = "/*" "*"+ [^/*] ~"*/"
/* identifiers */
Identifier = [:jletter:][:jletterdigit:]*
/* integer literals */
DecIntegerLiteral = 0 | [1-9][0-9]*
DecLongLiteral = {DecIntegerLiteral} [lL]
HexIntegerLiteral = 0 [xX] 0* {HexDigit} {1,8}
HexLongLiteral = 0 [xX] 0* {HexDigit} {1,16} [lL]
HexDigit = [0-9a-fA-F]
OctIntegerLiteral = 0+ [1-3]? {OctDigit} {1,15}
OctLongLiteral = 0+ 1? {OctDigit} {1,21} [lL]
OctDigit = [0-7]
/* floating point literals */
FloatLiteral = ({FLit1}|{FLit2}|{FLit3}) {Exponent}? [fF]
DoubleLiteral = ({FLit1}|{FLit2}|{FLit3}) {Exponent}?
FLit1 = [0-9]+ \. [0-9]*
FLit2 = \. [0-9]+
FLit3 = [0-9]+
Exponent = [eE] [+-]? [0-9]+
/* string and character literals */
StringCharacter = [^\r\n\"\\]
SingleCharacter = [^\r\n\'\\]
%state STRING, CHARLITERAL
%%
<YYINITIAL> {
/* keywords */
"abstract" { return symbol(ABSTRACT); }
"boolean" { return symbol(BOOLEAN); }
"break" { return symbol(BREAK); }
"byte" { return symbol(BYTE); }
"case" { return symbol(CASE); }
"catch" { return symbol(CATCH); }
"char" { return symbol(CHAR); }
"class" { return symbol(CLASS); }
"const" { return symbol(CONST); }
"continue" { return symbol(CONTINUE); }
"do" { return symbol(DO); }
"double" { return symbol(DOUBLE); }
"else" { return symbol(ELSE); }
"extends" { return symbol(EXTENDS); }
"final" { return symbol(FINAL); }
"finally" { return symbol(FINALLY); }
"float" { return symbol(FLOAT); }
"for" { return symbol(FOR); }
"default" { return symbol(DEFAULT); }
"implements" { return symbol(IMPLEMENTS); }
"import" { return symbol(IMPORT); }
"instanceof" { return symbol(INSTANCEOF); }
"int" { return symbol(INT); }
"interface" { return symbol(INTERFACE); }
"long" { return symbol(LONG); }
"native" { return symbol(NATIVE); }
"new" { return symbol(NEW); }
"goto" { return symbol(GOTO); }
"if" { return symbol(IF); }
"public" { return symbol(PUBLIC); }
"short" { return symbol(SHORT); }
"super" { return symbol(SUPER); }
"switch" { return symbol(SWITCH); }
"synchronized" { return symbol(SYNCHRONIZED); }
"package" { return symbol(PACKAGE); }
"private" { return symbol(PRIVATE); }
"protected" { return symbol(PROTECTED); }
"transient" { return symbol(TRANSIENT); }
"return" { return symbol(RETURN); }
"void" { return symbol(VOID); }
"static" { return symbol(STATIC); }
"while" { return symbol(WHILE); }
"this" { return symbol(THIS); }
"throw" { return symbol(THROW); }
"throws" { return symbol(THROWS); }
"try" { return symbol(TRY); }
"volatile" { return symbol(VOLATILE); }
"strictfp" { return symbol(STRICTFP); }
/* boolean literals */
"true" { return symbol(BOOLEAN_LITERAL, new Boolean(true)); }
"false" { return symbol(BOOLEAN_LITERAL, new Boolean(false)); }
/* null literal */
"null" { return symbol(NULL_LITERAL); }
/* separators */
"(" { return symbol(LPAREN); }
")" { return symbol(RPAREN); }
"{" { return symbol(LBRACE); }
"}" { return symbol(RBRACE); }
"[" { return symbol(LBRACK); }
"]" { return symbol(RBRACK); }
";" { return symbol(SEMICOLON); }
"," { return symbol(COMMA); }
"." { return symbol(DOT); }
/* operators */
"=" { return symbol(EQ); }
">" { return symbol(GT); }
"<" { return symbol(LT); }
"!" { return symbol(NOT); }
"~" { return symbol(COMP); }
"?" { return symbol(QUESTION); }
":" { return symbol(COLON); }
"==" { return symbol(EQEQ); }
"<=" { return symbol(LTEQ); }
">=" { return symbol(GTEQ); }
"!=" { return symbol(NOTEQ); }
"&&" { return symbol(ANDAND); }
"||" { return symbol(OROR); }
"++" { return symbol(PLUSPLUS); }
"--" { return symbol(MINUSMINUS); }
"+" { return symbol(PLUS); }
"-" { return symbol(MINUS); }
"*" { return symbol(MULT); }
"/" { return symbol(DIV); }
"&" { return symbol(AND); }
"|" { return symbol(OR); }
"^" { return symbol(XOR); }
"%" { return symbol(MOD); }
"<<" { return symbol(LSHIFT); }
">>" { return symbol(RSHIFT); }
">>>" { return symbol(URSHIFT); }
"+=" { return symbol(PLUSEQ); }
"-=" { return symbol(MINUSEQ); }
"*=" { return symbol(MULTEQ); }
"/=" { return symbol(DIVEQ); }
"&=" { return symbol(ANDEQ); }
"|=" { return symbol(OREQ); }
"^=" { return symbol(XOREQ); }
"%=" { return symbol(MODEQ); }
"<<=" { return symbol(LSHIFTEQ); }
">>=" { return symbol(RSHIFTEQ); }
">>>=" { return symbol(URSHIFTEQ); }
/* string literal */
\" { yybegin(STRING); string.setLength(0); }
/* character literal */
\' { yybegin(CHARLITERAL); }
/* numeric literals */
{DecIntegerLiteral} { return symbol(INTEGER_LITERAL, new Integer(yytext())); }
{DecLongLiteral} { return symbol(INTEGER_LITERAL, new Long(yytext().substring(0,yylength()-1))); }
{HexIntegerLiteral} { return symbol(INTEGER_LITERAL, new Integer((int) parseLong(2, yylength(), 16))); }
{HexLongLiteral} { return symbol(INTEGER_LITERAL, new Long(parseLong(2, yylength()-1, 16))); }
{OctIntegerLiteral} { return symbol(INTEGER_LITERAL, new Integer((int) parseLong(0, yylength(), 8))); }
{OctLongLiteral} { return symbol(INTEGER_LITERAL, new Long(parseLong(0, yylength()-1, 8))); }
{FloatLiteral} { return symbol(FLOATING_POINT_LITERAL, new Float(yytext().substring(0,yylength()-1))); }
{DoubleLiteral} { return symbol(FLOATING_POINT_LITERAL, new Double(yytext())); }
{DoubleLiteral}[dD] { return symbol(FLOATING_POINT_LITERAL, new Double(yytext().substring(0,yylength()-1))); }
/* comments */
{Comment} { /* ignore */ }
/* whitespace */
{WhiteSpace} { /* ignore */ }
/* identifiers */
{Identifier} { return symbol(IDENTIFIER, yytext()); }
}
<STRING> {
\" { yybegin(YYINITIAL); return symbol(STRING_LITERAL, string.toString()); }
{StringCharacter}+ { string.append( yytext() ); }
/* escape sequences */
"\\b" { string.append( '\b' ); }
"\\t" { string.append( '\t' ); }
"\\n" { string.append( '\n' ); }
"\\f" { string.append( '\f' ); }
"\\r" { string.append( '\r' ); }
"\\\"" { string.append( '\"' ); }
"\\'" { string.append( '\'' ); }
"\\\\" { string.append( '\\' ); }
\\[0-3]?{OctDigit}?{OctDigit} { char val = (char) Integer.parseInt(yytext().substring(1),8);
string.append( val ); }
/* error cases */
\\. { throw new RuntimeException("Illegal escape sequence \""+yytext()+"\""); }
{LineTerminator} { throw new RuntimeException("Unterminated string at end of line"); }
}
<CHARLITERAL> {
{SingleCharacter}\' { yybegin(YYINITIAL); return symbol(CHARACTER_LITERAL, new Character(yytext().charAt(0))); }
/* escape sequences */
"\\b"\' { yybegin(YYINITIAL); return symbol(CHARACTER_LITERAL, new Character('\b'));}
"\\t"\' { yybegin(YYINITIAL); return symbol(CHARACTER_LITERAL, new Character('\t'));}
"\\n"\' { yybegin(YYINITIAL); return symbol(CHARACTER_LITERAL, new Character('\n'));}
"\\f"\' { yybegin(YYINITIAL); return symbol(CHARACTER_LITERAL, new Character('\f'));}
"\\r"\' { yybegin(YYINITIAL); return symbol(CHARACTER_LITERAL, new Character('\r'));}
"\\\""\' { yybegin(YYINITIAL); return symbol(CHARACTER_LITERAL, new Character('\"'));}
"\\'"\' { yybegin(YYINITIAL); return symbol(CHARACTER_LITERAL, new Character('\''));}
"\\\\"\' { yybegin(YYINITIAL); return symbol(CHARACTER_LITERAL, new Character('\\')); }
\\[0-3]?{OctDigit}?{OctDigit}\' { yybegin(YYINITIAL);
int val = Integer.parseInt(yytext().substring(1,yylength()-1),8);
return symbol(CHARACTER_LITERAL, new Character((char)val)); }
/* error cases */
\\. { throw new RuntimeException("Illegal escape sequence \""+yytext()+"\""); }
{LineTerminator} { throw new RuntimeException("Unterminated character literal at end of line"); }
}
/* error fallback */
.|\n { throw new RuntimeException("Illegal character \""+yytext()+
"\" at line "+yyline+", column "+yycolumn); }
<<EOF>> { return symbol(EOF); }
tools/lexer/jflex-1.4/examples/java/java12.cup
-885
View File
@@ -1,885 +0,0 @@
import java_cup.runtime.*;
/* August 1999 - modified by Gerwin Klein <lsf@jflex.de>
to interface with JFlex scanners,
allows empty semicolon in class decls.
changed productions:
class_member_declaration ::=
field_declaration
| method_declaration
[..]
| interface_declaration
| SEMICOLON
;
interface_member_declaration ::=
constant_declaration
| abstract_method_declaration
| class_declaration
| interface_declaration
| SEMICOLON
;
*/
/* Java 1.2 parser for CUP.
* Copyright (C) 1998 C. Scott Ananian <cananian@alumni.princeton.edu>
* This program is released under the terms of the GPL; see the file
* COPYING for more details. There is NO WARRANTY on this code.
*/
/*
JDK 1.2 Features added:
strictfp modifier.
explicit_constructor_invocation ::= ...
| primary DOT THIS LPAREN argument_list_opt RPAREN SEMICOLON ;
field_access ::= ...
| name DOT SUPER DOT IDENTIFIER ;
method_invocation ::= ...
| name DOT SUPER DOT IDENTIFIER LPAREN argument_list_opt RPAREN ;
*/
parser code {:
public void report_error(String message, Object info) {
StringBuffer m = new StringBuffer("Error ");
if (info instanceof java_cup.runtime.Symbol)
m.append( "("+info.toString()+")" );
m.append(" : "+message);
System.out.println(m);
}
public void report_fatal_error(String message, Object info) {
report_error(message, info);
throw new RuntimeException("Fatal Syntax Error");
}
:};
terminal BOOLEAN; // primitive_type
terminal BYTE, SHORT, INT, LONG, CHAR; // integral_type
terminal FLOAT, DOUBLE; // floating_point_type
terminal LBRACK, RBRACK; // array_type
terminal DOT; // qualified_name
terminal SEMICOLON, MULT, COMMA, LBRACE, RBRACE, EQ, LPAREN, RPAREN, COLON;
terminal PACKAGE; // package_declaration
terminal IMPORT; // import_declaration
terminal PUBLIC, PROTECTED, PRIVATE; // modifier
terminal STATIC; // modifier
terminal ABSTRACT, FINAL, NATIVE, SYNCHRONIZED, TRANSIENT, VOLATILE;
terminal CLASS; // class_declaration
terminal EXTENDS; // super
terminal IMPLEMENTS; // interfaces
terminal VOID; // method_header
terminal THROWS; // throws
terminal THIS, SUPER; // explicit_constructor_invocation
terminal INTERFACE; // interface_declaration
terminal IF, ELSE; // if_then_statement, if_then_else_statement
terminal SWITCH; // switch_statement
terminal CASE, DEFAULT; // switch_label
terminal DO, WHILE; // while_statement, do_statement
terminal FOR; // for_statement
terminal BREAK; // break_statement
terminal CONTINUE; // continue_statement
terminal RETURN; // return_statement
terminal THROW; // throw_statement
terminal TRY; // try_statement
terminal CATCH; // catch_clause
terminal FINALLY; // finally
terminal NEW; // class_instance_creation_expression
terminal PLUSPLUS; // postincrement_expression
terminal MINUSMINUS; // postdecrement_expression
terminal PLUS, MINUS, COMP, NOT, DIV, MOD;
terminal LSHIFT, RSHIFT, URSHIFT; // shift_expression
terminal LT, GT, LTEQ, GTEQ, INSTANCEOF; // relational_expression
terminal EQEQ, NOTEQ; // equality_expression
terminal AND; // and_expression
terminal XOR; // exclusive_or_expression
terminal OR; // inclusive_or_expression
terminal ANDAND; // conditional_and_expression
terminal OROR; // conditional_or_expression
terminal QUESTION; // conditional_expression
terminal MULTEQ, DIVEQ, MODEQ, PLUSEQ, MINUSEQ; // assignment_operator
terminal LSHIFTEQ, RSHIFTEQ, URSHIFTEQ; // assignment_operator
terminal ANDEQ, XOREQ, OREQ; // assignment_operator
terminal java.lang.Number INTEGER_LITERAL;
terminal java.lang.Number FLOATING_POINT_LITERAL;
terminal java.lang.Boolean BOOLEAN_LITERAL;
terminal java.lang.Character CHARACTER_LITERAL;
terminal java.lang.String STRING_LITERAL;
terminal java.lang.String IDENTIFIER; // name
terminal NULL_LITERAL;
// strictfp keyword, new in Java 1.2
terminal STRICTFP;
// Reserved but unused:
terminal CONST, GOTO;
// 19.2) The Syntactic Grammar
non terminal goal;
// 19.3) Lexical Structure
non terminal literal;
// 19.4) Types, Values, and Variables
non terminal type, primitive_type, numeric_type;
non terminal integral_type, floating_point_type;
non terminal reference_type;
non terminal class_or_interface_type;
non terminal class_type, interface_type;
non terminal array_type;
// 19.5) Names
non terminal name, simple_name, qualified_name;
// 19.6) Packages
non terminal compilation_unit;
non terminal package_declaration_opt, package_declaration;
non terminal import_declarations_opt, import_declarations;
non terminal type_declarations_opt, type_declarations;
non terminal import_declaration;
non terminal single_type_import_declaration;
non terminal type_import_on_demand_declaration;
non terminal type_declaration;
// 19.7) Productions used only in the LALR(1) grammar
non terminal modifiers_opt, modifiers, modifier;
// 19.8.1) Class Declaration
non terminal class_declaration, super, super_opt;
non terminal interfaces, interfaces_opt, interface_type_list;
non terminal class_body;
non terminal class_body_declarations, class_body_declarations_opt;
non terminal class_body_declaration, class_member_declaration;
// 19.8.2) Field Declarations
non terminal field_declaration, variable_declarators, variable_declarator;
non terminal variable_declarator_id, variable_initializer;
// 19.8.3) Method Declarations
non terminal method_declaration, method_header, method_declarator;
non terminal formal_parameter_list_opt, formal_parameter_list;
non terminal formal_parameter;
non terminal throws_opt, throws;
non terminal class_type_list, method_body;
// 19.8.4) Static Initializers
non terminal static_initializer;
// 19.8.5) Constructor Declarations
non terminal constructor_declaration, constructor_declarator;
non terminal constructor_body;
non terminal explicit_constructor_invocation;
// 19.9.1) Interface Declarations
non terminal interface_declaration;
non terminal extends_interfaces_opt, extends_interfaces;
non terminal interface_body;
non terminal interface_member_declarations_opt, interface_member_declarations;
non terminal interface_member_declaration, constant_declaration;
non terminal abstract_method_declaration;
// 19.10) Arrays
non terminal array_initializer;
non terminal variable_initializers;
// 19.11) Blocks and Statements
non terminal block;
non terminal block_statements_opt, block_statements, block_statement;
non terminal local_variable_declaration_statement, local_variable_declaration;
non terminal statement, statement_no_short_if;
non terminal statement_without_trailing_substatement;
non terminal empty_statement;
non terminal labeled_statement, labeled_statement_no_short_if;
non terminal expression_statement, statement_expression;
non terminal if_then_statement;
non terminal if_then_else_statement, if_then_else_statement_no_short_if;
non terminal switch_statement, switch_block;
non terminal switch_block_statement_groups;
non terminal switch_block_statement_group;
non terminal switch_labels, switch_label;
non terminal while_statement, while_statement_no_short_if;
non terminal do_statement;
non terminal for_statement, for_statement_no_short_if;
non terminal for_init_opt, for_init;
non terminal for_update_opt, for_update;
non terminal statement_expression_list;
non terminal identifier_opt;
non terminal break_statement, continue_statement;
non terminal return_statement, throw_statement;
non terminal synchronized_statement, try_statement;
non terminal catches_opt, catches, catch_clause;
non terminal finally;
// 19.12) Expressions
non terminal primary, primary_no_new_array;
non terminal class_instance_creation_expression;
non terminal argument_list_opt, argument_list;
non terminal array_creation_expression;
non terminal dim_exprs, dim_expr, dims_opt, dims;
non terminal field_access, method_invocation, array_access;
non terminal postfix_expression;
non terminal postincrement_expression, postdecrement_expression;
non terminal unary_expression, unary_expression_not_plus_minus;
non terminal preincrement_expression, predecrement_expression;
non terminal cast_expression;
non terminal multiplicative_expression, additive_expression;
non terminal shift_expression, relational_expression, equality_expression;
non terminal and_expression, exclusive_or_expression, inclusive_or_expression;
non terminal conditional_and_expression, conditional_or_expression;
non terminal conditional_expression, assignment_expression;
non terminal assignment;
non terminal left_hand_side;
non terminal assignment_operator;
non terminal expression_opt, expression;
non terminal constant_expression;
start with goal;
// 19.2) The Syntactic Grammar
goal ::= compilation_unit
;
// 19.3) Lexical Structure.
literal ::= INTEGER_LITERAL
| FLOATING_POINT_LITERAL
| BOOLEAN_LITERAL
| CHARACTER_LITERAL
| STRING_LITERAL
| NULL_LITERAL
;
// 19.4) Types, Values, and Variables
type ::= primitive_type
| reference_type
;
primitive_type ::=
numeric_type
| BOOLEAN
;
numeric_type::= integral_type
| floating_point_type
;
integral_type ::=
BYTE
| SHORT
| INT
| LONG
| CHAR
;
floating_point_type ::=
FLOAT
| DOUBLE
;
reference_type ::=
class_or_interface_type
| array_type
;
class_or_interface_type ::= name;
class_type ::= class_or_interface_type;
interface_type ::= class_or_interface_type;
array_type ::= primitive_type dims
| name dims
;
// 19.5) Names
name ::= simple_name
| qualified_name
;
simple_name ::= IDENTIFIER
;
qualified_name ::=
name DOT IDENTIFIER
;
// 19.6) Packages
compilation_unit ::=
package_declaration_opt
import_declarations_opt
type_declarations_opt
;
package_declaration_opt ::= package_declaration | ;
import_declarations_opt ::= import_declarations | ;
type_declarations_opt ::= type_declarations | ;
import_declarations ::=
import_declaration
| import_declarations import_declaration
;
type_declarations ::=
type_declaration
| type_declarations type_declaration
;
package_declaration ::=
PACKAGE name SEMICOLON
;
import_declaration ::=
single_type_import_declaration
| type_import_on_demand_declaration
;
single_type_import_declaration ::=
IMPORT name SEMICOLON
;
type_import_on_demand_declaration ::=
IMPORT name DOT MULT SEMICOLON
;
type_declaration ::=
class_declaration
| interface_declaration
| SEMICOLON
;
// 19.7) Productions used only in the LALR(1) grammar
modifiers_opt::=
| modifiers
;
modifiers ::= modifier
| modifiers modifier
;
modifier ::= PUBLIC | PROTECTED | PRIVATE
| STATIC
| ABSTRACT | FINAL | NATIVE | SYNCHRONIZED | TRANSIENT | VOLATILE
| STRICTFP // note that semantic analysis must check that the
// context of the modifier allows strictfp.
;
// 19.8) Classes
// 19.8.1) Class Declaration:
class_declaration ::=
modifiers_opt CLASS IDENTIFIER super_opt interfaces_opt class_body
;
super ::= EXTENDS class_type
;
super_opt ::=
| super
;
interfaces ::= IMPLEMENTS interface_type_list
;
interfaces_opt::=
| interfaces
;
interface_type_list ::=
interface_type
| interface_type_list COMMA interface_type
;
class_body ::= LBRACE class_body_declarations_opt RBRACE
;
class_body_declarations_opt ::=
| class_body_declarations ;
class_body_declarations ::=
class_body_declaration
| class_body_declarations class_body_declaration
;
class_body_declaration ::=
class_member_declaration
| static_initializer
| constructor_declaration
| block
;
class_member_declaration ::=
field_declaration
| method_declaration
/* repeat the prod for 'class_declaration' here: */
| modifiers_opt CLASS IDENTIFIER super_opt interfaces_opt class_body
| interface_declaration
| SEMICOLON
;
// 19.8.2) Field Declarations
field_declaration ::=
modifiers_opt type variable_declarators SEMICOLON
;
variable_declarators ::=
variable_declarator
| variable_declarators COMMA variable_declarator
;
variable_declarator ::=
variable_declarator_id
| variable_declarator_id EQ variable_initializer
;
variable_declarator_id ::=
IDENTIFIER
| variable_declarator_id LBRACK RBRACK
;
variable_initializer ::=
expression
| array_initializer
;
// 19.8.3) Method Declarations
method_declaration ::=
method_header method_body
;
method_header ::=
modifiers_opt type method_declarator throws_opt
| modifiers_opt VOID method_declarator throws_opt
;
method_declarator ::=
IDENTIFIER LPAREN formal_parameter_list_opt RPAREN
| method_declarator LBRACK RBRACK // deprecated
// be careful; the above production also allows 'void foo() []'
;
formal_parameter_list_opt ::=
| formal_parameter_list
;
formal_parameter_list ::=
formal_parameter
| formal_parameter_list COMMA formal_parameter
;
formal_parameter ::=
type variable_declarator_id
| FINAL type variable_declarator_id
;
throws_opt ::=
| throws
;
throws ::= THROWS class_type_list
;
class_type_list ::=
class_type
| class_type_list COMMA class_type
;
method_body ::= block
| SEMICOLON
;
// 19.8.4) Static Initializers
static_initializer ::=
STATIC block
;
// 19.8.5) Constructor Declarations
constructor_declaration ::=
modifiers_opt constructor_declarator throws_opt
constructor_body
;
constructor_declarator ::=
simple_name LPAREN formal_parameter_list_opt RPAREN
;
constructor_body ::=
LBRACE explicit_constructor_invocation
block_statements RBRACE
| LBRACE explicit_constructor_invocation RBRACE
| LBRACE block_statements RBRACE
| LBRACE RBRACE
;
explicit_constructor_invocation ::=
THIS LPAREN argument_list_opt RPAREN SEMICOLON
| SUPER LPAREN argument_list_opt RPAREN SEMICOLON
| primary DOT THIS LPAREN argument_list_opt RPAREN SEMICOLON
| primary DOT SUPER LPAREN argument_list_opt RPAREN SEMICOLON
;
// 19.9) Interfaces
// 19.9.1) Interface Declarations
interface_declaration ::=
modifiers_opt INTERFACE IDENTIFIER extends_interfaces_opt
interface_body
;
extends_interfaces_opt ::=
| extends_interfaces
;
extends_interfaces ::=
EXTENDS interface_type
| extends_interfaces COMMA interface_type
;
interface_body ::=
LBRACE interface_member_declarations_opt RBRACE
;
interface_member_declarations_opt ::=
| interface_member_declarations
;
interface_member_declarations ::=
interface_member_declaration
| interface_member_declarations interface_member_declaration
;
interface_member_declaration ::=
constant_declaration
| abstract_method_declaration
| class_declaration
| interface_declaration
| SEMICOLON
;
constant_declaration ::=
field_declaration
;
abstract_method_declaration ::=
method_header SEMICOLON
;
// 19.10) Arrays
array_initializer ::=
LBRACE variable_initializers COMMA RBRACE
| LBRACE variable_initializers RBRACE
| LBRACE COMMA RBRACE
| LBRACE RBRACE
;
variable_initializers ::=
variable_initializer
| variable_initializers COMMA variable_initializer
;
// 19.11) Blocks and Statements
block ::= LBRACE block_statements_opt RBRACE
;
block_statements_opt ::=
| block_statements
;
block_statements ::=
block_statement
| block_statements block_statement
;
block_statement ::=
local_variable_declaration_statement
| statement
| class_declaration
| interface_declaration
;
local_variable_declaration_statement ::=
local_variable_declaration SEMICOLON
;
local_variable_declaration ::=
type variable_declarators
| FINAL type variable_declarators
;
statement ::= statement_without_trailing_substatement
| labeled_statement
| if_then_statement
| if_then_else_statement
| while_statement
| for_statement
;
statement_no_short_if ::=
statement_without_trailing_substatement
| labeled_statement_no_short_if
| if_then_else_statement_no_short_if
| while_statement_no_short_if
| for_statement_no_short_if
;
statement_without_trailing_substatement ::=
block
| empty_statement
| expression_statement
| switch_statement
| do_statement
| break_statement
| continue_statement
| return_statement
| synchronized_statement
| throw_statement
| try_statement
;
empty_statement ::=
SEMICOLON
;
labeled_statement ::=
IDENTIFIER COLON statement
;
labeled_statement_no_short_if ::=
IDENTIFIER COLON statement_no_short_if
;
expression_statement ::=
statement_expression SEMICOLON
;
statement_expression ::=
assignment
| preincrement_expression
| predecrement_expression
| postincrement_expression
| postdecrement_expression
| method_invocation
| class_instance_creation_expression
;
if_then_statement ::=
IF LPAREN expression RPAREN statement
;
if_then_else_statement ::=
IF LPAREN expression RPAREN statement_no_short_if
ELSE statement
;
if_then_else_statement_no_short_if ::=
IF LPAREN expression RPAREN statement_no_short_if
ELSE statement_no_short_if
;
switch_statement ::=
SWITCH LPAREN expression RPAREN switch_block
;
switch_block ::=
LBRACE switch_block_statement_groups switch_labels RBRACE
| LBRACE switch_block_statement_groups RBRACE
| LBRACE switch_labels RBRACE
| LBRACE RBRACE
;
switch_block_statement_groups ::=
switch_block_statement_group
| switch_block_statement_groups switch_block_statement_group
;
switch_block_statement_group ::=
switch_labels block_statements
;
switch_labels ::=
switch_label
| switch_labels switch_label
;
switch_label ::=
CASE constant_expression COLON
| DEFAULT COLON
;
while_statement ::=
WHILE LPAREN expression RPAREN statement
;
while_statement_no_short_if ::=
WHILE LPAREN expression RPAREN statement_no_short_if
;
do_statement ::=
DO statement WHILE LPAREN expression RPAREN SEMICOLON
;
for_statement ::=
FOR LPAREN for_init_opt SEMICOLON expression_opt SEMICOLON
for_update_opt RPAREN statement
;
for_statement_no_short_if ::=
FOR LPAREN for_init_opt SEMICOLON expression_opt SEMICOLON
for_update_opt RPAREN statement_no_short_if
;
for_init_opt ::=
| for_init
;
for_init ::= statement_expression_list
| local_variable_declaration
;
for_update_opt ::=
| for_update
;
for_update ::= statement_expression_list
;
statement_expression_list ::=
statement_expression
| statement_expression_list COMMA statement_expression
;
identifier_opt ::=
| IDENTIFIER
;
break_statement ::=
BREAK identifier_opt SEMICOLON
;
continue_statement ::=
CONTINUE identifier_opt SEMICOLON
;
return_statement ::=
RETURN expression_opt SEMICOLON
;
throw_statement ::=
THROW expression SEMICOLON
;
synchronized_statement ::=
SYNCHRONIZED LPAREN expression RPAREN block
;
try_statement ::=
TRY block catches
| TRY block catches_opt finally
;
catches_opt ::=
| catches
;
catches ::= catch_clause
| catches catch_clause
;
catch_clause ::=
CATCH LPAREN formal_parameter RPAREN block
;
finally ::= FINALLY block
;
// 19.12) Expressions
primary ::= primary_no_new_array
| array_creation_expression
;
primary_no_new_array ::=
literal
| THIS
| LPAREN expression RPAREN
| class_instance_creation_expression
| field_access
| method_invocation
| array_access
| primitive_type DOT CLASS
| VOID DOT CLASS
| array_type DOT CLASS
| name DOT CLASS
| name DOT THIS
;
class_instance_creation_expression ::=
NEW class_type LPAREN argument_list_opt RPAREN
| NEW class_type LPAREN argument_list_opt RPAREN class_body
| primary DOT NEW IDENTIFIER
LPAREN argument_list_opt RPAREN
| primary DOT NEW IDENTIFIER
LPAREN argument_list_opt RPAREN class_body
;
argument_list_opt ::=
| argument_list
;
argument_list ::=
expression
| argument_list COMMA expression
;
array_creation_expression ::=
NEW primitive_type dim_exprs dims_opt
| NEW class_or_interface_type dim_exprs dims_opt
| NEW primitive_type dims array_initializer
| NEW class_or_interface_type dims array_initializer
;
dim_exprs ::= dim_expr
| dim_exprs dim_expr
;
dim_expr ::= LBRACK expression RBRACK
;
dims_opt ::=
| dims
;
dims ::= LBRACK RBRACK
| dims LBRACK RBRACK
;
field_access ::=
primary DOT IDENTIFIER
| SUPER DOT IDENTIFIER
| name DOT SUPER DOT IDENTIFIER
;
method_invocation ::=
name LPAREN argument_list_opt RPAREN
| primary DOT IDENTIFIER LPAREN argument_list_opt RPAREN
| SUPER DOT IDENTIFIER LPAREN argument_list_opt RPAREN
| name DOT SUPER DOT IDENTIFIER LPAREN argument_list_opt RPAREN
;
array_access ::=
name LBRACK expression RBRACK
| primary_no_new_array LBRACK expression RBRACK
;
postfix_expression ::=
primary
| name
| postincrement_expression
| postdecrement_expression
;
postincrement_expression ::=
postfix_expression PLUSPLUS
;
postdecrement_expression ::=
postfix_expression MINUSMINUS
;
unary_expression ::=
preincrement_expression
| predecrement_expression
| PLUS unary_expression
| MINUS unary_expression
| unary_expression_not_plus_minus
;
preincrement_expression ::=
PLUSPLUS unary_expression
;
predecrement_expression ::=
MINUSMINUS unary_expression
;
unary_expression_not_plus_minus ::=
postfix_expression
| COMP unary_expression
| NOT unary_expression
| cast_expression
;
cast_expression ::=
LPAREN primitive_type dims_opt RPAREN unary_expression
| LPAREN expression RPAREN unary_expression_not_plus_minus
| LPAREN name dims RPAREN unary_expression_not_plus_minus
;
multiplicative_expression ::=
unary_expression
| multiplicative_expression MULT unary_expression
| multiplicative_expression DIV unary_expression
| multiplicative_expression MOD unary_expression
;
additive_expression ::=
multiplicative_expression
| additive_expression PLUS multiplicative_expression
| additive_expression MINUS multiplicative_expression
;
shift_expression ::=
additive_expression
| shift_expression LSHIFT additive_expression
| shift_expression RSHIFT additive_expression
| shift_expression URSHIFT additive_expression
;
relational_expression ::=
shift_expression
| relational_expression LT shift_expression
| relational_expression GT shift_expression
| relational_expression LTEQ shift_expression
| relational_expression GTEQ shift_expression
| relational_expression INSTANCEOF reference_type
;
equality_expression ::=
relational_expression
| equality_expression EQEQ relational_expression
| equality_expression NOTEQ relational_expression
;
and_expression ::=
equality_expression
| and_expression AND equality_expression
;
exclusive_or_expression ::=
and_expression
| exclusive_or_expression XOR and_expression
;
inclusive_or_expression ::=
exclusive_or_expression
| inclusive_or_expression OR exclusive_or_expression
;
conditional_and_expression ::=
inclusive_or_expression
| conditional_and_expression ANDAND inclusive_or_expression
;
conditional_or_expression ::=
conditional_and_expression
| conditional_or_expression OROR conditional_and_expression
;
conditional_expression ::=
conditional_or_expression
| conditional_or_expression QUESTION expression
COLON conditional_expression
;
assignment_expression ::=
conditional_expression
| assignment
;
assignment ::= left_hand_side assignment_operator assignment_expression
;
left_hand_side ::=
name
| field_access
| array_access
;
assignment_operator ::=
EQ
| MULTEQ
| DIVEQ
| MODEQ
| PLUSEQ
| MINUSEQ
| LSHIFTEQ
| RSHIFTEQ
| URSHIFTEQ
| ANDEQ
| XOREQ
| OREQ
;
expression_opt ::=
| expression
;
expression ::= assignment_expression
;
constant_expression ::=
expression
;
tools/lexer/jflex-1.4/examples/java/lexer-output.good
-376
View File
@@ -1,376 +0,0 @@
Lexing [TestLexer.java]
line:20 col:1 --import--IMPORT--
token: line 20, column 1, sym: 23
line:20 col:8 --java--IDENTIFIER--
token: line 20, column 8, sym: 98, value: 'java'
line:20 col:12 --.--DOT--
token: line 20, column 12, sym: 12
line:20 col:13 --io--IDENTIFIER--
token: line 20, column 13, sym: 98, value: 'io'
line:20 col:15 --.--DOT--
token: line 20, column 15, sym: 12
line:20 col:16 --*--MULT--
token: line 20, column 16, sym: 14
line:20 col:17 --;--SEMICOLON--
token: line 20, column 17, sym: 13
line:21 col:1 --import--IMPORT--
token: line 21, column 1, sym: 23
line:21 col:8 --java_cup--IDENTIFIER--
token: line 21, column 8, sym: 98, value: 'java_cup'
line:21 col:16 --.--DOT--
token: line 21, column 16, sym: 12
line:21 col:17 --runtime--IDENTIFIER--
token: line 21, column 17, sym: 98, value: 'runtime'
line:21 col:24 --.--DOT--
token: line 21, column 24, sym: 12
line:21 col:25 --Symbol--IDENTIFIER--
token: line 21, column 25, sym: 98, value: 'Symbol'
line:21 col:31 --;--SEMICOLON--
token: line 21, column 31, sym: 13
line:29 col:1 --public--PUBLIC--
token: line 29, column 1, sym: 24
line:29 col:8 --class--CLASS--
token: line 29, column 8, sym: 34
line:29 col:14 --TestLexer--IDENTIFIER--
token: line 29, column 14, sym: 98, value: 'TestLexer'
line:29 col:24 --{--LBRACE--
token: line 29, column 24, sym: 16
line:32 col:3 --int--INT--
token: line 32, column 3, sym: 5
line:32 col:7 --intDec--IDENTIFIER--
token: line 32, column 7, sym: 98, value: 'intDec'
line:32 col:14 --=--EQ--
token: line 32, column 14, sym: 18
line:32 col:16 --37--INTEGER_LITERAL--
token: line 32, column 16, sym: 93, value: '37'
line:32 col:18 --;--SEMICOLON--
token: line 32, column 18, sym: 13
line:33 col:3 --long--LONG--
token: line 33, column 3, sym: 6
line:33 col:8 --longDec--IDENTIFIER--
token: line 33, column 8, sym: 98, value: 'longDec'
line:33 col:16 --=--EQ--
token: line 33, column 16, sym: 18
line:33 col:18 --37l--INTEGER_LITERAL--
token: line 33, column 18, sym: 93, value: '37'
line:33 col:21 --;--SEMICOLON--
token: line 33, column 21, sym: 13
line:34 col:3 --int--INT--
token: line 34, column 3, sym: 5
line:34 col:7 --intHex--IDENTIFIER--
token: line 34, column 7, sym: 98, value: 'intHex'
line:34 col:14 --=--EQ--
token: line 34, column 14, sym: 18
line:34 col:16 --0x0001--INTEGER_LITERAL--
token: line 34, column 16, sym: 93, value: '1'
line:34 col:22 --;--SEMICOLON--
token: line 34, column 22, sym: 13
line:35 col:3 --long--LONG--
token: line 35, column 3, sym: 6
line:35 col:8 --longHex--IDENTIFIER--
token: line 35, column 8, sym: 98, value: 'longHex'
line:35 col:16 --=--EQ--
token: line 35, column 16, sym: 18
line:35 col:18 --0xFFFFl--INTEGER_LITERAL--
token: line 35, column 18, sym: 93, value: '65535'
line:35 col:25 --;--SEMICOLON--
token: line 35, column 25, sym: 13
line:36 col:3 --int--INT--
token: line 36, column 3, sym: 5
line:36 col:7 --intOct--IDENTIFIER--
token: line 36, column 7, sym: 98, value: 'intOct'
line:36 col:14 --=--EQ--
token: line 36, column 14, sym: 18
line:36 col:16 --0377--INTEGER_LITERAL--
token: line 36, column 16, sym: 93, value: '255'
line:36 col:20 --;--SEMICOLON--
token: line 36, column 20, sym: 13
line:37 col:3 --long--LONG--
token: line 37, column 3, sym: 6
line:37 col:8 --longOc--IDENTIFIER--
token: line 37, column 8, sym: 98, value: 'longOc'
line:37 col:15 --=--EQ--
token: line 37, column 15, sym: 18
line:37 col:17 --007l--INTEGER_LITERAL--
token: line 37, column 17, sym: 93, value: '7'
line:37 col:21 --;--SEMICOLON--
token: line 37, column 21, sym: 13
line:40 col:3 --public--PUBLIC--
token: line 40, column 3, sym: 24
line:40 col:10 --static--STATIC--
token: line 40, column 10, sym: 27
line:40 col:17 --void--VOID--
token: line 40, column 17, sym: 37
line:40 col:22 --main--IDENTIFIER--
token: line 40, column 22, sym: 98, value: 'main'
line:40 col:26 --(--LPAREN--
token: line 40, column 26, sym: 19
line:40 col:27 --String--IDENTIFIER--
token: line 40, column 27, sym: 98, value: 'String'
line:40 col:34 --argv--IDENTIFIER--
token: line 40, column 34, sym: 98, value: 'argv'
line:40 col:38 --[--LBRACK--
token: line 40, column 38, sym: 10
line:40 col:39 --]--RBRACK--
token: line 40, column 39, sym: 11
line:40 col:40 --)--RPAREN--
token: line 40, column 40, sym: 20
line:40 col:42 --{--LBRACE--
token: line 40, column 42, sym: 16
line:42 col:5 --for--FOR--
token: line 42, column 5, sym: 49
line:42 col:9 --(--LPAREN--
token: line 42, column 9, sym: 19
line:42 col:10 --int--INT--
token: line 42, column 10, sym: 5
line:42 col:14 --i--IDENTIFIER--
token: line 42, column 14, sym: 98, value: 'i'
line:42 col:16 --=--EQ--
token: line 42, column 16, sym: 18
line:42 col:18 --0--INTEGER_LITERAL--
token: line 42, column 18, sym: 93, value: '0'
line:42 col:19 --;--SEMICOLON--
token: line 42, column 19, sym: 13
line:42 col:21 --i--IDENTIFIER--
token: line 42, column 21, sym: 98, value: 'i'
line:42 col:23 --<--LT--
token: line 42, column 23, sym: 69
line:42 col:25 --argv--IDENTIFIER--
token: line 42, column 25, sym: 98, value: 'argv'
line:42 col:29 --.--DOT--
token: line 42, column 29, sym: 12
line:42 col:30 --length--IDENTIFIER--
token: line 42, column 30, sym: 98, value: 'length'
line:42 col:36 --;--SEMICOLON--
token: line 42, column 36, sym: 13
line:42 col:38 --i--IDENTIFIER--
token: line 42, column 38, sym: 98, value: 'i'
line:42 col:39 --++--PLUSPLUS--
token: line 42, column 39, sym: 58
line:42 col:41 --)--RPAREN--
token: line 42, column 41, sym: 20
line:42 col:43 --{--LBRACE--
token: line 42, column 43, sym: 16
line:43 col:7 --try--TRY--
token: line 43, column 7, sym: 54
line:43 col:11 --{--LBRACE--
token: line 43, column 11, sym: 16
line:44 col:9 --System--IDENTIFIER--
token: line 44, column 9, sym: 98, value: 'System'
line:44 col:15 --.--DOT--
token: line 44, column 15, sym: 12
line:44 col:16 --out--IDENTIFIER--
token: line 44, column 16, sym: 98, value: 'out'
line:44 col:19 --.--DOT--
token: line 44, column 19, sym: 12
line:44 col:20 --println--IDENTIFIER--
token: line 44, column 20, sym: 98, value: 'println'
line:44 col:27 --(--LPAREN--
token: line 44, column 27, sym: 19
line:44 col:37 --"--STRING_LITERAL--
token: line 44, column 37, sym: 97, value: 'Lexing ['
line:44 col:38 --+--PLUS--
token: line 44, column 38, sym: 60
line:44 col:39 --argv--IDENTIFIER--
token: line 44, column 39, sym: 98, value: 'argv'
line:44 col:43 --[--LBRACK--
token: line 44, column 43, sym: 10
line:44 col:44 --i--IDENTIFIER--
token: line 44, column 44, sym: 98, value: 'i'
line:44 col:45 --]--RBRACK--
token: line 44, column 45, sym: 11
line:44 col:46 --+--PLUS--
token: line 44, column 46, sym: 60
line:44 col:49 --"--STRING_LITERAL--
token: line 44, column 49, sym: 97, value: ']'
line:44 col:50 --)--RPAREN--
token: line 44, column 50, sym: 20
line:44 col:51 --;--SEMICOLON--
token: line 44, column 51, sym: 13
line:45 col:9 --Scanner--IDENTIFIER--
token: line 45, column 9, sym: 98, value: 'Scanner'
line:45 col:17 --scanner--IDENTIFIER--
token: line 45, column 17, sym: 98, value: 'scanner'
line:45 col:25 --=--EQ--
token: line 45, column 25, sym: 18
line:45 col:27 --new--NEW--
token: line 45, column 27, sym: 57
line:45 col:31 --Scanner--IDENTIFIER--
token: line 45, column 31, sym: 98, value: 'Scanner'
line:45 col:38 --(--LPAREN--
token: line 45, column 38, sym: 19
line:45 col:39 --new--NEW--
token: line 45, column 39, sym: 57
line:45 col:43 --UnicodeEscapes--IDENTIFIER--
token: line 45, column 43, sym: 98, value: 'UnicodeEscapes'
line:45 col:57 --(--LPAREN--
token: line 45, column 57, sym: 19
line:45 col:58 --new--NEW--
token: line 45, column 58, sym: 57
line:45 col:62 --FileReader--IDENTIFIER--
token: line 45, column 62, sym: 98, value: 'FileReader'
line:45 col:72 --(--LPAREN--
token: line 45, column 72, sym: 19
line:45 col:73 --argv--IDENTIFIER--
token: line 45, column 73, sym: 98, value: 'argv'
line:45 col:77 --[--LBRACK--
token: line 45, column 77, sym: 10
line:45 col:78 --i--IDENTIFIER--
token: line 45, column 78, sym: 98, value: 'i'
line:45 col:79 --]--RBRACK--
token: line 45, column 79, sym: 11
line:45 col:80 --)--RPAREN--
token: line 45, column 80, sym: 20
line:45 col:81 --)--RPAREN--
token: line 45, column 81, sym: 20
line:45 col:82 --)--RPAREN--
token: line 45, column 82, sym: 20
line:45 col:83 --;--SEMICOLON--
token: line 45, column 83, sym: 13
line:47 col:9 --Symbol--IDENTIFIER--
token: line 47, column 9, sym: 98, value: 'Symbol'
line:47 col:16 --s--IDENTIFIER--
token: line 47, column 16, sym: 98, value: 's'
line:47 col:17 --;--SEMICOLON--
token: line 47, column 17, sym: 13
line:48 col:9 --do--DO--
token: line 48, column 9, sym: 47
line:48 col:12 --{--LBRACE--
token: line 48, column 12, sym: 16
line:49 col:11 --s--IDENTIFIER--
token: line 49, column 11, sym: 98, value: 's'
line:49 col:13 --=--EQ--
token: line 49, column 13, sym: 18
line:49 col:15 --scanner--IDENTIFIER--
token: line 49, column 15, sym: 98, value: 'scanner'
line:49 col:22 --.--DOT--
token: line 49, column 22, sym: 12
line:49 col:23 --debug_next_token--IDENTIFIER--
token: line 49, column 23, sym: 98, value: 'debug_next_token'
line:49 col:39 --(--LPAREN--
token: line 49, column 39, sym: 19
line:49 col:40 --)--RPAREN--
token: line 49, column 40, sym: 20
line:49 col:41 --;--SEMICOLON--
token: line 49, column 41, sym: 13
line:50 col:11 --System--IDENTIFIER--
token: line 50, column 11, sym: 98, value: 'System'
line:50 col:17 --.--DOT--
token: line 50, column 17, sym: 12
line:50 col:18 --out--IDENTIFIER--
token: line 50, column 18, sym: 98, value: 'out'
line:50 col:21 --.--DOT--
token: line 50, column 21, sym: 12
line:50 col:22 --println--IDENTIFIER--
token: line 50, column 22, sym: 98, value: 'println'
line:50 col:29 --(--LPAREN--
token: line 50, column 29, sym: 19
line:50 col:38 --"--STRING_LITERAL--
token: line 50, column 38, sym: 97, value: 'token: '
line:50 col:39 --+--PLUS--
token: line 50, column 39, sym: 60
line:50 col:40 --s--IDENTIFIER--
token: line 50, column 40, sym: 98, value: 's'
line:50 col:41 --)--RPAREN--
token: line 50, column 41, sym: 20
line:50 col:42 --;--SEMICOLON--
token: line 50, column 42, sym: 13
line:51 col:9 --}--RBRACE--
token: line 51, column 9, sym: 17
line:51 col:11 --while--WHILE--
token: line 51, column 11, sym: 48
line:51 col:17 --(--LPAREN--
token: line 51, column 17, sym: 19
line:51 col:18 --s--IDENTIFIER--
token: line 51, column 18, sym: 98, value: 's'
line:51 col:19 --.--DOT--
token: line 51, column 19, sym: 12
line:51 col:20 --sym--IDENTIFIER--
token: line 51, column 20, sym: 98, value: 'sym'
line:51 col:24 --!=--NOTEQ--
token: line 51, column 24, sym: 75
line:51 col:27 --sym--IDENTIFIER--
token: line 51, column 27, sym: 98, value: 'sym'
line:51 col:30 --.--DOT--
token: line 51, column 30, sym: 12
line:51 col:31 --EOF--IDENTIFIER--
token: line 51, column 31, sym: 98, value: 'EOF'
line:51 col:34 --)--RPAREN--
token: line 51, column 34, sym: 20
line:51 col:35 --;--SEMICOLON--
token: line 51, column 35, sym: 13
line:53 col:9 --System--IDENTIFIER--
token: line 53, column 9, sym: 98, value: 'System'
line:53 col:15 --.--DOT--
token: line 53, column 15, sym: 12
line:53 col:16 --out--IDENTIFIER--
token: line 53, column 16, sym: 98, value: 'out'
line:53 col:19 --.--DOT--
token: line 53, column 19, sym: 12
line:53 col:20 --println--IDENTIFIER--
token: line 53, column 20, sym: 98, value: 'println'
line:53 col:27 --(--LPAREN--
token: line 53, column 27, sym: 19
line:53 col:39 --"--STRING_LITERAL--
token: line 53, column 39, sym: 97, value: 'No errors.'
line:53 col:40 --)--RPAREN--
token: line 53, column 40, sym: 20
line:53 col:41 --;--SEMICOLON--
token: line 53, column 41, sym: 13
line:54 col:7 --}--RBRACE--
token: line 54, column 7, sym: 17
line:55 col:7 --catch--CATCH--
token: line 55, column 7, sym: 55
line:55 col:13 --(--LPAREN--
token: line 55, column 13, sym: 19
line:55 col:14 --Exception--IDENTIFIER--
token: line 55, column 14, sym: 98, value: 'Exception'
line:55 col:24 --e--IDENTIFIER--
token: line 55, column 24, sym: 98, value: 'e'
line:55 col:25 --)--RPAREN--
token: line 55, column 25, sym: 20
line:55 col:27 --{--LBRACE--
token: line 55, column 27, sym: 16
line:56 col:9 --e--IDENTIFIER--
token: line 56, column 9, sym: 98, value: 'e'
line:56 col:10 --.--DOT--
token: line 56, column 10, sym: 12
line:56 col:11 --printStackTrace--IDENTIFIER--
token: line 56, column 11, sym: 98, value: 'printStackTrace'
line:56 col:26 --(--LPAREN--
token: line 56, column 26, sym: 19
line:56 col:27 --System--IDENTIFIER--
token: line 56, column 27, sym: 98, value: 'System'
line:56 col:33 --.--DOT--
token: line 56, column 33, sym: 12
line:56 col:34 --out--IDENTIFIER--
token: line 56, column 34, sym: 98, value: 'out'
line:56 col:37 --)--RPAREN--
token: line 56, column 37, sym: 20
line:56 col:38 --;--SEMICOLON--
token: line 56, column 38, sym: 13
line:57 col:9 --System--IDENTIFIER--
token: line 57, column 9, sym: 98, value: 'System'
line:57 col:15 --.--DOT--
token: line 57, column 15, sym: 12
line:57 col:16 --exit--IDENTIFIER--
token: line 57, column 16, sym: 98, value: 'exit'
line:57 col:20 --(--LPAREN--
token: line 57, column 20, sym: 19
line:57 col:21 --1--INTEGER_LITERAL--
token: line 57, column 21, sym: 93, value: '1'
line:57 col:22 --)--RPAREN--
token: line 57, column 22, sym: 20
line:57 col:23 --;--SEMICOLON--
token: line 57, column 23, sym: 13
line:58 col:7 --}--RBRACE--
token: line 58, column 7, sym: 17
line:59 col:5 --}--RBRACE--
token: line 59, column 5, sym: 17
line:60 col:3 --}--RBRACE--
token: line 60, column 3, sym: 17
line:61 col:1 --}--RBRACE--
token: line 61, column 1, sym: 17
line:62 col:1 ----EOF--
token: line 62, column 1, sym: 0
No errors.
tools/lexer/jflex-1.4/examples/java/unicode.flex
-149
View File
@@ -1,149 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 1998-2004 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* §3.3 of the Java Language Specification :
UnicodeInputCharacter:
UnicodeEscape
RawInputCharacter
UnicodeEscape:
\ UnicodeMarker HexDigit HexDigit HexDigit HexDigit
UnicodeMarker:
u
UnicodeMarker u
RawInputCharacter:
any Unicode character
HexDigit: one of
0 1 2 3 4 5 6 7 8 9 a b c d e f A B C D E F
only an even number of '\' is eligible to start a Unicode escape sequence
*/
import java.io.*;
%%
%public
%final
%class UnicodeEscapes
%extends FilterReader
%int
%function read
%switch
%16bit
UnicodeEscape = {UnicodeMarker} {HexDigit} {4}
UnicodeMarker = "u"+
HexDigit = [0-9a-fA-F]
%state DIGITS
%init{
super(in);
%init}
%{
private boolean even;
private int value() {
int r = 0;
for (int k = zzMarkedPos-4; k < zzMarkedPos; k++) {
int c = zzBuffer[k];
if (c >= 'a')
c-= 'a'-10;
else if (c >= 'A')
c-= 'A'-10;
else
c-= '0';
r <<= 4;
r += c;
}
return r;
}
public int read(char cbuf[], int off, int len) throws IOException {
if ( !ready() ) return -1;
len+= off;
for (int i=off; i<len; i++) {
int c = read();
if (c < 0)
return i-off;
else
cbuf[i] = (char) c;
}
return len-off;
}
public boolean markSupported() {
return false;
}
public boolean ready() throws IOException {
return !zzAtEOF && (zzCurrentPos < zzEndRead || zzReader.ready());
}
%}
%%
<YYINITIAL> {
\\ { even = false; return '\\'; }
\\ / \\ { even = !even; return '\\'; }
\\ / "u" {
if (even) {
even = false;
return '\\';
}
else
yybegin(DIGITS);
}
.|\n { return zzBuffer[zzStartRead]; }
<<EOF>> { return -1; }
}
<DIGITS> {
{UnicodeEscape} { yybegin(YYINITIAL); return value(); }
.|\n { throw new Error("incorrect Unicode escape"); }
<<EOF>> { throw new Error("EOF in Unicode escape"); }
}
tools/lexer/jflex-1.4/examples/simple/Makefile
-23
View File
@@ -1,23 +0,0 @@
JAVA=java
JAVAC=javac
JFLEX=jflex
CUP=$(JAVA) java_cup.Main <
all: test
test: output.txt
@(diff output.txt output.good && echo "Test OK!") || echo "Test failed!"
output.txt: Yylex.class test.txt
$(JAVA) Yylex test.txt > output.txt
Yylex.class: Yylex.java Yytoken.java Utility.java
%.class: %.java
$(JAVAC) $^
Yylex.java: simple.flex
$(JFLEX) simple.flex
clean:
rm -f Yylex.java output.txt *.class *~
tools/lexer/jflex-1.4/examples/simple/README
-29
View File
@@ -1,29 +0,0 @@
This directory contains a scanner for a toy programming language.
It is the example from the JLex website with some small modifications,
to make it a bit more readable.
It does nothing really useful, because there is no parser for the
toy programming language. It's just a demonstatration how a small simple
scanner looks like.
files:
YYtoken.java:
the tokens returned by the scanner
Utility.java:
small class collecting some utility functions
simple.flex:
the example specification
test.txt:
sample input
output.good:
output matching the sample input from test.txt
Makefile: (for Unix only)
make to generate scanner and run test
make clean to remove generated files
tools/lexer/jflex-1.4/examples/simple/Utility.java
-19
View File
@@ -1,19 +0,0 @@
class Utility {
private static final String errorMsg[] = {
"Error: Unmatched end-of-comment punctuation.",
"Error: Unmatched start-of-comment punctuation.",
"Error: Unclosed string.",
"Error: Illegal character."
};
public static final int E_ENDCOMMENT = 0;
public static final int E_STARTCOMMENT = 1;
public static final int E_UNCLOSEDSTR = 2;
public static final int E_UNMATCHED = 3;
public static void error(int code) {
System.out.println(errorMsg[code]);
}
}
tools/lexer/jflex-1.4/examples/simple/Yytoken.java
-24
View File
@@ -1,24 +0,0 @@
class Yytoken {
public int m_index;
public String m_text;
public int m_line;
public int m_charBegin;
public int m_charEnd;
Yytoken (int index, String text, int line, int charBegin, int charEnd) {
m_index = index;
m_text = text;
m_line = line;
m_charBegin = charBegin;
m_charEnd = charEnd;
}
public String toString() {
return "Text : "+m_text+
"\nindex : "+m_index+
"\nline : "+m_line+
"\ncBeg. : "+m_charBegin+
"\ncEnd. : "+m_charEnd;
}
}
tools/lexer/jflex-1.4/examples/simple/output.good
-660
View File
@@ -1,660 +0,0 @@
line: 1 match: --class--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : class
index : 43
line : 0
cBeg. : 0
cEnd. : 5
line: 1 match: -- --
action [76] { }
line: 1 match: --Utility--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : Utility
index : 43
line : 0
cBeg. : 6
cEnd. : 13
line: 1 match: -- --
action [76] { }
line: 1 match: --{--
action [36] { return (new Yytoken(7,yytext(),yyline,yychar,yychar+1)); }
Text : {
index : 7
line : 0
cBeg. : 14
cEnd. : 15
line: 1 match: --
--
action [76] { }
line: 3 match: --private--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : private
index : 43
line : 2
cBeg. : 21
cEnd. : 28
line: 3 match: -- --
action [76] { }
line: 3 match: --static--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : static
index : 43
line : 2
cBeg. : 29
cEnd. : 35
line: 3 match: -- --
action [76] { }
line: 3 match: --final--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : final
index : 43
line : 2
cBeg. : 36
cEnd. : 41
line: 3 match: -- --
action [76] { }
line: 3 match: --String--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : String
index : 43
line : 2
cBeg. : 42
cEnd. : 48
line: 3 match: -- --
action [76] { }
line: 3 match: --errorMsg--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : errorMsg
index : 43
line : 2
cBeg. : 49
cEnd. : 57
line: 3 match: --[--
action [34] { return (new Yytoken(5,yytext(),yyline,yychar,yychar+1)); }
Text : [
index : 5
line : 2
cBeg. : 57
cEnd. : 58
line: 3 match: --]--
action [35] { return (new Yytoken(6,yytext(),yyline,yychar,yychar+1)); }
Text : ]
index : 6
line : 2
cBeg. : 58
cEnd. : 59
line: 3 match: -- --
action [76] { }
line: 3 match: --=--
action [43] { return (new Yytoken(14,yytext(),yyline,yychar,yychar+1)); }
Text : =
index : 14
line : 2
cBeg. : 60
cEnd. : 61
line: 3 match: -- --
action [76] { }
line: 3 match: --{--
action [36] { return (new Yytoken(7,yytext(),yyline,yychar,yychar+1)); }
Text : {
index : 7
line : 2
cBeg. : 62
cEnd. : 63
line: 3 match: --
--
action [76] { }
line: 4 match: --"Error: Unmatched end-of-comment punctuation."--
action [57] { String str = yytext().substring(1,yylength()-1);
return (new Yytoken(40,str,yyline,yychar,yychar+yylength())); }
Text : Error: Unmatched end-of-comment punctuation.
index : 40
line : 3
cBeg. : 68
cEnd. : 114
line: 4 match: --,--
action [29] { return (new Yytoken(0,yytext(),yyline,yychar,yychar+1)); }
Text : ,
index : 0
line : 3
cBeg. : 114
cEnd. : 115
line: 4 match: --
--
action [76] { }
line: 5 match: --"Error: Unmatched start-of-comment punctuation."--
action [57] { String str = yytext().substring(1,yylength()-1);
return (new Yytoken(40,str,yyline,yychar,yychar+yylength())); }
Text : Error: Unmatched start-of-comment punctuation.
index : 40
line : 4
cBeg. : 120
cEnd. : 168
line: 5 match: --,--
action [29] { return (new Yytoken(0,yytext(),yyline,yychar,yychar+1)); }
Text : ,
index : 0
line : 4
cBeg. : 168
cEnd. : 169
line: 5 match: --
--
action [76] { }
line: 6 match: --"Error: Unclosed string."--
action [57] { String str = yytext().substring(1,yylength()-1);
return (new Yytoken(40,str,yyline,yychar,yychar+yylength())); }
Text : Error: Unclosed string.
index : 40
line : 5
cBeg. : 174
cEnd. : 199
line: 6 match: --,--
action [29] { return (new Yytoken(0,yytext(),yyline,yychar,yychar+1)); }
Text : ,
index : 0
line : 5
cBeg. : 199
cEnd. : 200
line: 6 match: --
--
action [76] { }
line: 7 match: --"Error: Illegal character."--
action [57] { String str = yytext().substring(1,yylength()-1);
return (new Yytoken(40,str,yyline,yychar,yychar+yylength())); }
Text : Error: Illegal character.
index : 40
line : 6
cBeg. : 205
cEnd. : 232
line: 7 match: --
--
action [76] { }
line: 8 match: --}--
action [37] { return (new Yytoken(8,yytext(),yyline,yychar,yychar+1)); }
Text : }
index : 8
line : 7
cBeg. : 237
cEnd. : 238
line: 8 match: --;--
action [31] { return (new Yytoken(2,yytext(),yyline,yychar,yychar+1)); }
Text : ;
index : 2
line : 7
cBeg. : 238
cEnd. : 239
line: 8 match: --
--
action [76] { }
line: 10 match: --public--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : public
index : 43
line : 9
cBeg. : 245
cEnd. : 251
line: 10 match: -- --
action [76] { }
line: 10 match: --static--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : static
index : 43
line : 9
cBeg. : 252
cEnd. : 258
line: 10 match: -- --
action [76] { }
line: 10 match: --final--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : final
index : 43
line : 9
cBeg. : 259
cEnd. : 264
line: 10 match: -- --
action [76] { }
line: 10 match: --int--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : int
index : 43
line : 9
cBeg. : 265
cEnd. : 268
line: 10 match: -- --
action [76] { }
line: 10 match: --E_ENDCOMMENT--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : E_ENDCOMMENT
index : 43
line : 9
cBeg. : 269
cEnd. : 281
line: 10 match: -- --
action [76] { }
line: 10 match: --=--
action [43] { return (new Yytoken(14,yytext(),yyline,yychar,yychar+1)); }
Text : =
index : 14
line : 9
cBeg. : 282
cEnd. : 283
line: 10 match: -- --
action [76] { }
line: 10 match: --0--
action [68] { return (new Yytoken(42,yytext(),yyline,yychar,yychar+yylength())); }
Text : 0
index : 42
line : 9
cBeg. : 284
cEnd. : 285
line: 10 match: --;--
action [31] { return (new Yytoken(2,yytext(),yyline,yychar,yychar+1)); }
Text : ;
index : 2
line : 9
cBeg. : 285
cEnd. : 286
line: 10 match: --
--
action [76] { }
line: 11 match: --public--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : public
index : 43
line : 10
cBeg. : 290
cEnd. : 296
line: 11 match: -- --
action [76] { }
line: 11 match: --static--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : static
index : 43
line : 10
cBeg. : 297
cEnd. : 303
line: 11 match: -- --
action [76] { }
line: 11 match: --final--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : final
index : 43
line : 10
cBeg. : 304
cEnd. : 309
line: 11 match: -- --
action [76] { }
line: 11 match: --int--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : int
index : 43
line : 10
cBeg. : 310
cEnd. : 313
line: 11 match: -- --
action [76] { }
line: 11 match: --E_STARTCOMMENT--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : E_STARTCOMMENT
index : 43
line : 10
cBeg. : 314
cEnd. : 328
line: 11 match: -- --
action [76] { }
line: 11 match: --=--
action [43] { return (new Yytoken(14,yytext(),yyline,yychar,yychar+1)); }
Text : =
index : 14
line : 10
cBeg. : 329
cEnd. : 330
line: 11 match: -- --
action [76] { }
line: 11 match: --1--
action [68] { return (new Yytoken(42,yytext(),yyline,yychar,yychar+yylength())); }
Text : 1
index : 42
line : 10
cBeg. : 331
cEnd. : 332
line: 11 match: --;--
action [31] { return (new Yytoken(2,yytext(),yyline,yychar,yychar+1)); }
Text : ;
index : 2
line : 10
cBeg. : 332
cEnd. : 333
line: 11 match: --
--
action [76] { }
line: 12 match: --public--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : public
index : 43
line : 11
cBeg. : 337
cEnd. : 343
line: 12 match: -- --
action [76] { }
line: 12 match: --static--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : static
index : 43
line : 11
cBeg. : 344
cEnd. : 350
line: 12 match: -- --
action [76] { }
line: 12 match: --final--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : final
index : 43
line : 11
cBeg. : 351
cEnd. : 356
line: 12 match: -- --
action [76] { }
line: 12 match: --int--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : int
index : 43
line : 11
cBeg. : 357
cEnd. : 360
line: 12 match: -- --
action [76] { }
line: 12 match: --E_UNCLOSEDSTR--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : E_UNCLOSEDSTR
index : 43
line : 11
cBeg. : 361
cEnd. : 374
line: 12 match: -- --
action [76] { }
line: 12 match: --=--
action [43] { return (new Yytoken(14,yytext(),yyline,yychar,yychar+1)); }
Text : =
index : 14
line : 11
cBeg. : 375
cEnd. : 376
line: 12 match: -- --
action [76] { }
line: 12 match: --2--
action [68] { return (new Yytoken(42,yytext(),yyline,yychar,yychar+yylength())); }
Text : 2
index : 42
line : 11
cBeg. : 377
cEnd. : 378
line: 12 match: --;--
action [31] { return (new Yytoken(2,yytext(),yyline,yychar,yychar+1)); }
Text : ;
index : 2
line : 11
cBeg. : 378
cEnd. : 379
line: 12 match: --
--
action [76] { }
line: 13 match: --public--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : public
index : 43
line : 12
cBeg. : 383
cEnd. : 389
line: 13 match: -- --
action [76] { }
line: 13 match: --static--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : static
index : 43
line : 12
cBeg. : 390
cEnd. : 396
line: 13 match: -- --
action [76] { }
line: 13 match: --final--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : final
index : 43
line : 12
cBeg. : 397
cEnd. : 402
line: 13 match: -- --
action [76] { }
line: 13 match: --int--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : int
index : 43
line : 12
cBeg. : 403
cEnd. : 406
line: 13 match: -- --
action [76] { }
line: 13 match: --E_UNMATCHED--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : E_UNMATCHED
index : 43
line : 12
cBeg. : 407
cEnd. : 418
line: 13 match: -- --
action [76] { }
line: 13 match: --=--
action [43] { return (new Yytoken(14,yytext(),yyline,yychar,yychar+1)); }
Text : =
index : 14
line : 12
cBeg. : 419
cEnd. : 420
line: 13 match: -- --
action [76] { }
line: 13 match: --3--
action [68] { return (new Yytoken(42,yytext(),yyline,yychar,yychar+yylength())); }
Text : 3
index : 42
line : 12
cBeg. : 421
cEnd. : 422
line: 13 match: --;--
action [31] { return (new Yytoken(2,yytext(),yyline,yychar,yychar+1)); }
Text : ;
index : 2
line : 12
cBeg. : 422
cEnd. : 423
line: 13 match: --
--
action [76] { }
line: 15 match: --public--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : public
index : 43
line : 14
cBeg. : 428
cEnd. : 434
line: 15 match: -- --
action [76] { }
line: 15 match: --static--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : static
index : 43
line : 14
cBeg. : 435
cEnd. : 441
line: 15 match: -- --
action [76] { }
line: 15 match: --void--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : void
index : 43
line : 14
cBeg. : 442
cEnd. : 446
line: 15 match: -- --
action [76] { }
line: 15 match: --error--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : error
index : 43
line : 14
cBeg. : 447
cEnd. : 452
line: 15 match: --(--
action [32] { return (new Yytoken(3,yytext(),yyline,yychar,yychar+1)); }
Text : (
index : 3
line : 14
cBeg. : 452
cEnd. : 453
line: 15 match: --int--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : int
index : 43
line : 14
cBeg. : 453
cEnd. : 456
line: 15 match: -- --
action [76] { }
line: 15 match: --code--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : code
index : 43
line : 14
cBeg. : 457
cEnd. : 461
line: 15 match: --)--
action [33] { return (new Yytoken(4,yytext(),yyline,yychar,yychar+1)); }
Text : )
index : 4
line : 14
cBeg. : 461
cEnd. : 462
line: 15 match: -- --
action [76] { }
line: 15 match: --{--
action [36] { return (new Yytoken(7,yytext(),yyline,yychar,yychar+1)); }
Text : {
index : 7
line : 14
cBeg. : 463
cEnd. : 464
line: 15 match: --
--
action [76] { }
line: 16 match: --System--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : System
index : 43
line : 15
cBeg. : 468
cEnd. : 474
line: 16 match: --.--
action [38] { return (new Yytoken(9,yytext(),yyline,yychar,yychar+1)); }
Text : .
index : 9
line : 15
cBeg. : 474
cEnd. : 475
line: 16 match: --out--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : out
index : 43
line : 15
cBeg. : 475
cEnd. : 478
line: 16 match: --.--
action [38] { return (new Yytoken(9,yytext(),yyline,yychar,yychar+1)); }
Text : .
index : 9
line : 15
cBeg. : 478
cEnd. : 479
line: 16 match: --println--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : println
index : 43
line : 15
cBeg. : 479
cEnd. : 486
line: 16 match: --(--
action [32] { return (new Yytoken(3,yytext(),yyline,yychar,yychar+1)); }
Text : (
index : 3
line : 15
cBeg. : 486
cEnd. : 487
line: 16 match: --errorMsg--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : errorMsg
index : 43
line : 15
cBeg. : 487
cEnd. : 495
line: 16 match: --[--
action [34] { return (new Yytoken(5,yytext(),yyline,yychar,yychar+1)); }
Text : [
index : 5
line : 15
cBeg. : 495
cEnd. : 496
line: 16 match: --code--
action [70] { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
Text : code
index : 43
line : 15
cBeg. : 496
cEnd. : 500
line: 16 match: --]--
action [35] { return (new Yytoken(6,yytext(),yyline,yychar,yychar+1)); }
Text : ]
index : 6
line : 15
cBeg. : 500
cEnd. : 501
line: 16 match: --)--
action [33] { return (new Yytoken(4,yytext(),yyline,yychar,yychar+1)); }
Text : )
index : 4
line : 15
cBeg. : 501
cEnd. : 502
line: 16 match: --;--
action [31] { return (new Yytoken(2,yytext(),yyline,yychar,yychar+1)); }
Text : ;
index : 2
line : 15
cBeg. : 502
cEnd. : 503
line: 16 match: --
--
action [76] { }
line: 17 match: --}--
action [37] { return (new Yytoken(8,yytext(),yyline,yychar,yychar+1)); }
Text : }
index : 8
line : 16
cBeg. : 506
cEnd. : 507
line: 17 match: --
--
action [76] { }
line: 18 match: --}--
action [37] { return (new Yytoken(8,yytext(),yyline,yychar,yychar+1)); }
Text : }
index : 8
line : 17
cBeg. : 508
cEnd. : 509
line: 18 match: --
--
action [76] { }
null
tools/lexer/jflex-1.4/examples/simple/simple.flex
-86
View File
@@ -1,86 +0,0 @@
/* this is the scanner example from the JLex website
(with small modifications to make it more readable) */
%%
%{
private int comment_count = 0;
%}
%line
%char
%state COMMENT
%full
%debug
ALPHA=[A-Za-z]
DIGIT=[0-9]
NONNEWLINE_WHITE_SPACE_CHAR=[\ \t\b\012]
NEWLINE=\r|\n|\r\n
WHITE_SPACE_CHAR=[\n\r\ \t\b\012]
STRING_TEXT=(\\\"|[^\n\r\"]|\\{WHITE_SPACE_CHAR}+\\)*
COMMENT_TEXT=([^*/\n]|[^*\n]"/"[^*\n]|[^/\n]"*"[^/\n]|"*"[^/\n]|"/"[^*\n])*
Ident = {ALPHA}({ALPHA}|{DIGIT}|_)*
%%
<YYINITIAL> {
"," { return (new Yytoken(0,yytext(),yyline,yychar,yychar+1)); }
":" { return (new Yytoken(1,yytext(),yyline,yychar,yychar+1)); }
";" { return (new Yytoken(2,yytext(),yyline,yychar,yychar+1)); }
"(" { return (new Yytoken(3,yytext(),yyline,yychar,yychar+1)); }
")" { return (new Yytoken(4,yytext(),yyline,yychar,yychar+1)); }
"[" { return (new Yytoken(5,yytext(),yyline,yychar,yychar+1)); }
"]" { return (new Yytoken(6,yytext(),yyline,yychar,yychar+1)); }
"{" { return (new Yytoken(7,yytext(),yyline,yychar,yychar+1)); }
"}" { return (new Yytoken(8,yytext(),yyline,yychar,yychar+1)); }
"." { return (new Yytoken(9,yytext(),yyline,yychar,yychar+1)); }
"+" { return (new Yytoken(10,yytext(),yyline,yychar,yychar+1)); }
"-" { return (new Yytoken(11,yytext(),yyline,yychar,yychar+1)); }
"*" { return (new Yytoken(12,yytext(),yyline,yychar,yychar+1)); }
"/" { return (new Yytoken(13,yytext(),yyline,yychar,yychar+1)); }
"=" { return (new Yytoken(14,yytext(),yyline,yychar,yychar+1)); }
"<>" { return (new Yytoken(15,yytext(),yyline,yychar,yychar+2)); }
"<" { return (new Yytoken(16,yytext(),yyline,yychar,yychar+1)); }
"<=" { return (new Yytoken(17,yytext(),yyline,yychar,yychar+2)); }
">" { return (new Yytoken(18,yytext(),yyline,yychar,yychar+1)); }
">=" { return (new Yytoken(19,yytext(),yyline,yychar,yychar+2)); }
"&" { return (new Yytoken(20,yytext(),yyline,yychar,yychar+1)); }
"|" { return (new Yytoken(21,yytext(),yyline,yychar,yychar+1)); }
":=" { return (new Yytoken(22,yytext(),yyline,yychar,yychar+2)); }
{NONNEWLINE_WHITE_SPACE_CHAR}+ { }
"/*" { yybegin(COMMENT); comment_count++; }
\"{STRING_TEXT}\" {
String str = yytext().substring(1,yylength()-1);
return (new Yytoken(40,str,yyline,yychar,yychar+yylength()));
}
\"{STRING_TEXT} {
String str = yytext().substring(1,yytext().length());
Utility.error(Utility.E_UNCLOSEDSTR);
return (new Yytoken(41,str,yyline,yychar,yychar + str.length()));
}
{DIGIT}+ { return (new Yytoken(42,yytext(),yyline,yychar,yychar+yylength())); }
{Ident} { return (new Yytoken(43,yytext(),yyline,yychar,yychar+yylength())); }
}
<COMMENT> {
"/*" { comment_count++; }
"*/" { if (--comment_count == 0) yybegin(YYINITIAL); }
{COMMENT_TEXT} { }
}
{NEWLINE} { }
. {
System.out.println("Illegal character: <" + yytext() + ">");
Utility.error(Utility.E_UNMATCHED);
}
tools/lexer/jflex-1.4/examples/simple/test.txt
-19
View File
@@ -1,19 +0,0 @@
class Utility {
private static final String errorMsg[] = {
"Error: Unmatched end-of-comment punctuation.",
"Error: Unmatched start-of-comment punctuation.",
"Error: Unclosed string.",
"Error: Illegal character."
};
public static final int E_ENDCOMMENT = 0;
public static final int E_STARTCOMMENT = 1;
public static final int E_UNCLOSEDSTR = 2;
public static final int E_UNMATCHED = 3;
public static void error(int code) {
System.out.println(errorMsg[code]);
}
}
tools/lexer/jflex-1.4/examples/standalone/Makefile
-13
View File
@@ -1,13 +0,0 @@
all: run
run: Subst.class
java Subst sample.inp
Subst.class: Subst.java
javac Subst.java
Subst.java: standalone.flex
jflex standalone.flex
clean:
rm -rf *.class *~ *.java
tools/lexer/jflex-1.4/examples/standalone/README
-20
View File
@@ -1,20 +0,0 @@
This is a small example of a standalone text substitution scanner.
It reads a name after the keyword name and replaces all occurences
of "hello" by "hello <name> !".
sample.inp
contains a sample input for the scanner
standalone.flex
the lexical specification for the scanner
build.xml
ant build file
to run the example use ant or type:
jflex standalone.flex
javac Subst.java
java Subst sample.inp
tools/lexer/jflex-1.4/examples/standalone/build.xml
-23
View File
@@ -1,23 +0,0 @@
<project name="standalone" default="run">
<taskdef classname="JFlex.anttask.JFlexTask" name="jflex" />
<target name="run" depends="compile">
<java classname="Subst"> <arg line="sample.inp"/> </java>
</target>
<target name="compile" depends="jflex">
<javac srcdir="." destdir="."/>
</target>
<target name="jflex"> <jflex file="standalone.flex"/> </target>
<target name="clean">
<delete file="Subst.java"/>
<delete>
<fileset dir="." includes="**/*~"/>
<fileset dir="." includes="**/*.class"/>
</delete>
</target>
</project>
tools/lexer/jflex-1.4/examples/standalone/sample.inp
-8
View File
@@ -1,8 +0,0 @@
name someone
Hello
This is a sample input file for the
standalone example scanner.
Have a nice day!
tools/lexer/jflex-1.4/examples/standalone/standalone.flex
-44
View File
@@ -1,44 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright (C) 1998-2004 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/**
This is a small example of a standalone text substitution scanner
It reads a name after the keyword name and substitutes all occurences
of "hello" with "hello <name>!". There is a sample input file
"sample.inp" provided in this directory
*/
%%
%public
%class Subst
%standalone
%unicode
%{
String name;
%}
%%
"name " [a-zA-Z]+ { name = yytext().substring(5); }
[Hh] "ello" { System.out.print(yytext()+" "+name+"!"); }
tools/lexer/jflex-1.4/jflex-1.4.3-unicode-character-category.patch
-7211
View File
File diff suppressed because it is too large Load Diff
tools/lexer/jflex-1.4/lib/JFlex.jar
BIN
View File
Binary file not shown.
tools/lexer/jflex-1.4/lib/jflex-mode.el
-114
View File
@@ -1,114 +0,0 @@
; -*- Mode: Emacs-Lisp; -*-
;;; jflex-mode
;;; author: Gerwin Klein <lsf@jflex.de>
;;; $Revision$, $Date$
(require 'derived)
(require 'font-lock)
(define-derived-mode jflex-mode java-mode "JFlex"
"Major mode for editing JFlex files"
;; set the indentation
(setq c-basic-offset 2)
(c-set-offset 'knr-argdecl-intro 0)
(c-set-offset 'topmost-intro-cont 0)
;; remove auto and hungry anything
(c-toggle-auto-hungry-state -1)
(c-toggle-auto-state -1)
(c-toggle-hungry-state -1)
(use-local-map jflex-mode-map)
;; get rid of that damn electric-brace
(define-key jflex-mode-map "{" 'self-insert-command)
(define-key jflex-mode-map "}" 'self-insert-command)
(define-key jflex-mode-map [tab] 'jflex-indent-command)
)
(defalias 'jflex-indent-command 'c-indent-command)
(defconst jflex-font-lock-keywords
(append
'(
("^%%" . font-lock-reference-face)
"^%{"
"^%init{"
"^%initthrow{"
"^%eof{"
"^%eofthrow{"
"^%yylexthrow{"
"^%eofval{"
"^%}"
"^%init}"
"^%initthrow}"
"^%eof}"
"^%eofthrow}"
"^%yylexthrow}"
"^%eofval}"
"^%standalone"
"^%scanerror"
"^%switch"
"^%states" ; fixme: state identifiers
"^%state"
"^%s"
"^%xstates"
"^%xstate"
"^%x"
"^%char"
"^%line"
"^%column"
"^%byaccj"
"^%cupsym"
"^%cupdebug"
"^%cup"
"^%eofclose"
"^%class"
"^%function"
"^%type"
"^%integer"
"^%intwrap"
"^%int"
"^%yyeof"
"^%notunix"
"^%7bit"
"^%full"
"^%8bit"
"^%unicode"
"^%16bit"
"^%caseless"
"^%ignorecase"
"^%implements"
"^%extends"
"^%public"
"^%apiprivate"
"^%final"
"^%abstract"
"^%debug"
"^%table"
"^%pack"
"^%include"
"^%buffer"
"^%initthrow"
"^%eofthrow"
"^%yylexthrow"
"^%throws"
("%[%{}0-9a-zA-Z]+" . font-lock-warning-face) ; errors
("{[ \t]*[a-zA-Z][0-9a-zA-Z_]+[ \t]*}" . font-lock-variable-name-face) ; macro uses
"<<EOF>>" ; special <<EOF>> symbol
("<[ \t]*[a-zA-Z][0-9a-zA-Z_]+[ \t]*\\(,[ \t]*[a-zA-Z][0-9a-zA-Z_]+[ \t]*\\)*>" . font-lock-type-face) ; lex state list
)
java-font-lock-keywords-2)
"JFlex keywords for font-lock mode")
(put 'jflex-mode 'font-lock-defaults
'(jflex-font-lock-keywords
nil nil ((?_ . "w")) beginning-of-defun))
(provide 'jflex-mode)
tools/lexer/jflex-1.4/lib/jflex.vim
-149
View File
@@ -1,149 +0,0 @@
" Vim syntax file
" Language: JFlex
" Maintainer: Gerwin Klein <lsf@jflex.de>
" Last Change: $Revision$, $Date$
" Thanks to Michael Brailsford for help and suggestions
" Quit when a syntax file was already loaded {{{
if exists("b:current_syntax")
finish
endif
"}}}
" Include java syntax {{{
if version >= 600
runtime! syntax/java.vim
unlet b:current_syntax
else
so $VIMRUNTIME/syntax/java.vim
endif
"}}}
syn cluster jflexOptions contains=jflexOption,jflexCodeInclude,jflexComment,jflexMacroIdent,jflexMacroRegExp,jflexOptionError
syn cluster jflexRules contains=jflexRule,jflexComment,jflexActionCode,jflexRuleStates,jflexRegExp
" java code section
syn region jflexStart start="/\*\|//\|import\|package\|class"me=s end="^%%"me=e-2 contains=@javaTop nextgroup=jflexOptionReg
" %%
" options
syn region jflexOptionReg matchgroup=jflexSectionSep start="^%%" end="^%%"me=e-2 contains=@jflexOptions nextgroup=jflexRulesReg
syn match jflexOptionError "%\i*" contained
syn match jflexOption "^\(%s\|%x\)" contained
syn match jflexOption "^%state" contained
syn match jflexOption "^%states" contained
syn match jflexOption "^%xstate" contained
syn match jflexOption "^%xstates" contained
syn match jflexOption "^%char" contained
syn match jflexOption "^%line" contained
syn match jflexOption "^%column" contained
syn match jflexOption "^%byaccj" contained
syn match jflexOption "^%cup" contained
syn match jflexOption "^%cupsym" contained
syn match jflexOption "^%cupdebug" contained
syn match jflexOption "^%eofclose" contained
syn match jflexOption "^%class" contained
syn match jflexOption "^%function" contained
syn match jflexOption "^%type" contained
syn match jflexOption "^%integer" contained
syn match jflexOption "^%int" contained
syn match jflexOption "^%intwrap" contained
syn match jflexOption "^%yyeof" contained
syn match jflexOption "^%notunix" contained
syn match jflexOption "^%7bit" contained
syn match jflexOption "^%8bit" contained
syn match jflexOption "^%full" contained
syn match jflexOption "^%16bit" contained
syn match jflexOption "^%unicode" contained
syn match jflexOption "^%caseless" contained
syn match jflexOption "^%ignorecase" contained
syn match jflexOption "^%implements" contained
syn match jflexOption "^%extends" contained
syn match jflexOption "^%public" contained
syn match jflexOption "^%apiprivate" contained
syn match jflexOption "^%final" contained
syn match jflexOption "^%abstract" contained
syn match jflexOption "^%debug" contained
syn match jflexOption "^%standalone" contained
syn match jflexOption "^%switch" contained
syn match jflexOption "^%table" contained
syn match jflexOption "^%pack" contained
syn match jflexOption "^%include" contained
syn match jflexOption "^%buffer" contained
syn match jflexOption "^%initthrow" contained
syn match jflexOption "^%eofthrow" contained
syn match jflexOption "^%yylexthrow" contained
syn match jflexOption "^%throws" contained
syn match jflexOption "^%scannerror" contained
syn match jflexMacroIdent "\I\i*\s*="me=e-1 contained nextgroup=jflexMacroRegExp
syn region jflexMacroRegExp matchgroup=jflexOperator start="=" end="^\(%\|\I\|\i\|/\)"me=e-1 contains=NONE contained
syn region jflexCodeInclude matchgroup=jflexCodeIncludeMark start="^%{" end="^%}" contains=@javaTop contained
syn region jflexCodeInclude matchgroup=jflexCodeIncludeMark start="^%init{" end="^%init}" contains=@javaTop contained
syn region jflexCodeInclude matchgroup=jflexCodeIncludeMark start="^%initthrow{" end="^%initthrow}" contains=@javaTop contained
syn region jflexCodeInclude matchgroup=jflexCodeIncludeMark start="^%eof{" end="^%eof}" contains=@javaTop contained
syn region jflexCodeInclude matchgroup=jflexCodeIncludeMark start="^%eofthrow{" end="^%eofthrow}" contains=@javaTop contained
syn region jflexCodeInclude matchgroup=jflexCodeIncludeMark start="^%yylexthrow{" end="^%yylexthrow}" contains=@javaTop contained
syn region jflexCodeInclude matchgroup=jflexCodeIncludeMark start="^%eofval{" end="^%eofval}" contains=@javaTop contained
" rules (end pattern shouldn't occur, if it does anyway we just stay in jflexRulesReg)
syn region jflexRulesReg matchgroup=jflexSectionSep start="^%%" end="^%%"me=e-2 contains=@jflexRules
" at first everything but strings is a regexp
syn match jflexRegExp "\([^\" \t]\|\\\"\)\+" contained
" take out comments
syn match jflexComment "//.*" contained
syn region jflexComment start="/\*" end="\*/" contained contains=jflexComment
" lex states
syn match jflexRuleStates "<\s*\I\i*\(\s*,\s*\I\i*\)*\s*>" contained skipnl skipwhite nextgroup=jflexStateGroup
" action code (only after states braces and macro use)
syn region jflexActionCode matchgroup=Delimiter start="{" end="}" contained contains=@javaTop,jflexJavaBraces
" macro use
syn match jflexRegExp "{\s*\I\i*\s*}" contained
" state braces (only active after <state>)
syn region jflexStateGroup matchgroup=jflexRuleStates start="{$" start="{\s" end="}" contained contains=@jflexRules
" string
syn region jflexRegExp matchgroup=String start=+"+ skip=+\\\\\|\\"+ end=+"+ contained
" not to be confused with a state
syn match jflexRegExp "<<EOF>>" contained
" escape sequence
syn match jflexRegExp "\\." contained
" keep braces in actions balanced
syn region jflexJavaBraces start="{" end="}" contained contains=@javaTop,jflexJavaBraces
" syncing
syn sync clear
syn sync minlines=10
syn sync match jflexSync grouphere jflexOptionReg "^%[a-z]"
syn sync match jflexSync grouphere jflexRulesReg "^<"
" highlighting
hi link jflexOption Special
hi link jflexMacroIdent Ident
hi link jflexMacroRegExp Macro
hi link jflexOptionError Error
hi link jflexComment Comment
hi link jflexOperator Operator
hi link jflexRuleStates Special
hi link jflexRegExp Function
hi jflexSectionSep guifg=yellow ctermfg=yellow guibg=blue ctermbg=blue gui=bold cterm=bold
hi link jflexCodeIncludeMark jflexSectionSep
let b:current_syntax="jflex"
tools/lexer/jflex-1.4/src/JFlex/Action.java
-261
View File
@@ -1,261 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* JFlex 1.4.3 *
* Copyright (C) 1998-2009 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
package JFlex;
/**
* Encapsulates an action in the specification.
*
* It stores the Java code as String together with a priority (line number in the specification).
*
* @author Gerwin Klein
* @version $Revision: 1.4.3 $, $Date: 2009/12/21 15:58:48 $
*/
final public class Action {
/** A normal action */
public final static int NORMAL = 0;
/** Action of a lookahead expression r1/r2 with fixed length r1 */
public final static int FIXED_BASE = 1;
/** Action of a lookahead expression r1/r2 with fixed length r2 */
public final static int FIXED_LOOK = 2;
/** Action of a lookahead expression r1/r2 with a finite choice of
* fixed lengths in r2 */
public final static int FINITE_CHOICE = 3;
/** Action of a general lookahead expression */
public final static int GENERAL_LOOK = 4;
/** Action of the 2nd forward pass for lookahead */
public final static int FORWARD_ACTION = 5;
/** Action of the backward pass for lookahead */
public final static int BACKWARD_ACTION = 6;
/**
* The Java code this Action represents
*/
String content;
/**
* The priority (i.e. line number in the specification) of this Action.
*/
int priority;
/**
* Which kind of action this is.
* (normal, <code>a/b</code> with fixed length a, fixed length b, etc)
*/
private int kind = NORMAL;
/** The length of the lookahead (if fixed) */
private int len;
/** The entry state of the corresponding forward DFA (if general lookahead) */
private int entryState;
/**
* Creates a new Action object with specified content and line number.
*
* @param content java code
* @param priority line number
*/
public Action(String content, int priority) {
this.content = content.trim();
this.priority = priority;
}
/**
* Creates a new Action object of the specified kind. Only
* accepts FORWARD_ACTION or BACKWARD_ACTION.
*
* @param kind the kind of action
*
* @see #FORWARD_ACTION
* @see #BACKWARD_ACTION
*/
public Action(int kind) {
if (kind != FORWARD_ACTION && kind != BACKWARD_ACTION)
throw new GeneratorException();
this.content = "";
this.priority = Integer.MAX_VALUE;
this.kind = kind;
}
/**
* Compares the priority value of this Action with the specified action.
*
* @param other the other Action to compare this Action with.
*
* @return this Action if it has higher priority - the specified one, if not.
*/
public Action getHigherPriority(Action other) {
if (other == null) return this;
// the smaller the number the higher the priority
if (other.priority > this.priority)
return this;
else
return other;
}
/**
* Returns the String representation of this object.
*
* @return string representation of the action
*/
public String toString() {
return "Action (priority "+priority+", lookahead "+kind+") :" +
Out.NL+content; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
}
/**
* Returns <code>true</code> iff the parameter is an
* Action with the same content as this one.
*
* @param a the object to compare this Action with
* @return true if the action strings are equal
*/
public boolean isEquiv(Action a) {
return this == a ||
(this.content.equals(a.content) &&
this.kind == a.kind &&
this.len == a.len &&
this.entryState == a.entryState);
}
/**
* Calculate hash value.
*
* @return a hash value for this Action
*/
public int hashCode() {
return content.hashCode();
}
/**
* Test for equality to another object.
*
* This action equals another object if the other
* object is an equivalent action.
*
* @param o the other object.
*
* @see Action#isEquiv(Action)
*/
public boolean equals(Object o) {
if (o instanceof Action)
return isEquiv((Action) o);
else
return false;
}
/**
* Return true iff this is action belongs to a general lookahead rule.
*
* @return true if this actions belongs to a general lookahead rule.
*/
public boolean isGenLookAction() {
return kind == GENERAL_LOOK;
}
/**
* Return true if code for this is action should be emitted, false
* if it is a BACK/FORWARD lookahead action.
*
* @return true if code should be emitted for this action.
*/
public boolean isEmittable() {
return kind != BACKWARD_ACTION && kind != FORWARD_ACTION;
}
/**
* Return kind of lookahead.
*/
public int lookAhead() {
return kind;
}
/**
* Sets the lookahead kind and data for this action
*
* @param kind which kind of lookahead it is
* @param data the length for fixed length look aheads.
*
*/
public void setLookAction(int kind, int data) {
this.kind = kind;
this.len = data;
}
/**
* The length of the lookahead or base if this is a fixed length
* lookahead action.
*/
public int getLookLength() {
return len;
}
/**
* Return the corresponding entry state for the forward DFA (if this
* is a general lookahead expression)
*
* @return the forward DFA entry state (+1 is the backward DFA)
*/
public int getEntryState() {
return entryState;
}
/**
* Set the corresponding entry state for the forward DFA of this action
* (if this is a general lookahead expression)
*
* @param the entry state for the forward DFA of this action
*/
public void setEntryState(int entryState) {
this.entryState = entryState;
}
public Action copyChoice(int length) {
Action a = new Action(this.content, this.priority);
a.setLookAction(FINITE_CHOICE, length);
return a;
}
/**
* String representation of the lookahead kind of this action.
*
* @return the string representation
*/
public String lookString() {
switch (kind) {
case NORMAL: return "";
case BACKWARD_ACTION: return "LOOK_BACK";
case FIXED_BASE: return "FIXED_BASE";
case FIXED_LOOK: return "FIXED_LOOK";
case FINITE_CHOICE: return "FINITE_CHOICE";
case FORWARD_ACTION: return "LOOK_FORWARD";
case GENERAL_LOOK: return "LOOK_ACTION";
default: return "unknown lookahead type";
}
}
}
tools/lexer/jflex-1.4/src/JFlex/CharClassException.java
-49
View File
@@ -1,49 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* JFlex 1.4.3 *
* Copyright (C) 1998-2009 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
package JFlex;
/**
* This Exception is used in class CharClasses.
*
* @author Gerwin Klein
* @version $Revision: 1.4.3 $, $Date: 2009/12/21 15:58:48 $
*/
public class CharClassException extends RuntimeException {
/**
* Creates a new CharClassException without message
*/
public CharClassException() {
}
/**
* Creates a new CharClassException with the specified message
*
* @param message the error description presented to the user.
*/
public CharClassException(String message) {
super(message);
}
}
tools/lexer/jflex-1.4/src/JFlex/CharClassInterval.java
-73
View File
@@ -1,73 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* JFlex 1.4.3 *
* Copyright (C) 1998-2009 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
package JFlex;
/**
* Stores an interval of characters together with the character class
*
* A character belongs to an interval, if its Unicode value is greater than or equal
* to the Unicode value of <CODE>start</code> and smaller than or euqal to the Unicode
* value of <CODE>end</code>.
*
* All characters of the interval must belong to the same character class.
*
* @author Gerwin Klein
* @version $Revision: 1.4.3 $, $Date: 2009/12/21 15:58:48 $
*/
public class CharClassInterval {
/**
* The first character of the interval
*/
int start;
/**
* The last character of the interval
*/
int end;
/**
* The code of the class all characters of this interval belong to.
*/
int charClass;
/**
* Creates a new CharClassInterval from <CODE>start</code> to <CODE>end</code>
* that belongs to character class <CODE>charClass</code>.
*
* @param start The first character of the interval
* @param end The last character of the interval
* @param charClass The code of the class all characters of this interval belong to.
*/
public CharClassInterval(int start, int end, int charClass) {
this.start = start;
this.end = end;
this.charClass = charClass;
}
/**
* returns string representation of this class interval
*/
public String toString() {
return "["+start+"-"+end+"="+charClass+"]";
}
}
tools/lexer/jflex-1.4/src/JFlex/CharClasses.java
-388
View File
@@ -1,388 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* JFlex 1.4.3 *
* Copyright (C) 1998-2009 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
package JFlex;
import java.util.*;
/**
*
* @author Gerwin Klein
* @version $Revision: 1.4.3 $, $Date: 2009/12/21 15:58:48 $
*/
public class CharClasses {
/** debug flag (for char classes only) */
private static final boolean DEBUG = false;
/** the largest character that can be used in char classes */
public static final char maxChar = '\uFFFF';
/** the char classes */
private Vector /* of IntCharSet */ classes;
/** the largest character actually used in a specification */
private char maxCharUsed;
/**
* Constructs a new CharClass object that provides space for
* classes of characters from 0 to maxCharCode.
*
* Initially all characters are in class 0.
*
* @param maxCharCode the last character code to be
* considered. (127 for 7bit Lexers,
* 255 for 8bit Lexers and 0xFFFF
* for Unicode Lexers).
*/
public CharClasses(int maxCharCode) {
if (maxCharCode < 0 || maxCharCode > 0xFFFF)
throw new IllegalArgumentException();
maxCharUsed = (char) maxCharCode;
classes = new Vector();
classes.addElement(new IntCharSet(new Interval((char) 0, maxChar)));
}
/**
* Returns the greatest Unicode value of the current input character set.
*/
public char getMaxCharCode() {
return maxCharUsed;
}
/**
* Sets the largest Unicode value of the current input character set.
*
* @param charCode the largest character code, used for the scanner
* (i.e. %7bit, %8bit, %16bit etc.)
*/
public void setMaxCharCode(int charCode) {
if (charCode < 0 || charCode > 0xFFFF)
throw new IllegalArgumentException();
maxCharUsed = (char) charCode;
}
/**
* Returns the current number of character classes.
*/
public int getNumClasses() {
return classes.size();
}
/**
* Updates the current partition, so that the specified set of characters
* gets a new character class.
*
* Characters that are elements of <code>set</code> are not in the same
* equivalence class with characters that are not elements of <code>set</code>.
*
* @param set the set of characters to distinguish from the rest
* @param caseless if true upper/lower/title case are considered equivalent
*/
public void makeClass(IntCharSet set, boolean caseless) {
if (caseless) set = set.getCaseless();
if ( DEBUG ) {
Out.dump("makeClass("+set+")");
dump();
}
int oldSize = classes.size();
for (int i = 0; i < oldSize; i++) {
IntCharSet x = (IntCharSet) classes.elementAt(i);
if (x.equals(set)) return;
IntCharSet and = x.and(set);
if ( and.containsElements() ) {
if ( x.equals(and) ) {
set.sub(and);
continue;
}
else if ( set.equals(and) ) {
x.sub(and);
classes.addElement(and);
if (DEBUG) {
Out.dump("makeClass(..) finished");
dump();
}
return;
}
set.sub(and);
x.sub(and);
classes.addElement(and);
}
}
if (DEBUG) {
Out.dump("makeClass(..) finished");
dump();
}
}
/**
* Returns the code of the character class the specified character belongs to.
*/
public int getClassCode(char letter) {
int i = -1;
while (true) {
IntCharSet x = (IntCharSet) classes.elementAt(++i);
if ( x.contains(letter) ) return i;
}
}
/**
* Dump charclasses to the dump output stream
*/
public void dump() {
Out.dump(toString());
}
/**
* Return a string representation of one char class
*
* @param theClass the index of the class to
*/
public String toString(int theClass) {
return classes.elementAt(theClass).toString();
}
/**
* Return a string representation of the char classes
* stored in this class.
*
* Enumerates the classes by index.
*/
public String toString() {
StringBuffer result = new StringBuffer("CharClasses:");
result.append(Out.NL);
for (int i = 0; i < classes.size(); i++)
result.append("class "+i+":"+Out.NL+classes.elementAt(i)+Out.NL);
return result.toString();
}
/**
* Creates a new character class for the single character <code>singleChar</code>.
*
* @param caseless if true upper/lower/title case are considered equivalent
*/
public void makeClass(char singleChar, boolean caseless) {
makeClass(new IntCharSet(singleChar), caseless);
}
/**
* Creates a new character class for each character of the specified String.
*
* @param caseless if true upper/lower/title case are considered equivalent
*/
public void makeClass(String str, boolean caseless) {
for (int i = 0; i < str.length(); i++) makeClass(str.charAt(i), caseless);
}
/**
* Updates the current partition, so that the specified set of characters
* gets a new character class.
*
* Characters that are elements of the set <code>v</code> are not in the same
* equivalence class with characters that are not elements of the set <code>v</code>.
*
* @param v a Vector of Interval objects.
* This Vector represents a set of characters. The set of characters is
* the union of all intervals in the Vector.
*
* @param caseless if true upper/lower/title case are considered equivalent
*/
public void makeClass(Vector /* Interval */ v, boolean caseless) {
makeClass(new IntCharSet(v), caseless);
}
/**
* Updates the current partition, so that the set of all characters not contained in the specified
* set of characters gets a new character class.
*
* Characters that are elements of the set <code>v</code> are not in the same
* equivalence class with characters that are not elements of the set <code>v</code>.
*
* This method is equivalent to <code>makeClass(v)</code>
*
* @param v a Vector of Interval objects.
* This Vector represents a set of characters. The set of characters is
* the union of all intervals in the Vector.
*
* @param caseless if true upper/lower/title case are considered equivalent
*/
public void makeClassNot(Vector v, boolean caseless) {
makeClass(new IntCharSet(v), caseless);
}
/**
* Returns an array that contains the character class codes of all characters
* in the specified set of input characters.
*/
private int [] getClassCodes(IntCharSet set, boolean negate) {
if (DEBUG) {
Out.dump("getting class codes for "+set);
if (negate)
Out.dump("[negated]");
}
int size = classes.size();
// [fixme: optimize]
int temp [] = new int [size];
int length = 0;
for (int i = 0; i < size; i++) {
IntCharSet x = (IntCharSet) classes.elementAt(i);
if ( negate ) {
if ( !set.and(x).containsElements() ) {
temp[length++] = i;
if (DEBUG) Out.dump("code "+i);
}
}
else {
if ( set.and(x).containsElements() ) {
temp[length++] = i;
if (DEBUG) Out.dump("code "+i);
}
}
}
int result [] = new int [length];
System.arraycopy(temp, 0, result, 0, length);
return result;
}
/**
* Returns an array that contains the character class codes of all characters
* in the specified set of input characters.
*
* @param intervallVec a Vector of Intervals, the set of characters to get
* the class codes for
*
* @return an array with the class codes for intervallVec
*/
public int [] getClassCodes(Vector /* Interval */ intervallVec) {
return getClassCodes(new IntCharSet(intervallVec), false);
}
/**
* Returns an array that contains the character class codes of all characters
* that are <strong>not</strong> in the specified set of input characters.
*
* @param intervallVec a Vector of Intervals, the complement of the
* set of characters to get the class codes for
*
* @return an array with the class codes for the complement of intervallVec
*/
public int [] getNotClassCodes(Vector /* Interval */ intervallVec) {
return getClassCodes(new IntCharSet(intervallVec), true);
}
/**
* Check consistency of the stored classes [debug].
*
* all classes must be disjoint, checks if all characters
* have a class assigned.
*/
public void check() {
for (int i = 0; i < classes.size(); i++)
for (int j = i+1; j < classes.size(); j++) {
IntCharSet x = (IntCharSet) classes.elementAt(i);
IntCharSet y = (IntCharSet) classes.elementAt(j);
if ( x.and(y).containsElements() ) {
System.out.println("Error: non disjoint char classes "+i+" and "+j);
System.out.println("class "+i+": "+x);
System.out.println("class "+j+": "+y);
}
}
// check if each character has a classcode
// (= if getClassCode terminates)
for (char c = 0; c < maxChar; c++) {
getClassCode(c);
if (c % 100 == 0) System.out.print(".");
}
getClassCode(maxChar);
}
/**
* Returns an array of all CharClassIntervalls in this
* char class collection.
*
* The array is ordered by char code, i.e.
* <code>result[i+1].start = result[i].end+1</code>
*
* Each CharClassInterval contains the number of the
* char class it belongs to.
*/
public CharClassInterval [] getIntervals() {
int i, c;
int size = classes.size();
int numIntervalls = 0;
for (i = 0; i < size; i++)
numIntervalls+= ((IntCharSet) classes.elementAt(i)).numIntervalls();
CharClassInterval [] result = new CharClassInterval[numIntervalls];
i = 0;
c = 0;
while (i < numIntervalls) {
int code = getClassCode((char) c);
IntCharSet set = (IntCharSet) classes.elementAt(code);
Interval iv = set.getNext();
result[i++] = new CharClassInterval(iv.start, iv.end, code);
c = iv.end+1;
}
return result;
}
}
tools/lexer/jflex-1.4/src/JFlex/CharSet.java
-112
View File
@@ -1,112 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* JFlex 1.4.3 *
* Copyright (C) 1998-2009 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
package JFlex;
/**
*
* @author Gerwin Klein
* @version $Revision: 1.4.3 $, $Date: 2009/12/21 15:58:48 $
*/
public final class CharSet {
final static int BITS = 6; // the number of bits to shift (2^6 = 64)
final static int MOD = (1<<BITS)-1; // modulus
long bits[];
private int numElements;
public CharSet() {
bits = new long[1];
}
public CharSet(int initialSize, int character) {
bits = new long[(initialSize >> BITS)+1];
add(character);
}
public void add(int character) {
resize(character);
if ( (bits[character >> BITS] & (1L << (character & MOD))) == 0) numElements++;
bits[character >> BITS] |= (1L << (character & MOD));
}
private int nbits2size (int nbits) {
return ((nbits >> BITS) + 1);
}
private void resize(int nbits) {
int needed = nbits2size(nbits);
if (needed < bits.length) return;
long newbits[] = new long[Math.max(bits.length*2,needed)];
System.arraycopy(bits, 0, newbits, 0, bits.length);
bits = newbits;
}
public boolean isElement(int character) {
int index = character >> BITS;
if (index >= bits.length) return false;
return (bits[index] & (1L << (character & MOD))) != 0;
}
public CharSetEnumerator characters() {
return new CharSetEnumerator(this);
}
public boolean containsElements() {
return numElements > 0;
}
public int size() {
return numElements;
}
public String toString() {
CharSetEnumerator set = characters();
StringBuffer result = new StringBuffer("{");
if ( set.hasMoreElements() ) result.append(""+set.nextElement());
while ( set.hasMoreElements() ) {
int i = set.nextElement();
result.append( ", "+i);
}
result.append("}");
return result.toString();
}
}
tools/lexer/jflex-1.4/src/JFlex/CharSetEnumerator.java
-88
View File
@@ -1,88 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* JFlex 1.4.3 *
* Copyright (C) 1998-2009 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
package JFlex;
/**
* Enumerator for the elements of a CharSet.
*
* Does not implement java.util.Enumeration, but supports the same protocol.
*
* @author Gerwin Klein
* @version $Revision: 1.4.3 $, $Date: 2009/12/21 15:58:48 $
*/
final public class CharSetEnumerator {
private int index;
private int offset;
private long mask = 1;
private CharSet set;
public CharSetEnumerator(CharSet characters) {
set = characters;
while (index < set.bits.length && set.bits[index] == 0)
index++;
if (index >= set.bits.length) return;
while (offset <= CharSet.MOD && ((set.bits[index] & mask) == 0)) {
mask<<= 1;
offset++;
}
}
private void advance() {
do {
offset++;
mask<<= 1;
} while (offset <= CharSet.MOD && ((set.bits[index] & mask) == 0));
if (offset > CharSet.MOD) {
do
index++;
while (index < set.bits.length && set.bits[index] == 0);
if (index >= set.bits.length) return;
offset = 0;
mask = 1;
while (offset <= CharSet.MOD && ((set.bits[index] & mask) == 0)) {
mask<<= 1;
offset++;
}
}
}
public boolean hasMoreElements() {
return index < set.bits.length;
}
public int nextElement() {
int x = (index << CharSet.BITS) + offset;
advance();
return x;
}
}
tools/lexer/jflex-1.4/src/JFlex/CountEmitter.java
-115
View File
@@ -1,115 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* jflex 1.4 *
* Copyright (C) 1998-2009 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
package JFlex;
/**
* An emitter for an array encoded as count/value pairs in a string.
*
* @author Gerwin Klein
* @version $Revision: 1.4.3 $, $Date: 2009/12/21 15:58:48 $
*/
public class CountEmitter extends PackEmitter {
/** number of entries in expanded array */
private int numEntries;
/** translate all values by this amount */
private int translate = 0;
/**
* Create a count/value emitter for a specific field.
*
* @param name name of the generated array
*/
protected CountEmitter(String name) {
super(name);
}
/**
* Emits count/value unpacking code for the generated array.
*
* @see JFlex.PackEmitter#emitUnPack()
*/
public void emitUnpack() {
// close last string chunk:
println("\";");
nl();
println(" private static int [] zzUnpack"+name+"() {");
println(" int [] result = new int["+numEntries+"];");
println(" int offset = 0;");
for (int i = 0; i < chunks; i++) {
println(" offset = zzUnpack"+name+"("+constName()+"_PACKED_"+i+", offset, result);");
}
println(" return result;");
println(" }");
nl();
println(" private static int zzUnpack"+name+"(String packed, int offset, int [] result) {");
println(" int i = 0; /* index in packed string */");
println(" int j = offset; /* index in unpacked array */");
println(" int l = packed.length();");
println(" while (i < l) {");
println(" int count = packed.charAt(i++);");
println(" int value = packed.charAt(i++);");
if (translate == 1) {
println(" value--;");
}
else if (translate != 0) {
println(" value-= "+translate);
}
println(" do result[j++] = value; while (--count > 0);");
println(" }");
println(" return j;");
println(" }");
}
/**
* Translate all values by given amount.
*
* Use to move value interval from [0, 0xFFFF] to something different.
*
* @param i amount the value will be translated by.
* Example: <code>i = 1</code> allows values in [-1, 0xFFFE].
*/
public void setValTranslation(int i) {
this.translate = i;
}
/**
* Emit one count/value pair.
*
* Automatically translates value by the <code>translate</code> value.
*
* @param count
* @param value
*
* @see CountEmitter#setValTranslation(int)
*/
public void emit(int count, int value) {
numEntries+= count;
breaks();
emitUC(count);
emitUC(value+translate);
}
}
tools/lexer/jflex-1.4/src/JFlex/DFA.java
-929
View File
@@ -1,929 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* JFlex 1.4.3 *
* Copyright (C) 1998-2009 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
package JFlex;
import java.io.*;
import java.util.*;
/**
* DFA representation in JFlex.
* Contains minimization algorithm.
*
* @author Gerwin Klein
* @version $Revision: 1.4.3 $, $Date: 2009/12/21 15:58:48 $
*/
final public class DFA {
/**
* The initial number of states
*/
private static final int STATES = 500;
/**
* The code for "no target state" in the transition table.
*/
public static final int NO_TARGET = -1;
/**
* table[current_state][character] is the next state for <code>current_state</code>
* with input <code>character</code>, <code>NO_TARGET</code> if there is no transition for
* this input in <code>current_state</code>
*/
int [][] table;
/**
* <code>isFinal[state] == true</code> <=> the state <code>state</code> is
* a final state.
*/
boolean [] isFinal;
/**
* <code>action[state]</code> is the action that is to be carried out in
* state <code>state</code>, <code>null</code> if there is no action.
*/
Action [] action;
/**
* entryState[i] is the start-state of lexical state i or
* lookahead DFA i
*/
int entryState [];
/**
* The number of states in this DFA
*/
int numStates;
/**
* The current maximum number of input characters
*/
int numInput;
/**
* The number of lexical states (2*numLexStates <= entryState.length)
*/
int numLexStates;
/**
* all actions that are used in this DFA
*/
Hashtable usedActions = new Hashtable();
/** True iff this DFA contains general lookahead */
boolean lookaheadUsed;
public DFA(int numEntryStates, int numInp, int numLexStates) {
numInput = numInp;
int statesNeeded = Math.max(numEntryStates, STATES);
table = new int [statesNeeded] [numInput];
action = new Action [statesNeeded];
isFinal = new boolean [statesNeeded];
entryState = new int [numEntryStates];
numStates = 0;
this.numLexStates = numLexStates;
for (int i = 0; i < statesNeeded; i++) {
for (char j = 0; j < numInput; j++)
table [i][j] = NO_TARGET;
}
}
public void setEntryState(int eState, int trueState) {
entryState[eState] = trueState;
}
private void ensureStateCapacity(int newNumStates) {
int oldLength = isFinal.length;
if ( newNumStates < oldLength ) return;
int newLength = oldLength*2;
while ( newLength <= newNumStates ) newLength*= 2;
boolean [] newFinal = new boolean [newLength];
boolean [] newPushback = new boolean [newLength];
Action [] newAction = new Action [newLength];
int [] [] newTable = new int [newLength] [numInput];
System.arraycopy(isFinal,0,newFinal,0,numStates);
System.arraycopy(action,0,newAction,0,numStates);
System.arraycopy(table,0,newTable,0,oldLength);
int i,j;
for (i = oldLength; i < newLength; i++) {
for (j = 0; j < numInput; j++) {
newTable[i][j] = NO_TARGET;
}
}
isFinal = newFinal;
action = newAction;
table = newTable;
}
public void setAction(int state, Action stateAction) {
action[state] = stateAction;
if (stateAction != null) {
usedActions.put(stateAction,stateAction);
lookaheadUsed |= stateAction.isGenLookAction();
}
}
public void setFinal(int state, boolean isFinalState) {
isFinal[state] = isFinalState;
}
public void addTransition(int start, char input, int dest) {
int max = Math.max(start,dest)+1;
ensureStateCapacity(max);
if (max > numStates) numStates = max;
// Out.debug("Adding DFA transition ("+start+", "+(int)input+", "+dest+")");
table[start][input] = dest;
}
public String toString() {
StringBuffer result = new StringBuffer();
for (int i=0; i < numStates; i++) {
result.append("State ");
if ( isFinal[i] ) {
result.append("[FINAL");
String l = action[i].lookString();
if (!l.equals("")) {
result.append(", ");
result.append(l);
}
result.append("] ");
}
result.append(i+":"+Out.NL);
for (char j=0; j < numInput; j++) {
if ( table[i][j] >= 0 )
result.append(" with "+(int)j+" in "+table[i][j]+Out.NL);
}
}
return result.toString();
}
public void writeDot(File file) {
try {
PrintWriter writer = new PrintWriter(new FileWriter(file));
writer.println(dotFormat());
writer.close();
}
catch (IOException e) {
Out.error(ErrorMessages.FILE_WRITE, file);
throw new GeneratorException();
}
}
public String dotFormat() {
StringBuffer result = new StringBuffer();
result.append("digraph DFA {"+Out.NL);
result.append("rankdir = LR"+Out.NL);
for (int i=0; i < numStates; i++) {
if ( isFinal[i] ) {
result.append(i);
result.append(" [shape = doublecircle]");
result.append(Out.NL);
}
}
for (int i=0; i < numStates; i++) {
for (int input = 0; input < numInput; input++) {
if ( table[i][input] >= 0 ) {
result.append(i+" -> "+table[i][input]);
result.append(" [label=\"["+input+"]\"]"+Out.NL);
// result.append(" [label=\"["+classes.toString(input)+"]\"]\n");
}
}
}
result.append("}"+Out.NL);
return result.toString();
}
/**
* Check that all actions can actually be matched in this DFA.
*/
public void checkActions(LexScan scanner, LexParse parser) {
EOFActions eofActions = parser.getEOFActions();
Enumeration l = scanner.actions.elements();
while (l.hasMoreElements()) {
Action a = (Action) l.nextElement();
if ( !a.equals(usedActions.get(a)) && !eofActions.isEOFAction(a) ) {
Out.warning(scanner.file, ErrorMessages.NEVER_MATCH, a.priority-1, -1);
}
}
}
/**
* Implementation of Hopcroft's O(n log n) minimization algorithm, follows
* description by D. Gries.
*
* Time: O(n log n)
* Space: O(c n), size < 4*(5*c*n + 13*n + 3*c) byte
*/
public void minimize() {
Out.print(numStates+" states before minimization, ");
if (numStates == 0) {
Out.error(ErrorMessages.ZERO_STATES);
throw new GeneratorException();
}
if (Options.no_minimize) {
Out.println("minimization skipped.");
return;
}
// the algorithm needs the DFA to be total, so we add an error state 0,
// and translate the rest of the states by +1
final int n = numStates+1;
// block information:
// [0..n-1] stores which block a state belongs to,
// [n..2*n-1] stores how many elements each block has
int [] block = new int[2*n];
// implements a doubly linked list of states (these are the actual blocks)
int [] b_forward = new int[2*n];
int [] b_backward = new int[2*n];
// the last of the blocks currently in use (in [n..2*n-1])
// (end of list marker, points to the last used block)
int lastBlock = n; // at first we start with one empty block
final int b0 = n; // the first block
// the circular doubly linked list L of pairs (B_i, c)
// (B_i, c) in L iff l_forward[(B_i-n)*numInput+c] > 0 // numeric value of block 0 = n!
int [] l_forward = new int[n*numInput+1];
int [] l_backward = new int[n*numInput+1];
int anchorL = n*numInput; // list anchor
// inverse of the transition table
// if t = inv_delta[s][c] then { inv_delta_set[t], inv_delta_set[t+1], .. inv_delta_set[k] }
// is the set of states, with inv_delta_set[k] = -1 and inv_delta_set[j] >= 0 for t <= j < k
int [] [] inv_delta = new int[n][numInput];
int [] inv_delta_set = new int [2*n*numInput];
// twin stores two things:
// twin[0]..twin[numSplit-1] is the list of blocks that have been split
// twin[B_i] is the twin of block B_i
int [] twin = new int[2*n];
int numSplit;
// SD[B_i] is the the number of states s in B_i with delta(s,a) in B_j
// if SD[B_i] == block[B_i], there is no need to split
int [] SD = new int[2*n]; // [only SD[n..2*n-1] is used]
// for fixed (B_j,a), the D[0]..D[numD-1] are the inv_delta(B_j,a)
int [] D = new int[n];
int numD;
// initialize inverse of transition table
int lastDelta = 0;
int [] inv_lists = new int[n]; // holds a set of lists of states
int [] inv_list_last = new int[n]; // the last element
for (int c = 0; c < numInput; c++) {
// clear "head" and "last element" pointers
for (int s = 0; s < n; s++) {
inv_list_last[s] = -1;
inv_delta[s][c] = -1;
}
// the error state has a transition for each character into itself
inv_delta[0][c] = 0;
inv_list_last[0] = 0;
// accumulate states of inverse delta into lists (inv_delta serves as head of list)
for (int s = 1; s < n; s++) {
int t = table[s-1][c]+1;
if (inv_list_last[t] == -1) { // if there are no elements in the list yet
inv_delta[t][c] = s; // mark t as first and last element
inv_list_last[t] = s;
}
else {
inv_lists[inv_list_last[t]] = s; // link t into chain
inv_list_last[t] = s; // and mark as last element
}
}
// now move them to inv_delta_set in sequential order,
// and update inv_delta accordingly
for (int s = 0; s < n; s++) {
int i = inv_delta[s][c]; inv_delta[s][c] = lastDelta;
int j = inv_list_last[s];
boolean go_on = (i != -1);
while (go_on) {
go_on = (i != j);
inv_delta_set[lastDelta++] = i;
i = inv_lists[i];
}
inv_delta_set[lastDelta++] = -1;
}
} // of initialize inv_delta
// printInvDelta(inv_delta, inv_delta_set);
// initialize blocks
// make b0 = {0} where 0 = the additional error state
b_forward[b0] = 0;
b_backward[b0] = 0;
b_forward[0] = b0;
b_backward[0] = b0;
block[0] = b0;
block[b0] = 1;
for (int s = 1; s < n; s++) {
// System.out.println("Checking state ["+(s-1)+"]");
// search the blocks if it fits in somewhere
// (fit in = same pushback behavior, same finalness, same lookahead behavior, same action)
int b = b0+1; // no state can be equivalent to the error state
boolean found = false;
while (!found && b <= lastBlock) {
// get some state out of the current block
int t = b_forward[b];
// System.out.println(" picking state ["+(t-1)+"]");
// check, if s could be equivalent with t
if (isFinal[s-1]) {
found = isFinal[t-1] && action[s-1].isEquiv(action[t-1]);
}
else {
found = !isFinal[t-1];
}
if (found) { // found -> add state s to block b
// System.out.println("Found! Adding to block "+(b-b0));
// update block information
block[s] = b;
block[b]++;
// chain in the new element
int last = b_backward[b];
b_forward[last] = s;
b_forward[s] = b;
b_backward[b] = s;
b_backward[s] = last;
}
b++;
}
if (!found) { // fits in nowhere -> create new block
// System.out.println("not found, lastBlock = "+lastBlock);
// update block information
block[s] = b;
block[b]++;
// chain in the new element
b_forward[b] = s;
b_forward[s] = b;
b_backward[b] = s;
b_backward[s] = b;
lastBlock++;
}
} // of initialize blocks
// printBlocks(block,b_forward,b_backward,lastBlock);
// initialize worklist L
// first, find the largest block B_max, then, all other (B_i,c) go into the list
int B_max = b0;
int B_i;
for (B_i = b0+1; B_i <= lastBlock; B_i++)
if (block[B_max] < block[B_i]) B_max = B_i;
// L = empty
l_forward[anchorL] = anchorL;
l_backward[anchorL] = anchorL;
// set up the first list element
if (B_max == b0) B_i = b0+1; else B_i = b0; // there must be at least two blocks
int index = (B_i-b0)*numInput; // (B_i, 0)
while (index < (B_i+1-b0)*numInput) {
int last = l_backward[anchorL];
l_forward[last] = index;
l_forward[index] = anchorL;
l_backward[index] = last;
l_backward[anchorL] = index;
index++;
}
// now do the rest of L
while (B_i <= lastBlock) {
if (B_i != B_max) {
index = (B_i-b0)*numInput;
while (index < (B_i+1-b0)*numInput) {
int last = l_backward[anchorL];
l_forward[last] = index;
l_forward[index] = anchorL;
l_backward[index] = last;
l_backward[anchorL] = index;
index++;
}
}
B_i++;
}
// end of setup L
// start of "real" algorithm
// int step = 0;
// System.out.println("max_steps = "+(n*numInput));
// while L not empty
while (l_forward[anchorL] != anchorL) {
// System.out.println("step : "+(step++));
// printL(l_forward, l_backward, anchorL);
// pick and delete (B_j, a) in L:
// pick
int B_j_a = l_forward[anchorL];
// delete
l_forward[anchorL] = l_forward[B_j_a];
l_backward[l_forward[anchorL]] = anchorL;
l_forward[B_j_a] = 0;
// take B_j_a = (B_j-b0)*numInput+c apart into (B_j, a)
int B_j = b0 + B_j_a / numInput;
int a = B_j_a % numInput;
// printL(l_forward, l_backward, anchorL);
// System.out.println("picked ("+B_j+","+a+")");
// printL(l_forward, l_backward, anchorL);
// determine splittings of all blocks wrt (B_j, a)
// i.e. D = inv_delta(B_j,a)
numD = 0;
int s = b_forward[B_j];
while (s != B_j) {
// System.out.println("splitting wrt. state "+s);
int t = inv_delta[s][a];
// System.out.println("inv_delta chunk "+t);
while (inv_delta_set[t] != -1) {
// System.out.println("D+= state "+inv_delta_set[t]);
D[numD++] = inv_delta_set[t++];
}
s = b_forward[s];
}
// clear the twin list
numSplit = 0;
// System.out.println("splitting blocks according to D");
// clear SD and twins (only those B_i that occur in D)
for (int indexD = 0; indexD < numD; indexD++) { // for each s in D
s = D[indexD];
B_i = block[s];
SD[B_i] = -1;
twin[B_i] = 0;
}
// count how many states of each B_i occurring in D go with a into B_j
// Actually we only check, if *all* t in B_i go with a into B_j.
// In this case SD[B_i] == block[B_i] will hold.
for (int indexD = 0; indexD < numD; indexD++) { // for each s in D
s = D[indexD];
B_i = block[s];
// only count, if we haven't checked this block already
if (SD[B_i] < 0) {
SD[B_i] = 0;
int t = b_forward[B_i];
while (t != B_i && (t != 0 || block[0] == B_j) &&
(t == 0 || block[table[t-1][a]+1] == B_j)) {
SD[B_i]++;
t = b_forward[t];
}
}
}
// split each block according to D
for (int indexD = 0; indexD < numD; indexD++) { // for each s in D
s = D[indexD];
B_i = block[s];
// System.out.println("checking if block "+(B_i-b0)+" must be split because of state "+s);
if (SD[B_i] != block[B_i]) {
// System.out.println("state "+(s-1)+" must be moved");
int B_k = twin[B_i];
if (B_k == 0) {
// no twin for B_i yet -> generate new block B_k, make it B_i's twin
B_k = ++lastBlock;
// System.out.println("creating block "+(B_k-n));
// printBlocks(block,b_forward,b_backward,lastBlock-1);
b_forward[B_k] = B_k;
b_backward[B_k] = B_k;
twin[B_i] = B_k;
// mark B_i as split
twin[numSplit++] = B_i;
}
// move s from B_i to B_k
// remove s from B_i
b_forward[b_backward[s]] = b_forward[s];
b_backward[b_forward[s]] = b_backward[s];
// add s to B_k
int last = b_backward[B_k];
b_forward[last] = s;
b_forward[s] = B_k;
b_backward[s] = last;
b_backward[B_k] = s;
block[s] = B_k;
block[B_k]++;
block[B_i]--;
SD[B_i]--; // there is now one state less in B_i that goes with a into B_j
// printBlocks(block, b_forward, b_backward, lastBlock);
// System.out.println("finished move");
}
} // of block splitting
// printBlocks(block, b_forward, b_backward, lastBlock);
// System.out.println("updating L");
// update L
for (int indexTwin = 0; indexTwin < numSplit; indexTwin++) {
B_i = twin[indexTwin];
int B_k = twin[B_i];
for (int c = 0; c < numInput; c++) {
int B_i_c = (B_i-b0)*numInput+c;
int B_k_c = (B_k-b0)*numInput+c;
if (l_forward[B_i_c] > 0) {
// (B_i,c) already in L --> put (B_k,c) in L
int last = l_backward[anchorL];
l_backward[anchorL] = B_k_c;
l_forward[last] = B_k_c;
l_backward[B_k_c] = last;
l_forward[B_k_c] = anchorL;
}
else {
// put the smaller block in L
if (block[B_i] <= block[B_k]) {
int last = l_backward[anchorL];
l_backward[anchorL] = B_i_c;
l_forward[last] = B_i_c;
l_backward[B_i_c] = last;
l_forward[B_i_c] = anchorL;
}
else {
int last = l_backward[anchorL];
l_backward[anchorL] = B_k_c;
l_forward[last] = B_k_c;
l_backward[B_k_c] = last;
l_forward[B_k_c] = anchorL;
}
}
}
}
}
// System.out.println("Result");
// printBlocks(block,b_forward,b_backward,lastBlock);
/*
System.out.println("Old minimization:");
boolean [] [] equiv = old_minimize();
boolean error = false;
for (int i = 1; i < equiv.length; i++) {
for (int j = 0; j < equiv[i].length; j++) {
if (equiv[i][j] != (block[i+1] == block[j+1])) {
System.out.println("error: equiv["+i+"]["+j+"] = "+equiv[i][j]+
", block["+i+"] = "+block[i+1]+", block["+j+"] = "+block[j]);
error = true;
}
}
}
if (error) System.exit(1);
System.out.println("check");
*/
// transform the transition table
// trans[i] is the state j that will replace state i, i.e.
// states i and j are equivalent
int trans [] = new int [numStates];
// kill[i] is true iff state i is redundant and can be removed
boolean kill [] = new boolean [numStates];
// move[i] is the amount line i has to be moved in the transition table
// (because states j < i have been removed)
int move [] = new int [numStates];
// fill arrays trans[] and kill[] (in O(n))
for (int b = b0+1; b <= lastBlock; b++) { // b0 contains the error state
// get the state with smallest value in current block
int s = b_forward[b];
int min_s = s; // there are no empty blocks!
for (; s != b; s = b_forward[s])
if (min_s > s) min_s = s;
// now fill trans[] and kill[] for this block
// (and translate states back to partial DFA)
min_s--;
for (s = b_forward[b]-1; s != b-1; s = b_forward[s+1]-1) {
trans[s] = min_s;
kill[s] = s != min_s;
}
}
// fill array move[] (in O(n))
int amount = 0;
for (int i = 0; i < numStates; i++) {
if ( kill[i] )
amount++;
else
move[i] = amount;
}
int i,j;
// j is the index in the new transition table
// the transition table is transformed in place (in O(c n))
for (i = 0, j = 0; i < numStates; i++) {
// we only copy lines that have not been removed
if ( !kill[i] ) {
// translate the target states
for (int c = 0; c < numInput; c++) {
if ( table[i][c] >= 0 ) {
table[j][c] = trans[ table[i][c] ];
table[j][c]-= move[ table[j][c] ];
}
else {
table[j][c] = table[i][c];
}
}
isFinal[j] = isFinal[i];
action[j] = action[i];
j++;
}
}
numStates = j;
// translate lexical states
for (i = 0; i < entryState.length; i++) {
entryState[i] = trans[ entryState[i] ];
entryState[i]-= move[ entryState[i] ];
}
Out.println(numStates+" states in minimized DFA");
}
public String toString(int [] a) {
String r = "{";
int i;
for (i = 0; i < a.length-1; i++) r += a[i]+",";
return r+a[i]+"}";
}
public void printBlocks(int [] b, int [] b_f, int [] b_b, int last) {
Out.dump("block : "+toString(b));
Out.dump("b_forward : "+toString(b_f));
Out.dump("b_backward: "+toString(b_b));
Out.dump("lastBlock : "+last);
final int n = numStates+1;
for (int i = n; i <= last; i ++) {
Out.dump("Block "+(i-n)+" (size "+b[i]+"):");
String line = "{";
int s = b_f[i];
while (s != i) {
line = line+(s-1);
int t = s;
s = b_f[s];
if (s != i) {
line = line+",";
if (b[s] != i) Out.dump("consistency error for state "+(s-1)+" (block "+b[s]+")");
}
if (b_b[s] != t) Out.dump("consistency error for b_back in state "+(s-1)+" (back = "+b_b[s]+", should be = "+t+")");
}
Out.dump(line+"}");
}
}
public void printL(int [] l_f, int [] l_b, int anchor) {
String l = "L = {";
int bc = l_f[anchor];
while (bc != anchor) {
int b = bc / numInput;
int c = bc % numInput;
l+= "("+b+","+c+")";
int old_bc = bc;
bc = l_f[bc];
if (bc != anchor) l+= ",";
if (l_b[bc] != old_bc) Out.dump("consistency error for ("+b+","+c+")");
}
Out.dump(l+"}");
}
/**
* Much simpler, but slower and less memory efficient minimization algorithm.
*
* @return the equivalence relation on states.
*/
public boolean [] [] old_minimize() {
int i,j;
char c;
Out.print(numStates+" states before minimization, ");
if (numStates == 0) {
Out.error(ErrorMessages.ZERO_STATES);
throw new GeneratorException();
}
if (Options.no_minimize) {
Out.println("minimization skipped.");
return null;
}
// equiv[i][j] == true <=> state i and state j are equivalent
boolean [] [] equiv = new boolean [numStates] [];
// list[i][j] contains all pairs of states that have to be marked "not equivalent"
// if states i and j are recognized to be not equivalent
StatePairList [] [] list = new StatePairList [numStates] [];
// construct a triangular matrix equiv[i][j] with j < i
// and mark pairs (final state, not final state) as not equivalent
for (i = 1; i < numStates; i++) {
list[i] = new StatePairList[i];
equiv[i] = new boolean [i];
for (j = 0; j < i; j++) {
// i and j are equivalent, iff :
// i and j are both final and their actions are equivalent and have same pushback behaviour or
// i and j are both not final
if ( isFinal[i] && isFinal[j] )
equiv[i][j] = action[i].isEquiv(action[j]);
else
equiv[i][j] = !isFinal[j] && !isFinal[i];
}
}
for (i = 1; i < numStates; i++) {
Out.debug("Testing state "+i);
for (j = 0; j < i; j++) {
if ( equiv[i][j] ) {
for (c = 0; c < numInput; c++) {
if (equiv[i][j]) {
int p = table[i][c];
int q = table[j][c];
if (p < q) {
int t = p;
p = q;
q = t;
}
if ( p >= 0 || q >= 0 ) {
// Out.debug("Testing input '"+c+"' for ("+i+","+j+")");
// Out.debug("Target states are ("+p+","+q+")");
if ( p!=q && (p == -1 || q == -1 || !equiv[p][q]) ) {
equiv[i][j] = false;
if (list[i][j] != null) list[i][j].markAll(list,equiv);
}
// printTable(equiv);
} // if (p >= 0) ..
} // if (equiv[i][j]
} // for (char c = 0; c < numInput ..
// if i and j are still marked equivalent..
if ( equiv[i][j] ) {
// Out.debug("("+i+","+j+") are still marked equivalent");
for (c = 0; c < numInput; c++) {
int p = table[i][c];
int q = table[j][c];
if (p < q) {
int t = p;
p = q;
q = t;
}
if (p != q && p >= 0 && q >= 0) {
if ( list[p][q] == null ) {
list[p][q] = new StatePairList();
}
list[p][q].addPair(i,j);
}
}
}
else {
// Out.debug("("+i+","+j+") are not equivalent");
}
} // of first if (equiv[i][j])
} // of for j
} // of for i
// }
// printTable(equiv);
return equiv;
}
public void printInvDelta(int [] [] inv_delta, int [] inv_delta_set) {
Out.dump("Inverse of transition table: ");
for (int s = 0; s < numStates+1; s++) {
Out.dump("State ["+(s-1)+"]");
for (int c = 0; c < numInput; c++) {
String line = "With <"+c+"> in {";
int t = inv_delta[s][c];
while (inv_delta_set[t] != -1) {
line += inv_delta_set[t++]-1;
if (inv_delta_set[t] != -1) line += ",";
}
if (inv_delta_set[inv_delta[s][c]] != -1)
Out.dump(line+"}");
}
}
}
public void printTable(boolean [] [] equiv) {
Out.dump("Equivalence table is : ");
for (int i = 1; i < numStates; i++) {
String line = i+" :";
for (int j = 0; j < i; j++) {
if (equiv[i][j])
line+= " E";
else
line+= " x";
}
Out.dump(line);
}
}
}
tools/lexer/jflex-1.4/src/JFlex/EOFActions.java
-94
View File
@@ -1,94 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* JFlex 1.4.3 *
* Copyright (C) 1998-2009 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
package JFlex;
import java.util.*;
/**
* A simple table to store EOF actions for each lexical state.
*
* @author Gerwin Klein
* @version $Revision: 1.4.3 $, $Date: 2009/12/21 15:58:48 $
*/
public class EOFActions {
/** maps lexical states to actions */
private Hashtable /* Integer -> Action */ actions = new Hashtable();
private Action defaultAction;
private int numLexStates;
public void setNumLexStates(int num) {
numLexStates = num;
}
public void add(Vector stateList, Action action) {
if (stateList != null && stateList.size() > 0) {
Enumeration states = stateList.elements();
while (states.hasMoreElements())
add( (Integer) states.nextElement(), action );
}
else {
defaultAction = action.getHigherPriority(defaultAction);
for (int i = 0; i < numLexStates; i++) {
Integer state = new Integer(i);
if ( actions.get(state) != null ) {
Action oldAction = (Action) actions.get(state);
actions.put(state, oldAction.getHigherPriority(action));
}
}
}
}
public void add(Integer state, Action action) {
if ( actions.get(state) == null )
actions.put(state, action);
else {
Action oldAction = (Action) actions.get(state);
actions.put(state, oldAction.getHigherPriority(action));
}
}
public boolean isEOFAction(Object a) {
if (a == defaultAction) return true;
Enumeration e = actions.elements();
while ( e.hasMoreElements() )
if (a == e.nextElement()) return true;
return false;
}
public Action getAction(int state) {
return (Action) actions.get(new Integer(state));
}
public Action getDefault() {
return defaultAction;
}
public int numActions() {
return actions.size();
}
}
tools/lexer/jflex-1.4/src/JFlex/Emitter.java
-1671
View File
File diff suppressed because it is too large Load Diff
tools/lexer/jflex-1.4/src/JFlex/ErrorMessages.java
-149
View File
@@ -1,149 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* JFlex 1.4.3 *
* Copyright (C) 1998-2009 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software); you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY); without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program); if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
package JFlex;
import java.text.MessageFormat;
import java.util.MissingResourceException;
import java.util.ResourceBundle;
/**
* Central class for all kinds of JFlex messages.
*
* [Is not yet used exclusively, but should]
*
* @author Gerwin Klein
* @version $Revision: 1.4.3 $, $Date: 2009/12/21 15:58:48 $
*/
public class ErrorMessages {
private String key;
/* not final static, because initializing here seems too early
* for OS/2 JDK 1.1.8. See bug 1065521.
*/
private static ResourceBundle resourceBundle = null;
private ErrorMessages(String key) {
this.key = key;
}
public static String get(ErrorMessages msg) {
if (resourceBundle == null) {
resourceBundle = ResourceBundle.getBundle("JFlex.Messages");
}
try {
return resourceBundle.getString(msg.key);
} catch (MissingResourceException e) {
return '!' + msg.key + '!';
}
}
public static String get(ErrorMessages msg, String data) {
Object [] args = { data };
return MessageFormat.format(get(msg),args);
}
public static String get(ErrorMessages msg, String data1, String data2) {
Object [] args = { data1, data2 };
return MessageFormat.format(get(msg),args);
}
public static String get(ErrorMessages msg, int data) {
Object [] args = { new Integer(data) };
return MessageFormat.format(get(msg),args);
}
// typesafe enumeration (generated, do not edit)
public static ErrorMessages UNTERMINATED_STR = new ErrorMessages("UNTERMINATED_STR");
public static ErrorMessages EOF_WO_ACTION = new ErrorMessages("EOF_WO_ACTION");
public static ErrorMessages EOF_SINGLERULE = new ErrorMessages("EOF_SINGLERULE");
public static ErrorMessages UNKNOWN_OPTION = new ErrorMessages("UNKNOWN_OPTION");
public static ErrorMessages UNEXPECTED_CHAR = new ErrorMessages("UNEXPECTED_CHAR");
public static ErrorMessages UNEXPECTED_NL = new ErrorMessages("UNEXPECTED_NL");
public static ErrorMessages LEXSTATE_UNDECL = new ErrorMessages("LEXSTATE_UNDECL");
public static ErrorMessages STATE_IDENT_EXP = new ErrorMessages("STATE_IDENT_EXP");
public static ErrorMessages REPEAT_ZERO = new ErrorMessages("REPEAT_ZERO");
public static ErrorMessages REPEAT_GREATER = new ErrorMessages("REPEAT_GREATER");
public static ErrorMessages REGEXP_EXPECTED = new ErrorMessages("REGEXP_EXPECTED");
public static ErrorMessages MACRO_UNDECL = new ErrorMessages("MACRO_UNDECL");
public static ErrorMessages CHARSET_2_SMALL = new ErrorMessages("CHARSET_2_SMALL");
public static ErrorMessages CS2SMALL_STRING = new ErrorMessages("CS2SMALL_STRING");
public static ErrorMessages CS2SMALL_CHAR = new ErrorMessages("CS2SMALL_CHAR");
public static ErrorMessages CHARCLASS_MACRO = new ErrorMessages("CHARCLASS_MACRO");
public static ErrorMessages UNKNOWN_SYNTAX = new ErrorMessages("UNKNOWN_SYNTAX");
public static ErrorMessages SYNTAX_ERROR = new ErrorMessages("SYNTAX_ERROR");
public static ErrorMessages NOT_AT_BOL = new ErrorMessages("NOT_AT_BOL");
public static ErrorMessages NO_MATCHING_BR = new ErrorMessages("NO_MATCHING_BR");
public static ErrorMessages EOF_IN_ACTION = new ErrorMessages("EOF_IN_ACTION");
public static ErrorMessages EOF_IN_COMMENT = new ErrorMessages("EOF_IN_COMMENT");
public static ErrorMessages EOF_IN_STRING = new ErrorMessages("EOF_IN_STRING");
public static ErrorMessages EOF_IN_MACROS = new ErrorMessages("EOF_IN_MACROS");
public static ErrorMessages EOF_IN_STATES = new ErrorMessages("EOF_IN_STATES");
public static ErrorMessages EOF_IN_REGEXP = new ErrorMessages("EOF_IN_REGEXP");
public static ErrorMessages UNEXPECTED_EOF = new ErrorMessages("UNEXPECTED_EOF");
public static ErrorMessages NO_LEX_SPEC = new ErrorMessages("NO_LEX_SPEC");
public static ErrorMessages NO_LAST_ACTION = new ErrorMessages("NO_LAST_ACTION");
public static ErrorMessages NO_DIRECTORY = new ErrorMessages("NO_DIRECTORY");
public static ErrorMessages NO_SKEL_FILE = new ErrorMessages("NO_SKEL_FILE");
public static ErrorMessages WRONG_SKELETON = new ErrorMessages("WRONG_SKELETON");
public static ErrorMessages OUT_OF_MEMORY = new ErrorMessages("OUT_OF_MEMORY");
public static ErrorMessages QUIL_INITTHROW = new ErrorMessages("QUIL_INITTHROW");
public static ErrorMessages QUIL_EOFTHROW = new ErrorMessages("QUIL_EOFTHROW");
public static ErrorMessages QUIL_YYLEXTHROW = new ErrorMessages("QUIL_YYLEXTHROW");
public static ErrorMessages ZERO_STATES = new ErrorMessages("ZERO_STATES");
public static ErrorMessages NO_BUFFER_SIZE = new ErrorMessages("NO_BUFFER_SIZE");
public static ErrorMessages NOT_READABLE = new ErrorMessages("NOT_READABLE");
public static ErrorMessages FILE_CYCLE = new ErrorMessages("FILE_CYCLE");
public static ErrorMessages FILE_WRITE = new ErrorMessages("FILE_WRITE");
public static ErrorMessages QUIL_SCANERROR = new ErrorMessages("QUIL_SCANERROR");
public static ErrorMessages NEVER_MATCH = new ErrorMessages("NEVER_MATCH");
public static ErrorMessages QUIL_THROW = new ErrorMessages("QUIL_THROW");
public static ErrorMessages EOL_IN_CHARCLASS = new ErrorMessages("EOL_IN_CHARCLASS");
public static ErrorMessages QUIL_CUPSYM = new ErrorMessages("QUIL_CUPSYM");
public static ErrorMessages CUPSYM_AFTER_CUP = new ErrorMessages("CUPSYM_AFTER_CUP");
public static ErrorMessages ALREADY_RUNNING = new ErrorMessages("ALREADY_RUNNING");
public static ErrorMessages CANNOT_READ_SKEL = new ErrorMessages("CANNOT_READ_SKEL");
public static ErrorMessages READING_SKEL = new ErrorMessages("READING_SKEL");
public static ErrorMessages SKEL_IO_ERROR = new ErrorMessages("SKEL_IO_ERROR");
public static ErrorMessages SKEL_IO_ERROR_DEFAULT = new ErrorMessages("SKEL_IO_ERROR_DEFAULT");
public static ErrorMessages READING = new ErrorMessages("READING");
public static ErrorMessages CANNOT_OPEN = new ErrorMessages("CANNOT_OPEN");
public static ErrorMessages NFA_IS = new ErrorMessages("NFA_IS");
public static ErrorMessages NFA_STATES = new ErrorMessages("NFA_STATES");
public static ErrorMessages DFA_TOOK = new ErrorMessages("DFA_TOOK");
public static ErrorMessages DFA_IS = new ErrorMessages("DFA_IS");
public static ErrorMessages MIN_TOOK = new ErrorMessages("MIN_TOOK");
public static ErrorMessages MIN_DFA_IS = new ErrorMessages("MIN_DFA_IS");
public static ErrorMessages WRITE_TOOK = new ErrorMessages("WRITE_TOOK");
public static ErrorMessages TOTAL_TIME = new ErrorMessages("TOTAL_TIME");
public static ErrorMessages IO_ERROR = new ErrorMessages("IO_ERROR");
public static ErrorMessages THIS_IS_JFLEX = new ErrorMessages("THIS_IS_JFLEX");
public static ErrorMessages UNKNOWN_COMMANDLINE = new ErrorMessages("UNKNOWN_COMMANDLINE");
public static ErrorMessages MACRO_CYCLE = new ErrorMessages("MACRO_CYCLE");
public static ErrorMessages MACRO_DEF_MISSING = new ErrorMessages("MACRO_DEF_MISSING");
public static ErrorMessages PARSING_TOOK = new ErrorMessages("PARSING_TOOK");
public static ErrorMessages NFA_TOOK = new ErrorMessages("NFA_TOOK");
public static ErrorMessages LOOKAHEAD_NEEDS_ACTION = new ErrorMessages("LOOKAHEAD_NEEDS_ACTION");
public static ErrorMessages EMPTY_MATCH = new ErrorMessages("EMPTY_MATCH");
public static ErrorMessages CTOR_ARG = new ErrorMessages("CTOR_ARG");
public static ErrorMessages CTOR_DEBUG = new ErrorMessages("CTOR_DEBUG");
public static ErrorMessages INT_AND_TYPE = new ErrorMessages("INT_AND_TYPE");
}
tools/lexer/jflex-1.4/src/JFlex/GeneratorException.java
-36
View File
@@ -1,36 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* JFlex 1.4.3 *
* Copyright (C) 1998-2009 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
package JFlex;
/**
* Thrown when code generation has to be aborted.
*
* @author Gerwin Klein
* @version $Revision: 1.4.3 $, $Date: 2009/12/21 15:58:48 $
*/
public class GeneratorException extends RuntimeException {
public GeneratorException() {
super("Generation aborted");
}
}
tools/lexer/jflex-1.4/src/JFlex/HiLowEmitter.java
-88
View File
@@ -1,88 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* jflex *
* Copyright (C) 1998-2009 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
package JFlex;
/**
* HiLowEmitter
*
* @author Gerwin Klein
* @version $Revision: 1.4.3 $, $Date: 2009/12/21 15:58:48 $
*/
public class HiLowEmitter extends PackEmitter {
/** number of entries in expanded array */
private int numEntries;
/**
* Create new emitter for values in [0, 0xFFFFFFFF] using hi/low encoding.
*
* @param name the name of the generated array
*/
public HiLowEmitter(String name) {
super(name);
}
/**
* Emits hi/low pair unpacking code for the generated array.
*
* @see JFlex.PackEmitter#emitUnPack()
*/
public void emitUnpack() {
// close last string chunk:
println("\";");
nl();
println(" private static int [] zzUnpack"+name+"() {");
println(" int [] result = new int["+numEntries+"];");
println(" int offset = 0;");
for (int i = 0; i < chunks; i++) {
println(" offset = zzUnpack"+name+"("+constName()+"_PACKED_"+i+", offset, result);");
}
println(" return result;");
println(" }");
nl();
println(" private static int zzUnpack"+name+"(String packed, int offset, int [] result) {");
println(" int i = 0; /* index in packed string */");
println(" int j = offset; /* index in unpacked array */");
println(" int l = packed.length();");
println(" while (i < l) {");
println(" int high = packed.charAt(i++) << 16;");
println(" result[j++] = high | packed.charAt(i++);");
println(" }");
println(" return j;");
println(" }");
}
/**
* Emit one value using two characters.
*
* @param val the value to emit
* @prec 0 <= val <= 0xFFFFFFFF
*/
public void emit(int val) {
numEntries+= 1;
breaks();
emitUC(val >> 16);
emitUC(val & 0xFFFF);
}
}
tools/lexer/jflex-1.4/src/JFlex/IntCharSet.java
-411
View File
@@ -1,411 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* JFlex 1.4.3 *
* Copyright (C) 1998-2009 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
package JFlex;
import java.util.*;
/**
* CharSet implemented with intervalls
*
* [fixme: optimizations possible]
*
* @author Gerwin Klein
* @version $Revision: 1.4.3 $, $Date: 2009/12/21 15:58:48 $
*/
public final class IntCharSet {
private final static boolean DEBUG = false;
/* invariant: all intervals are disjoint, ordered */
private Vector intervalls;
private int pos;
public IntCharSet() {
this.intervalls = new Vector();
}
public IntCharSet(char c) {
this(new Interval(c,c));
}
public IntCharSet(Interval intervall) {
this();
intervalls.addElement(intervall);
}
public IntCharSet(Vector /* Interval */ chars) {
int size = chars.size();
this.intervalls = new Vector(size);
for (int i = 0; i < size; i++)
add( (Interval) chars.elementAt(i) );
}
/**
* returns the index of the intervall that contains
* the character c, -1 if there is no such intevall
*
* @prec: true
* @post: -1 <= return < intervalls.size() &&
* (return > -1 --> intervalls[return].contains(c))
*
* @param c the character
* @return the index of the enclosing interval, -1 if no such interval
*/
private int indexOf(char c) {
int start = 0;
int end = intervalls.size()-1;
while (start <= end) {
int check = (start+end) / 2;
Interval i = (Interval) intervalls.elementAt(check);
if (start == end)
return i.contains(c) ? start : -1;
if (c < i.start) {
end = check-1;
continue;
}
if (c > i.end) {
start = check+1;
continue;
}
return check;
}
return -1;
}
public IntCharSet add(IntCharSet set) {
for (int i = 0; i < set.intervalls.size(); i++)
add( (Interval) set.intervalls.elementAt(i) );
return this;
}
public void add(Interval intervall) {
int size = intervalls.size();
for (int i = 0; i < size; i++) {
Interval elem = (Interval) intervalls.elementAt(i);
if ( elem.end+1 < intervall.start ) continue;
if ( elem.contains(intervall) ) return;
if ( elem.start > intervall.end+1 ) {
intervalls.insertElementAt(new Interval(intervall), i);
return;
}
if (intervall.start < elem.start)
elem.start = intervall.start;
if (intervall.end <= elem.end)
return;
elem.end = intervall.end;
i++;
// delete all x with x.contains( intervall.end )
while (i < size) {
Interval x = (Interval) intervalls.elementAt(i);
if (x.start > elem.end+1) return;
elem.end = x.end;
intervalls.removeElementAt(i);
size--;
}
return;
}
intervalls.addElement(new Interval(intervall));
}
public void add(char c) {
int size = intervalls.size();
for (int i = 0; i < size; i++) {
Interval elem = (Interval) intervalls.elementAt(i);
if (elem.end+1 < c) continue;
if (elem.contains(c)) return; // already there, nothing to do
// assert(elem.end+1 >= c && (elem.start > c || elem.end < c));
if (elem.start > c+1) {
intervalls.insertElementAt(new Interval(c,c), i);
return;
}
// assert(elem.end+1 >= c && elem.start <= c+1 && (elem.start > c || elem.end < c));
if (c+1 == elem.start) {
elem.start = c;
return;
}
// assert(elem.end+1 == c);
elem.end = c;
// merge with next interval if it contains c
if (i+1 >= size) return;
Interval x = (Interval) intervalls.elementAt(i+1);
if (x.start <= c+1) {
elem.end = x.end;
intervalls.removeElementAt(i+1);
}
return;
}
// end reached but nothing found -> append at end
intervalls.addElement(new Interval(c,c));
}
public boolean contains(char singleChar) {
return indexOf(singleChar) >= 0;
}
/**
* o instanceof Interval
*/
public boolean equals(Object o) {
IntCharSet set = (IntCharSet) o;
if ( intervalls.size() != set.intervalls.size() ) return false;
for (int i = 0; i < intervalls.size(); i++) {
if ( !intervalls.elementAt(i).equals( set.intervalls.elementAt(i)) )
return false;
}
return true;
}
private char min(char a, char b) {
return a <= b ? a : b;
}
private char max(char a, char b) {
return a >= b ? a : b;
}
/* intersection */
public IntCharSet and(IntCharSet set) {
if (DEBUG) {
Out.dump("intersection");
Out.dump("this : "+this);
Out.dump("other : "+set);
}
IntCharSet result = new IntCharSet();
int i = 0; // index in this.intervalls
int j = 0; // index in set.intervalls
int size = intervalls.size();
int setSize = set.intervalls.size();
while (i < size && j < setSize) {
Interval x = (Interval) this.intervalls.elementAt(i);
Interval y = (Interval) set.intervalls.elementAt(j);
if (x.end < y.start) {
i++;
continue;
}
if (y.end < x.start) {
j++;
continue;
}
result.intervalls.addElement(
new Interval(
max(x.start, y.start),
min(x.end, y.end)
)
);
if (x.end >= y.end) j++;
if (y.end >= x.end) i++;
}
if (DEBUG) {
Out.dump("result: "+result);
}
return result;
}
/* complement */
/* prec: this.contains(set), set != null */
public void sub(IntCharSet set) {
if (DEBUG) {
Out.dump("complement");
Out.dump("this : "+this);
Out.dump("other : "+set);
}
int i = 0; // index in this.intervalls
int j = 0; // index in set.intervalls
int setSize = set.intervalls.size();
while (i < intervalls.size() && j < setSize) {
Interval x = (Interval) this.intervalls.elementAt(i);
Interval y = (Interval) set.intervalls.elementAt(j);
if (DEBUG) {
Out.dump("this : "+this);
Out.dump("this ["+i+"] : "+x);
Out.dump("other ["+j+"] : "+y);
}
if (x.end < y.start) {
i++;
continue;
}
if (y.end < x.start) {
j++;
continue;
}
// x.end >= y.start && y.end >= x.start ->
// x.end <= y.end && x.start >= y.start (prec)
if ( x.start == y.start && x.end == y.end ) {
intervalls.removeElementAt(i);
j++;
continue;
}
// x.end <= y.end && x.start >= y.start &&
// (x.end < y.end || x.start > y.start) ->
// x.start < x.end
if ( x.start == y.start ) {
x.start = (char) (y.end+1);
j++;
continue;
}
if ( x.end == y.end ) {
x.end = (char) (y.start-1);
i++;
j++;
continue;
}
intervalls.insertElementAt(new Interval(x.start, (char) (y.start-1)), i);
x.start = (char) (y.end+1);
i++;
j++;
}
if (DEBUG) {
Out.dump("result: "+this);
}
}
public boolean containsElements() {
return intervalls.size() > 0;
}
public int numIntervalls() {
return intervalls.size();
}
// beware: depends on caller protocol, single user only
public Interval getNext() {
if (pos == intervalls.size()) pos = 0;
return (Interval) intervalls.elementAt(pos++);
}
/**
* Create a caseless version of this charset.
* <p>
* The caseless version contains all characters of this char set,
* and additionally all lower/upper/title case variants of the
* characters in this set.
*
* @return a caseless copy of this set
*/
public IntCharSet getCaseless() {
IntCharSet n = copy();
int size = intervalls.size();
for (int i=0; i < size; i++) {
Interval elem = (Interval) intervalls.elementAt(i);
for (char c = elem.start; c <= elem.end; c++) {
n.add(Character.toLowerCase(c));
n.add(Character.toUpperCase(c));
n.add(Character.toTitleCase(c));
}
}
return n;
}
/**
* Make a string representation of this char set.
*
* @return a string representing this char set.
*/
public String toString() {
StringBuffer result = new StringBuffer("{ ");
for (int i = 0; i < intervalls.size(); i++)
result.append( intervalls.elementAt(i) );
result.append(" }");
return result.toString();
}
/**
* Return a (deep) copy of this char set
*
* @return the copy
*/
public IntCharSet copy() {
IntCharSet result = new IntCharSet();
int size = intervalls.size();
for (int i=0; i < size; i++) {
Interval iv = ((Interval) intervalls.elementAt(i)).copy();
result.intervalls.addElement(iv);
}
return result;
}
}
tools/lexer/jflex-1.4/src/JFlex/IntPair.java
-57
View File
@@ -1,57 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* JFlex 1.4.3 *
* Copyright (C) 1998-2009 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
package JFlex;
/**
* Simple pair of integers.
*
* Used in NFA to represent a partial NFA by its start and end state.
*
* @author Gerwin Klein
* @version $Revision: 1.4.3 $, $Date: 2009/12/21 15:58:48 $
*/
final class IntPair {
int start;
int end;
IntPair(int start, int end) {
this.start = start;
this.end = end;
}
public int hashCode() {
return end + (start << 8);
}
public boolean equals(Object o) {
if ( o instanceof IntPair ) {
IntPair p = (IntPair) o;
return start == p.start && end == p.end;
}
return false;
}
public String toString() {
return "("+start+","+end+")";
}
}
tools/lexer/jflex-1.4/src/JFlex/Interval.java
-163
View File
@@ -1,163 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* JFlex 1.4.3 *
* Copyright (C) 1998-2009 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
package JFlex;
/**
* An intervall of characters with basic operations.
*
* @author Gerwin Klein
* @version $Revision: 1.4.3 $, $Date: 2009/12/21 15:58:48 $
*/
public final class Interval {
/* start and end of the intervall */
public char start, end;
/**
* Constuct a new intervall from <code>start</code> to <code>end</code>.
*
* @param start first character the intervall should contain
* @param end last character the intervall should contain
*/
public Interval(char start, char end) {
this.start = start;
this.end = end;
}
/**
* Copy constructor
*/
public Interval(Interval other) {
this.start = other.start;
this.end = other.end;
}
/**
* Return <code>true</code> iff <code>point</code> is contained in this intervall.
*
* @param point the character to check
*/
public boolean contains(char point) {
return start <= point && end >= point;
}
/**
* Return <code>true</code> iff this intervall completely contains the
* other one.
*
* @param other the other intervall
*/
public boolean contains(Interval other) {
return this.start <= other.start && this.end >= other.end;
}
/**
* Return <code>true</code> if <code>o</code> is an intervall
* with the same borders.
*
* @param o the object to check equality with
*/
public boolean equals(Object o) {
if ( o == this ) return true;
if ( !(o instanceof Interval) ) return false;
Interval other = (Interval) o;
return other.start == this.start && other.end == this.end;
}
/**
* Set a new last character
*
* @param end the new last character of this intervall
*/
public void setEnd(char end) {
this.end = end;
}
/**
* Set a new first character
*
* @param start the new first character of this intervall
*/
public void setStart(char start) {
this.start = start;
}
/**
* Check wether a character is printable.
*
* @param c the character to check
*/
private static boolean isPrintable(char c) {
// fixme: should make unicode test here
return c > 31 && c < 127;
}
/**
* Get a String representation of this intervall.
*
* @return a string <code>"[start-end]"</code> or
* <code>"[start]"</code> (if there is only one character in
* the intervall) where <code>start</code> and
* <code>end</code> are either a number (the character code)
* or something of the from <code>'a'</code>.
*/
public String toString() {
StringBuffer result = new StringBuffer("[");
if ( isPrintable(start) )
result.append("'"+start+"'");
else
result.append( (int) start );
if (start != end) {
result.append("-");
if ( isPrintable(end) )
result.append("'"+end+"'");
else
result.append( (int) end );
}
result.append("]");
return result.toString();
}
/**
* Make a copy of this interval.
*
* @return the copy
*/
public Interval copy() {
return new Interval(start,end);
}
}
tools/lexer/jflex-1.4/src/JFlex/LexParse.cup
-807
View File
@@ -1,807 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* JFlex 1.4.3 *
* Copyright (C) 1998-2009 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
package JFlex;
import java.util.*;
/* customizing code */
action code {:
LexScan scanner;
CharClasses charClasses = new CharClasses(Options.jlex ? 127 : 0xFFFF);
RegExps regExps = new RegExps();
Macros macros = new Macros();
Integer stateNumber;
Timer t = new Timer();
EOFActions eofActions = new EOFActions();
void fatalError(ErrorMessages message, int line, int col) {
syntaxError(message, line, col);
throw new GeneratorException();
}
void fatalError(ErrorMessages message) {
fatalError(message, scanner.currentLine(), -1);
throw new GeneratorException();
}
void syntaxError(ErrorMessages message) {
Out.error(scanner.file, message, scanner.currentLine(), -1);
}
void syntaxError(ErrorMessages message, int line) {
Out.error(scanner.file, message, line, -1);
}
void syntaxError(ErrorMessages message, int line, int col) {
Out.error(scanner.file, message, line, col);
}
private boolean check(int type, char c) {
switch (type) {
case sym.JLETTERCLASS:
return Character.isJavaIdentifierStart(c);
case sym.JLETTERDIGITCLASS:
return Character.isJavaIdentifierPart(c);
case sym.LETTERCLASS:
return Character.isLetter(c);
case sym.DIGITCLASS:
return Character.isDigit(c);
case sym.UPPERCLASS:
return Character.isUpperCase(c);
case sym.LOWERCLASS:
return Character.isLowerCase(c);
case sym.UNICODE_UNASSIGNED:
return Character.getType(c) == Character.UNASSIGNED;
case sym.UNICODE_UPPERCASE_LETTER:
return Character.getType(c) == Character.UPPERCASE_LETTER;
case sym.UNICODE_LOWERCASE_LETTER:
return Character.getType(c) == Character.LOWERCASE_LETTER;
case sym.UNICODE_TITLECASE_LETTER:
return Character.getType(c) == Character.TITLECASE_LETTER;
case sym.UNICODE_MODIFIER_LETTER:
return Character.getType(c) == Character.MODIFIER_LETTER;
case sym.UNICODE_OTHER_LETTER:
return Character.getType(c) == Character.OTHER_LETTER;
case sym.UNICODE_NON_SPACING_MARK:
return Character.getType(c) == Character.NON_SPACING_MARK;
case sym.UNICODE_ENCLOSING_MARK:
return Character.getType(c) == Character.ENCLOSING_MARK;
case sym.UNICODE_COMBINING_SPACING_MARK:
return Character.getType(c) == Character.COMBINING_SPACING_MARK;
case sym.UNICODE_DECIMAL_DIGIT_NUMBER:
return Character.getType(c) == Character.DECIMAL_DIGIT_NUMBER;
case sym.UNICODE_LETTER_NUMBER:
return Character.getType(c) == Character.LETTER_NUMBER;
case sym.UNICODE_OTHER_NUMBER:
return Character.getType(c) == Character.OTHER_NUMBER;
case sym.UNICODE_SPACE_SEPARATOR:
return Character.getType(c) == Character.SPACE_SEPARATOR;
case sym.UNICODE_LINE_SEPARATOR:
return Character.getType(c) == Character.LINE_SEPARATOR;
case sym.UNICODE_PARAGRAPH_SEPARATOR:
return Character.getType(c) == Character.PARAGRAPH_SEPARATOR;
case sym.UNICODE_CONTROL:
return Character.getType(c) == Character.CONTROL;
case sym.UNICODE_FORMAT:
return Character.getType(c) == Character.FORMAT;
case sym.UNICODE_PRIVATE_USE:
return Character.getType(c) == Character.PRIVATE_USE;
case sym.UNICODE_SURROGATE:
return Character.getType(c) == Character.SURROGATE;
case sym.UNICODE_DASH_PUNCTUATION:
return Character.getType(c) == Character.DASH_PUNCTUATION;
case sym.UNICODE_START_PUNCTUATION:
return Character.getType(c) == Character.START_PUNCTUATION;
case sym.UNICODE_END_PUNCTUATION:
return Character.getType(c) == Character.END_PUNCTUATION;
case sym.UNICODE_CONNECTOR_PUNCTUATION:
return Character.getType(c) == Character.CONNECTOR_PUNCTUATION;
case sym.UNICODE_OTHER_PUNCTUATION:
return Character.getType(c) == Character.OTHER_PUNCTUATION;
case sym.UNICODE_MATH_SYMBOL:
return Character.getType(c) == Character.MATH_SYMBOL;
case sym.UNICODE_CURRENCY_SYMBOL:
return Character.getType(c) == Character.CURRENCY_SYMBOL;
case sym.UNICODE_MODIFIER_SYMBOL:
return Character.getType(c) == Character.MODIFIER_SYMBOL;
case sym.UNICODE_OTHER_SYMBOL:
return Character.getType(c) == Character.OTHER_SYMBOL;
case sym.UNICODE_INITIAL_QUOTE_PUNCTUATION:
return Character.getType(c) == Character.INITIAL_QUOTE_PUNCTUATION;
case sym.UNICODE_FINAL_QUOTE_PUNCTUATION:
return Character.getType(c) == Character.FINAL_QUOTE_PUNCTUATION;
default: return false;
}
}
private Vector makePreClass(int type) {
Vector result = new Vector();
char c = 0;
char start = 0;
char last = charClasses.getMaxCharCode();
boolean prev, current;
prev = check(type,'\u0000');
for (c = 1; c < last; c++) {
current = check(type,c);
if (!prev && current) start = c;
if (prev && !current) {
result.addElement(new Interval(start, (char)(c-1)));
}
prev = current;
}
// the last iteration is moved out of the loop to
// avoid an endless loop if last == maxCharCode and
// last+1 == 0
current = check(type,c);
if (!prev && current) result.addElement(new Interval(c,c));
if (prev && current) result.addElement(new Interval(start, c));
if (prev && !current) result.addElement(new Interval(start, (char)(c-1)));
return result;
}
private RegExp makeRepeat(RegExp r, int n1, int n2, int line, int col) {
if (n1 <= 0 && n2 <= 0) {
syntaxError(ErrorMessages.REPEAT_ZERO, line, col);
return null;
}
if (n1 > n2) {
syntaxError(ErrorMessages.REPEAT_GREATER, line, col);
return null;
}
int i;
RegExp result;
if (n1 > 0) {
result = r;
n1--; n2--; // we need one concatenation less than the number of expressions to match
}
else {
result = new RegExp1(sym.QUESTION,r);
n2--;
}
for (i = 0; i < n1; i++)
result = new RegExp2(sym.CONCAT, result, r);
n2-= n1;
for (i = 0; i < n2; i++)
result = new RegExp2(sym.CONCAT, result, new RegExp1(sym.QUESTION,r));
return result;
}
private RegExp makeNL() {
Vector list = new Vector();
list.addElement(new Interval('\n','\r'));
list.addElement(new Interval('\u0085','\u0085'));
list.addElement(new Interval('\u2028','\u2029'));
// assumption: line feeds are caseless
charClasses.makeClass(list, false);
charClasses.makeClass('\n', false);
charClasses.makeClass('\r', false);
RegExp1 c = new RegExp1(sym.CCLASS, list);
Character n = new Character('\n');
Character r = new Character('\r');
return new RegExp2(sym.BAR,
c,
new RegExp2(sym.CONCAT,
new RegExp1(sym.CHAR, r),
new RegExp1(sym.CHAR, n)));
}
:};
parser code {:
public LexScan scanner;
public LexParse(LexScan scanner) {
super(scanner);
this.scanner = scanner;
}
public CharClasses getCharClasses() {
return action_obj.charClasses;
}
public EOFActions getEOFActions() {
return action_obj.eofActions;
}
public void report_error(String message, Object info) {
if ( info instanceof java_cup.runtime.Symbol ) {
java_cup.runtime.Symbol s = (java_cup.runtime.Symbol) info;
if (s.sym == sym.EOF)
Out.error(ErrorMessages.UNEXPECTED_EOF);
else
Out.error(scanner.file, ErrorMessages.SYNTAX_ERROR, s.left, s.right);
}
else
Out.error(ErrorMessages.UNKNOWN_SYNTAX);
}
public void report_fatal_error(String message, Object info) {
// report_error(message, info);
throw new GeneratorException();
}
:};
init with {:
action_obj.scanner = this.scanner;
:};
/* token declarations */
terminal OPENBRACKET, CLOSEBRACKET, HAT, DOLLAR, OPENCLASS,
CLOSECLASS, DASH, DELIMITER, EQUALS, COMMA, LESSTHAN,
MORETHAN, LBRACE, RBRACE, ASCII, FULL, UNICODE, REGEXPEND;
terminal JLETTERCLASS, JLETTERDIGITCLASS, LETTERCLASS, DIGITCLASS,
UPPERCLASS, LOWERCLASS, EOFRULE, NOACTION, LOOKAHEAD,
UNICODE_UNASSIGNED, UNICODE_UPPERCASE_LETTER, UNICODE_LOWERCASE_LETTER, UNICODE_TITLECASE_LETTER,
UNICODE_MODIFIER_LETTER, UNICODE_OTHER_LETTER, UNICODE_NON_SPACING_MARK, UNICODE_ENCLOSING_MARK,
UNICODE_COMBINING_SPACING_MARK, UNICODE_DECIMAL_DIGIT_NUMBER, UNICODE_LETTER_NUMBER, UNICODE_OTHER_NUMBER,
UNICODE_SPACE_SEPARATOR, UNICODE_LINE_SEPARATOR, UNICODE_PARAGRAPH_SEPARATOR, UNICODE_CONTROL, UNICODE_FORMAT,
UNICODE_PRIVATE_USE, UNICODE_SURROGATE, UNICODE_DASH_PUNCTUATION, UNICODE_START_PUNCTUATION,
UNICODE_END_PUNCTUATION, UNICODE_CONNECTOR_PUNCTUATION, UNICODE_OTHER_PUNCTUATION, UNICODE_MATH_SYMBOL,
UNICODE_CURRENCY_SYMBOL, UNICODE_MODIFIER_SYMBOL, UNICODE_OTHER_SYMBOL, UNICODE_INITIAL_QUOTE_PUNCTUATION,
UNICODE_FINAL_QUOTE_PUNCTUATION;
terminal Action ACTION;
terminal String IDENT, USERCODE;
terminal Integer REPEAT;
/* tokens used in RegExp parse tree */
terminal STAR, PLUS, BAR, QUESTION, POINT, BANG, TILDE;
terminal Character CHAR;
terminal String STRING, MACROUSE;
/* symbols *only* used in the parse tree (not in the grammar) */
terminal CCLASS, CCLASSNOT, CONCAT;
terminal STRING_I, CHAR_I; /* case insensitive strings/chars */
non terminal macros, macro;
non terminal Integer rule;
non terminal NFA specification;
non terminal RegExp series, concs, nregexp, regexp, charclass, lookahead;
non terminal Interval classcontentelem;
non terminal Vector states, statesOPT, classcontent, preclass, rules;
non terminal Boolean hatOPT;
non terminal Action act, actions;
/* grammar specification */
start with specification;
specification ::= USERCODE
/* delimiter is checked in lexer */
macros
DELIMITER
rules
{:
scanner.t.stop();
Out.checkErrors();
Out.time(ErrorMessages.PARSING_TOOK, t);
macros.expand();
Enumeration unused = macros.unused();
while ( unused.hasMoreElements() ) {
Out.warning("Macro \""+unused.nextElement()+"\" has been declared but never used.");
}
SemCheck.check(regExps, macros, scanner.file);
regExps.checkActions();
regExps.checkLookAheads();
Out.checkErrors();
if (Options.dump) charClasses.dump();
Out.print("Constructing NFA : ");
t.start();
int num = regExps.getNum();
RESULT = new NFA(charClasses.getNumClasses(),
scanner, regExps, macros, charClasses);
eofActions.setNumLexStates(scanner.states.number());
for (int i = 0; i < num; i++) {
if (regExps.isEOF(i))
eofActions.add( regExps.getStates(i), regExps.getAction(i) );
else
RESULT.addRegExp(i);
}
if (scanner.standalone) RESULT.addStandaloneRule();
t.stop();
Out.time("");
Out.time(ErrorMessages.NFA_TOOK, t);
:}
| /* emtpy spec. error */
{:
fatalError(ErrorMessages.NO_LEX_SPEC);
:}
;
macros ::= /* empty, most switches & state declarations are parsed in lexer */
| macros macro
| error;
macro ::= ASCII
{: charClasses.setMaxCharCode(127); :}
| FULL
{: charClasses.setMaxCharCode(255); :}
| UNICODE
{: charClasses.setMaxCharCode(0xFFFF); :}
| IDENT:name EQUALS series:definition REGEXPEND
{: macros.insert(name, definition); :}
| IDENT EQUALS:e
{: syntaxError(ErrorMessages.REGEXP_EXPECTED, eleft, eright); :}
;
rules ::= rules:rlist rule:r
{: rlist.addElement(r); RESULT = rlist; :}
| rules:rlist1 LESSTHAN states:states MORETHAN LBRACE rules:rlist2 RBRACE
{:
Enumeration rs = rlist2.elements();
while ( rs.hasMoreElements() ) {
Integer elem = (Integer) rs.nextElement();
// might be null for error case of "rule"
if (elem != null) {
regExps.addStates( elem.intValue(), states );
}
rlist1.addElement( elem );
}
RESULT = rlist1;
:}
| LESSTHAN states:states MORETHAN LBRACE rules:rlist RBRACE
{:
Enumeration rs = rlist.elements();
while ( rs.hasMoreElements() ) {
Integer elem = (Integer) rs.nextElement();
// might be null for error case of "rule"
if (elem != null) {
regExps.addStates( elem.intValue(), states );
}
}
RESULT = rlist;
:}
| rule:r
{: RESULT = new Vector(); RESULT.addElement(r); :}
;
rule ::= statesOPT:s hatOPT:bol series:r actions:a
{: RESULT = new Integer(regExps.insert(rleft, s, r, a, bol, null)); :}
| statesOPT:s hatOPT:bol series:r lookahead:l act:a
{: RESULT = new Integer(regExps.insert(rleft, s, r, a, bol, l)); :}
| statesOPT:s hatOPT:bol series:r lookahead:l NOACTION:a
{: syntaxError(ErrorMessages.LOOKAHEAD_NEEDS_ACTION, aleft, aright+1); :}
| statesOPT:s EOFRULE ACTION:a
{: RESULT = new Integer(regExps.insert(s, a)); :}
| error
;
lookahead ::= DOLLAR
{: RESULT = makeNL(); :}
| LOOKAHEAD series:r
{: RESULT = r; :}
| LOOKAHEAD series:s DOLLAR
{: RESULT = new RegExp2(sym.CONCAT, s, makeNL()); :}
;
act ::= REGEXPEND ACTION:a
{: RESULT = a; :}
;
actions ::= act:a
{: RESULT = a; :}
| NOACTION
;
statesOPT ::= LESSTHAN states:list MORETHAN
{: RESULT = list; :}
| /* empty */
{: RESULT = new Vector(); :}
;
states ::= IDENT:id COMMA states:list
{:
stateNumber = scanner.states.getNumber( id );
if ( stateNumber != null )
list.addElement( stateNumber );
else {
throw new ScannerException(scanner.file, ErrorMessages.LEXSTATE_UNDECL,
idleft, idright);
}
RESULT = list;
:}
| IDENT:id
{:
Vector list = new Vector();
stateNumber = scanner.states.getNumber( id );
if ( stateNumber != null )
list.addElement( stateNumber );
else {
throw new ScannerException(scanner.file, ErrorMessages.LEXSTATE_UNDECL,
idleft, idright);
}
RESULT = list;
:}
| IDENT COMMA:c
{: syntaxError(ErrorMessages.REGEXP_EXPECTED, cleft, cright+1); :}
;
hatOPT ::= HAT
{: // assumption: there is no upper case for \n
charClasses.makeClass('\n', false);
RESULT = new Boolean(true); :}
| /* empty */
{: RESULT = new Boolean(false); :}
;
series ::= series:r1 BAR concs:r2
{: RESULT = new RegExp2(sym.BAR, r1, r2); :}
| concs:r
{: RESULT = r; :}
| BAR:b
{: syntaxError(ErrorMessages.REGEXP_EXPECTED, bleft, bright); :}
;
concs ::= concs:r1 nregexp:r2
{: RESULT = new RegExp2(sym.CONCAT, r1, r2); :}
| nregexp:r
{: RESULT = r; :}
;
nregexp ::= regexp:r
{: RESULT = r; :}
| BANG nregexp:r
{: RESULT = new RegExp1(sym.BANG, r); :}
| TILDE nregexp:r
{: RESULT = new RegExp1(sym.TILDE, r); :}
;
regexp ::= regexp:r STAR
{: RESULT = new RegExp1(sym.STAR, r); :}
| regexp:r PLUS
{: RESULT = new RegExp1(sym.PLUS, r); :}
| regexp:r QUESTION
{: RESULT = new RegExp1(sym.QUESTION, r); :}
| regexp:r REPEAT:n RBRACE:b
{: RESULT = makeRepeat(r, n.intValue(), n.intValue(), bleft, bright); :}
| regexp:r REPEAT:n1 REPEAT:n2 RBRACE
{: RESULT = makeRepeat(r, n1.intValue(), n2.intValue(), n1left, n2right); :}
| OPENBRACKET series:r CLOSEBRACKET
{: RESULT = r; :}
| MACROUSE:ident
{:
if ( !scanner.macroDefinition ) {
if ( ! macros.markUsed(ident) )
throw new ScannerException(scanner.file, ErrorMessages.MACRO_UNDECL,
identleft, identright);
}
RESULT = new RegExp1(sym.MACROUSE, ident);
:}
| charclass:c
{: RESULT = c; :}
| preclass:list
{:
try {
// assumption [correct?]: preclasses are already closed under case
charClasses.makeClass(list, false);
}
catch (CharClassException e) {
syntaxError(ErrorMessages.CHARSET_2_SMALL, listleft);
}
RESULT = new RegExp1(sym.CCLASS, list);
:}
| STRING:str
{:
try {
if ( scanner.caseless ) {
charClasses.makeClass(str, true);
RESULT = new RegExp1(sym.STRING_I, str);
}
else {
charClasses.makeClass(str, false);
RESULT = new RegExp1(sym.STRING, str);
}
}
catch (CharClassException e) {
syntaxError(ErrorMessages.CS2SMALL_STRING, strleft, strright);
}
:}
| POINT
{:
Vector any = new Vector();
any.addElement(new Interval('\n','\n'));
// assumption: there is no upper case for \n
charClasses.makeClass('\n', false);
RESULT = new RegExp1(sym.CCLASSNOT, any);
:}
| CHAR:c
{:
try {
if ( scanner.caseless ) {
charClasses.makeClass(c.charValue(), true);
RESULT = new RegExp1(sym.CHAR_I, c);
}
else {
charClasses.makeClass(c.charValue(), false);
RESULT = new RegExp1(sym.CHAR, c);
}
}
catch (CharClassException e) {
syntaxError(ErrorMessages.CS2SMALL_CHAR, cleft, cright);
}
:}
;
charclass ::= OPENCLASS CLOSECLASS
{:
RESULT = new RegExp1(sym.CCLASS,null);
:}
| OPENCLASS classcontent:list CLOSECLASS:close
{:
try {
charClasses.makeClass(list, Options.jlex && scanner.caseless);
}
catch (CharClassException e) {
syntaxError(ErrorMessages.CHARSET_2_SMALL, closeleft, closeright);
}
RESULT = new RegExp1(sym.CCLASS,list);
:}
| OPENCLASS HAT CLOSECLASS:close
{:
Vector list = new Vector();
list.addElement(new Interval((char)0,CharClasses.maxChar));
try {
charClasses.makeClass(list, false);
}
catch (CharClassException e) {
syntaxError(ErrorMessages.CHARSET_2_SMALL, closeleft, closeright);
}
RESULT = new RegExp1(sym.CCLASS,list);
:}
| OPENCLASS HAT classcontent:list CLOSECLASS:close
{:
try {
charClasses.makeClassNot(list, Options.jlex && scanner.caseless);
}
catch (CharClassException e) {
syntaxError(ErrorMessages.CHARSET_2_SMALL, closeleft, closeright);
}
RESULT = new RegExp1(sym.CCLASSNOT,list);
:}
| OPENCLASS DASH classcontent:list CLOSECLASS:close
{:
try {
list.addElement(new Interval('-','-'));
charClasses.makeClass(list, Options.jlex && scanner.caseless);
}
catch (CharClassException e) {
syntaxError(ErrorMessages.CHARSET_2_SMALL, closeleft, closeright);
}
RESULT = new RegExp1(sym.CCLASS,list);
:}
| OPENCLASS HAT DASH classcontent:list CLOSECLASS:close
{:
try {
list.addElement(new Interval('-','-'));
charClasses.makeClassNot(list, Options.jlex && scanner.caseless);
}
catch (CharClassException e) {
syntaxError(ErrorMessages.CHARSET_2_SMALL, closeleft, closeright);
}
RESULT = new RegExp1(sym.CCLASSNOT,list);
:}
;
classcontent ::= classcontent:list classcontentelem:elem
{:
list.addElement(elem);
RESULT = list;
:}
| classcontentelem:elem
{:
Vector list = new Vector();
list.addElement(elem);
RESULT = list;
:}
| classcontent:list preclass:plist
{:
for (Enumeration e = plist.elements(); e.hasMoreElements();)
list.addElement(e.nextElement());
RESULT = list;
:}
| preclass:list
{: RESULT = list; :}
| classcontent:list STRING:s
{:
for (int i = 0; i < s.length(); i++)
list.addElement(new Interval(s.charAt(i),s.charAt(i)));
RESULT = list;
:}
| STRING:s
{:
RESULT = new Vector();
for (int i = 0; i < s.length(); i++)
RESULT.addElement(new Interval(s.charAt(i),s.charAt(i)));
:}
| classcontent:list MACROUSE:ident
{:
syntaxError(ErrorMessages.CHARCLASS_MACRO, identleft, identright);
:}
| MACROUSE:ident
{:
syntaxError(ErrorMessages.CHARCLASS_MACRO, identleft, identright);
:}
;
classcontentelem ::= CHAR:c1 DASH CHAR:c2
{: RESULT = new Interval(c1.charValue(), c2.charValue()); :}
| CHAR:c
{: RESULT = new Interval(c.charValue(), c.charValue()); :}
;
preclass ::= JLETTERCLASS
{: RESULT = makePreClass(sym.JLETTERCLASS); :}
| JLETTERDIGITCLASS
{: RESULT = makePreClass(sym.JLETTERDIGITCLASS); :}
| LETTERCLASS
{: RESULT = makePreClass(sym.LETTERCLASS); :}
| DIGITCLASS
{: RESULT = makePreClass(sym.DIGITCLASS); :}
| UPPERCLASS
{: RESULT = makePreClass(sym.UPPERCLASS); :}
| LOWERCLASS
{: RESULT = makePreClass(sym.LOWERCLASS); :}
| UNICODE_UNASSIGNED
{: RESULT = makePreClass(sym.UNICODE_UNASSIGNED); :}
| UNICODE_UPPERCASE_LETTER
{: RESULT = makePreClass(sym.UNICODE_UPPERCASE_LETTER); :}
| UNICODE_LOWERCASE_LETTER
{: RESULT = makePreClass(sym.UNICODE_LOWERCASE_LETTER); :}
| UNICODE_TITLECASE_LETTER
{: RESULT = makePreClass(sym.UNICODE_TITLECASE_LETTER); :}
| UNICODE_MODIFIER_LETTER
{: RESULT = makePreClass(sym.UNICODE_MODIFIER_LETTER); :}
| UNICODE_OTHER_LETTER
{: RESULT = makePreClass(sym.UNICODE_OTHER_LETTER); :}
| UNICODE_NON_SPACING_MARK
{: RESULT = makePreClass(sym.UNICODE_NON_SPACING_MARK); :}
| UNICODE_ENCLOSING_MARK
{: RESULT = makePreClass(sym.UNICODE_ENCLOSING_MARK); :}
| UNICODE_COMBINING_SPACING_MARK
{: RESULT = makePreClass(sym.UNICODE_COMBINING_SPACING_MARK); :}
| UNICODE_DECIMAL_DIGIT_NUMBER
{: RESULT = makePreClass(sym.UNICODE_DECIMAL_DIGIT_NUMBER); :}
| UNICODE_LETTER_NUMBER
{: RESULT = makePreClass(sym.UNICODE_LETTER_NUMBER); :}
| UNICODE_OTHER_NUMBER
{: RESULT = makePreClass(sym.UNICODE_OTHER_NUMBER); :}
| UNICODE_SPACE_SEPARATOR
{: RESULT = makePreClass(sym.UNICODE_SPACE_SEPARATOR); :}
| UNICODE_LINE_SEPARATOR
{: RESULT = makePreClass(sym.UNICODE_LINE_SEPARATOR); :}
| UNICODE_PARAGRAPH_SEPARATOR
{: RESULT = makePreClass(sym.UNICODE_PARAGRAPH_SEPARATOR); :}
| UNICODE_CONTROL
{: RESULT = makePreClass(sym.UNICODE_CONTROL); :}
| UNICODE_FORMAT
{: RESULT = makePreClass(sym.UNICODE_FORMAT); :}
| UNICODE_PRIVATE_USE
{: RESULT = makePreClass(sym.UNICODE_PRIVATE_USE); :}
| UNICODE_SURROGATE
{: RESULT = makePreClass(sym.UNICODE_SURROGATE); :}
| UNICODE_DASH_PUNCTUATION
{: RESULT = makePreClass(sym.UNICODE_DASH_PUNCTUATION); :}
| UNICODE_START_PUNCTUATION
{: RESULT = makePreClass(sym.UNICODE_START_PUNCTUATION); :}
| UNICODE_END_PUNCTUATION
{: RESULT = makePreClass(sym.UNICODE_END_PUNCTUATION); :}
| UNICODE_CONNECTOR_PUNCTUATION
{: RESULT = makePreClass(sym.UNICODE_CONNECTOR_PUNCTUATION); :}
| UNICODE_OTHER_PUNCTUATION
{: RESULT = makePreClass(sym.UNICODE_OTHER_PUNCTUATION); :}
| UNICODE_MATH_SYMBOL
{: RESULT = makePreClass(sym.UNICODE_MATH_SYMBOL); :}
| UNICODE_CURRENCY_SYMBOL
{: RESULT = makePreClass(sym.UNICODE_CURRENCY_SYMBOL); :}
| UNICODE_MODIFIER_SYMBOL
{: RESULT = makePreClass(sym.UNICODE_MODIFIER_SYMBOL); :}
| UNICODE_OTHER_SYMBOL
{: RESULT = makePreClass(sym.UNICODE_OTHER_SYMBOL); :}
| UNICODE_INITIAL_QUOTE_PUNCTUATION
{: RESULT = makePreClass(sym.UNICODE_INITIAL_QUOTE_PUNCTUATION); :}
| UNICODE_FINAL_QUOTE_PUNCTUATION
{: RESULT = makePreClass(sym.UNICODE_FINAL_QUOTE_PUNCTUATION); :}
;
tools/lexer/jflex-1.4/src/JFlex/LexParse.java
-2795
View File
File diff suppressed because it is too large Load Diff
tools/lexer/jflex-1.4/src/JFlex/LexScan.flex
-613
View File
@@ -1,613 +0,0 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* JFlex 1.4.3 *
* Copyright (C) 1998-2009 Gerwin Klein <lsf@jflex.de> *
* All rights reserved. *
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License. See the file *
* COPYRIGHT for more information. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License along *
* with this program; if not, write to the Free Software Foundation, Inc., *
* 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA *
* *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
package JFlex;
import java_cup.runtime.Symbol;
import java.util.Vector;
import java.io.*;
import java.util.Stack;
/**
* The lexer of JFlex.
*
* Generated by <a href="http://www.jflex.de/">JFlex</a>.
*
* @author Gerwin Klein
* @version $Revision: 1.4.3 $, $Date: 2009/12/21 15:58:48 $
*/
%%
%final
%public
%class LexScan
%implements sym, java_cup.runtime.Scanner
%function next_token
%type Symbol
%unicode
%column
%line
%eofclose
%state COMMENT, STATELIST, MACROS, REGEXPSTART
%state REGEXP, JAVA_CODE, STATES, STRING_CONTENT
%state CHARCLASS, COPY, REPEATEXP, EATWSPNL
%state CTOR_ARG
%cupdebug
%{
int balance = 0;
int commentbalance = 0;
int action_line = 0;
int bufferSize = 16384;
File file;
Stack files = new Stack();
StringBuffer userCode = new StringBuffer();
String classCode;
String initCode;
String initThrow;
String eofCode;
String eofThrow;
String lexThrow;
String eofVal;
String scanErrorException;
String cupSymbol = "sym";
StringBuffer actionText = new StringBuffer();
StringBuffer string = new StringBuffer();
boolean charCount;
boolean lineCount;
boolean columnCount;
boolean cupCompatible;
boolean cupDebug;
boolean isInteger;
boolean isIntWrap;
boolean isYYEOF;
boolean notUnix;
boolean isPublic;
boolean isFinal;
boolean isAbstract;
boolean bolUsed;
boolean standalone;
boolean debugOption;
boolean useRowMap = Options.gen_method == Options.PACK || Options.gen_method == Options.TABLE;
boolean packed = Options.gen_method == Options.PACK;
boolean caseless;
boolean inclusive_states;
boolean eofclose;
boolean emitInputStreamCtor = Options.emitInputStreamCtor;
String isImplementing;
String isExtending;
String className = "Yylex";
String functionName;
String tokenType;
String visibility = "public";
Vector /* String */ ctorArgs = new Vector();
Vector /* String */ ctorTypes = new Vector();
LexicalStates states = new LexicalStates();
Vector actions = new Vector();
private int nextState;
boolean macroDefinition;
Timer t = new Timer();
public int currentLine() {
return yyline;
}
public void setFile(File file) {
this.file = file;
}
private Symbol symbol(int type, Object value) {
return new Symbol(type, yyline, yycolumn, value);
}
private Symbol symbol(int type) {
return new Symbol(type, yyline, yycolumn);
}
// updates line and column count to the beginning of the first
// non whitespace character in yytext, but leaves yyline+yycolumn
// untouched
private Symbol symbol_countUpdate(int type, Object value) {
int lc = yyline;
int cc = yycolumn;
String text = yytext();
for (int i=0; i < text.length(); i++) {
char c = text.charAt(i);
if (c != '\n' && c != '\r' && c != ' ' && c != '\t' )
return new Symbol(type, lc, cc, value);
if (c == '\n') {
lc++;
cc = 0;
}
else
cc++;
}
return new Symbol(type, yyline, yycolumn, value);
}
private String makeMacroIdent() {
String matched = yytext().trim();
return matched.substring(1, matched.length()-1).trim();
}
public static String conc(Object a, Object b) {
if (a == null && b == null) return null;
if (a == null) return b.toString();
if (b == null) return a.toString();
return a.toString()+b.toString();
}
public static String concExc(Object a, Object b) {
if (a == null && b == null) return null;
if (a == null) return b.toString();
if (b == null) return a.toString();
return a.toString()+", "+b.toString();
}
%}
%init{
states.insert("YYINITIAL", true);
%init}
Digit = [0-9]
HexDigit = [0-9a-fA-F]
OctDigit = [0-7]
Number = {Digit}+
HexNumber = \\ x {HexDigit} {2}
Unicode = \\ u {HexDigit} {1, 4}
OctNumber = \\ [0-3]? {OctDigit} {1, 2}
// see http://www.unicode.org/unicode/reports/tr18/
WSP = [ \t\b]
WSPNL = [\u2028\u2029\u000A\u000B\u000C\u000D\u0085\t\b\ ]
NL = [\u2028\u2029\u000A\u000B\u000C\u000D\u0085] | \u000D\u000A
NNL = [^\u2028\u2029\u000A\u000B\u000C\u000D\u0085]
Ident = {IdentStart} {IdentPart}*
QualIdent = {Ident} ( {WSP}* "." {WSP}* {Ident} )*
QUIL = {QualIdent} ( {WSP}* "," {WSP}* {QualIdent} )*
Array = "[" {WSP}* "]"
ParamPart = {IdentStart}|{IdentPart}|"<"|">"|","|{WSP}|"&"|"?"|"."
GenParam = "<" {ParamPart}+ ">"
ClassT = {Ident} ({WSP}* {GenParam})?
QClassT = {QualIdent} ({WSP}* {GenParam})?
ArrType = ({GenParam} {WSP}*)? {QClassT} ({WSP}* {Array})*
IdentStart = [:jletter:]
IdentPart = [:jletterdigit:]
JFlexCommentChar = [^*/]|"/"+[^*/]|"*"+[^*/]
JFlexComment = {JFlexCommentChar}*
/* Java comments */
JavaComment = {TraditionalComment}|{EndOfLineComment}
TraditionalComment = "/*"{CommentContent}\*+"/"
EndOfLineComment = "//".*{NL}
CommentContent = ([^*]|\*+[^*/])*
StringCharacter = [^\u2028\u2029\u000A\u000B\u000C\u000D\u0085\"\\]
CharLiteral = \'([^\u2028\u2029\u000A\u000B\u000C\u000D\u0085\'\\]|{EscapeSequence})\'
StringLiteral = \"({StringCharacter}|{EscapeSequence})*\"
EscapeSequence = \\[^\u2028\u2029\u000A\u000B\u000C\u000D\u0085]|\\+u{HexDigit}{4}|\\[0-3]?{OctDigit}{1,2}
/* \\(b|t|n|f|r|\"|\'|\\|[0-3]?{OctDigit}{1,2}|u{HexDigit}{4}) */
JavaRest = [^\{\}\"\'/]|"/"[^*/]
JavaCode = ({JavaRest}|{StringLiteral}|{CharLiteral}|{JavaComment})+
%%
<YYINITIAL> {
"%%".*{NL}? {
t.start();
yybegin(MACROS);
macroDefinition = true;
return symbol(USERCODE,userCode);
}
.*{NL} { userCode.append(yytext()); }
.* { return symbol(EOF); }
}
<MACROS> ("%{"|"%init{"|"%initthrow{"|"%eof{"|"%eofthrow{"|"%yylexthrow{"|"%eofval{").*{NL}
{ string.setLength(0); yybegin(COPY); }
<COPY> {
"%}".*{NL} { classCode = conc(classCode,string); yybegin(MACROS); }
"%init}".*{NL} { initCode = conc(initCode,string); yybegin(MACROS); }
"%initthrow}".*{NL} { initThrow = concExc(initThrow,string); yybegin(MACROS); }
"%eof}".*{NL} { eofCode = conc(eofCode,string); yybegin(MACROS); }
"%eofthrow}".*{NL} { eofThrow = concExc(eofThrow,string); yybegin(MACROS); }
"%yylexthrow}".*{NL} { lexThrow = concExc(lexThrow,string); yybegin(MACROS); }
"%eofval}".*{NL} { eofVal = string.toString(); yybegin(MACROS); }
.*{NL} { string.append(yytext()); }
<<EOF>> { throw new ScannerException(file,ErrorMessages.EOF_IN_MACROS); }
}
<MACROS> ^"%s" ("tate" "s"?)? {WSP}+ { inclusive_states = true; yybegin(STATELIST); }
<MACROS> ^"%x" ("state" "s"?)? {WSP}+ { inclusive_states = false; yybegin(STATELIST); }
<STATELIST> {
{Ident} { states.insert(yytext(),inclusive_states); }
([\ \t]*","[\ \t]*)|([\ \t]+) { }
{NL} { yybegin(MACROS); }
<<EOF>> { throw new ScannerException(file,ErrorMessages.EOF_IN_MACROS); }
}
<MACROS> {
"%char" { charCount = true; }
"%line" { lineCount = true; }
"%column" { columnCount = true; }
"%byaccj" { isInteger = true;
if (eofVal == null)
eofVal = "return 0;";
eofclose = true;
}
"%cup" { cupCompatible = true;
isImplementing = concExc(isImplementing, "java_cup.runtime.Scanner");
if (functionName == null)
functionName = "next_token";
if (tokenType == null)
tokenType = "java_cup.runtime.Symbol";
if (eofVal == null)
eofVal = "return new java_cup.runtime.Symbol("+cupSymbol+".EOF);";
if (!Options.jlex) eofclose = true;
}
"%cupsym"{WSP}+{QualIdent} {WSP}* { cupSymbol = yytext().substring(8).trim();
if (cupCompatible) Out.warning(ErrorMessages.CUPSYM_AFTER_CUP, yyline); }
"%cupsym"{WSP}+{NNL}* { throw new ScannerException(file,ErrorMessages.QUIL_CUPSYM, yyline); }
"%cupdebug" { cupDebug = true; }
"%eofclose"({WSP}+"true")? { eofclose = true; }
"%eofclose"({WSP}+"false") { eofclose = false; }
"%class"{WSP}+{ClassT} {WSP}* { className = yytext().substring(7).trim(); }
"%ctorarg"{WSP}+{ArrType}{WSP}+ { yybegin(CTOR_ARG); ctorTypes.add(yytext().substring(8).trim()); }
"%function"{WSP}+{Ident} {WSP}* { functionName = yytext().substring(10).trim(); }
"%type"{WSP}+{ArrType} {WSP}* { tokenType = yytext().substring(6).trim(); }
"%integer"|"%int" { isInteger = true; }
"%intwrap" { isIntWrap = true; }
"%yyeof" { isYYEOF = true; }
"%notunix" { notUnix = true; }
"%7bit" { return symbol(ASCII); }
"%full"|"%8bit" { return symbol(FULL); }
"%unicode"|"%16bit" { return symbol(UNICODE); }
"%caseless"|"%ignorecase" { caseless = true; }
"%implements"{WSP}+.* { isImplementing = concExc(isImplementing, yytext().substring(12).trim()); }
"%extends"{WSP}+{QClassT}{WSP}* { isExtending = yytext().substring(9).trim(); }
"%public" { isPublic = true; }
"%apiprivate" { visibility = "private"; Skeleton.makePrivate(); }
"%final" { isFinal = true; }
"%abstract" { isAbstract = true; }
"%debug" { debugOption = true; }
"%standalone" { standalone = true; isInteger = true; }
"%switch" { packed = false; useRowMap = false; }
"%table" { packed = false; useRowMap = true; }
"%pack" { packed = true; useRowMap = true; }
"%include" {WSP}+ .* { File f = new File(yytext().substring(9).trim());
if ( !f.canRead() )
throw new ScannerException(file,ErrorMessages.NOT_READABLE, yyline);
// check for cycle
if (files.search(f) > 0)
throw new ScannerException(file,ErrorMessages.FILE_CYCLE, yyline);
try {
yypushStream( new FileReader(f) );
files.push(file);
file = f;
Out.println("Including \""+file+"\"");
}
catch (FileNotFoundException e) {
throw new ScannerException(file,ErrorMessages.NOT_READABLE, yyline);
}
}
"%buffer" {WSP}+ {Number} {WSP}* { bufferSize = Integer.parseInt(yytext().substring(8).trim()); }
"%buffer" {WSP}+ {NNL}* { throw new ScannerException(file,ErrorMessages.NO_BUFFER_SIZE, yyline); }
"%initthrow" {WSP}+ {QUIL} {WSP}* { initThrow = concExc(initThrow,yytext().substring(11).trim()); }
"%initthrow" {WSP}+ {NNL}* { throw new ScannerException(file,ErrorMessages.QUIL_INITTHROW, yyline); }
"%eofthrow" {WSP}+ {QUIL} {WSP}* { eofThrow = concExc(eofThrow,yytext().substring(10).trim()); }
"%eofthrow" {WSP}+ {NNL}* { throw new ScannerException(file,ErrorMessages.QUIL_EOFTHROW, yyline); }
"%yylexthrow"{WSP}+ {QUIL} {WSP}* { lexThrow = concExc(lexThrow,yytext().substring(12).trim()); }
"%throws" {WSP}+ {QUIL} {WSP}* { lexThrow = concExc(lexThrow,yytext().substring(8).trim()); }
"%yylexthrow"{WSP}+ {NNL}* { throw new ScannerException(file,ErrorMessages.QUIL_YYLEXTHROW, yyline); }
"%throws" {WSP}+ {NNL}* { throw new ScannerException(file,ErrorMessages.QUIL_THROW, yyline); }
"%scanerror" {WSP}+ {QualIdent} {WSP}* { scanErrorException = yytext().substring(11).trim(); }
"%scanerror" {WSP}+ {NNL}* { throw new ScannerException(file,ErrorMessages.QUIL_SCANERROR, yyline); }
{Ident} { return symbol(IDENT, yytext()); }
"="{WSP}* { yybegin(REGEXP); return symbol(EQUALS); }
"/*" { nextState = MACROS; yybegin(COMMENT); }
{EndOfLineComment} { }
/* no {NL} at the end of this expression, because <REGEXPSTART>
needs at least one {WSPNL} to start a regular expression! */
^"%%" {NNL}* { macroDefinition = false; yybegin(REGEXPSTART); return symbol(DELIMITER); }
"%"{Ident} { throw new ScannerException(file,ErrorMessages.UNKNOWN_OPTION, yyline, yycolumn); }
"%" { throw new ScannerException(file,ErrorMessages.UNKNOWN_OPTION, yyline, yycolumn); }
^{WSP}+"%" { Out.warning(ErrorMessages.NOT_AT_BOL, yyline); yypushback(1); }
{WSP}+ { }
{NL}+ { }
<<EOF>> { if ( yymoreStreams() ) {
file = (File) files.pop();
yypopStream();
}
else
throw new ScannerException(file,ErrorMessages.EOF_IN_MACROS);
}
}
<CTOR_ARG> {
{Ident} {WSP}* { yybegin(MACROS); ctorArgs.add(yytext().trim()); }
[^] { throw new ScannerException(file,ErrorMessages.CTOR_ARG,yyline,yycolumn); }
}
<REGEXPSTART> {
{WSPNL}* "/*" { nextState = REGEXPSTART; yybegin(COMMENT); }
{WSPNL}+ { yybegin(REGEXP); }
{WSPNL}* "<" { yybegin(STATES); return symbol_countUpdate(LESSTHAN, null); }
{WSPNL}* "}" { return symbol_countUpdate(RBRACE, null); }
{WSPNL}* "//" {NNL}* { }
{WSPNL}* "<<EOF>>" {WSPNL}* "{"
{ actionText.setLength(0); yybegin(JAVA_CODE);
Symbol s = symbol_countUpdate(EOFRULE, null);
action_line = s.left+1;
return s;
}
}
<STATES> {
{Ident} { return symbol(IDENT, yytext()); }
"," { return symbol(COMMA); }
{WSPNL}+ { }
// "{" will be caught in REGEXP
">"{WSPNL}* { yybegin(REGEXP); return symbol(MORETHAN); }
<<EOF>> { throw new ScannerException(file,ErrorMessages.EOF_IN_STATES); }
}
<REGEXP> {
"<<EOF>>" {WSPNL}+ "{" { actionText.setLength(0); yybegin(JAVA_CODE); action_line = yyline+1; return symbol(EOFRULE); }
"<<EOF>>" { throw new ScannerException(file,ErrorMessages.EOF_WO_ACTION); }
{WSPNL}*"|"{WSP}*$ { if (macroDefinition) {
yybegin(EATWSPNL);
return symbol(BAR);
}
else {
yybegin(REGEXPSTART);
return symbol(NOACTION);
}
}
// stategroup
"{" { yybegin(REGEXPSTART); return symbol(LBRACE); }
{WSPNL}*"|" { return symbol(BAR); }
{WSPNL}*\" { string.setLength(0); nextState = REGEXP; yybegin(STRING_CONTENT); }
{WSPNL}*"!" { return symbol(BANG); }
{WSPNL}*"~" { return symbol(TILDE); }
{WSPNL}*"(" { return symbol(OPENBRACKET); }
{WSPNL}*")" { return symbol(CLOSEBRACKET); }
{WSPNL}*"*" { return symbol(STAR); }
{WSPNL}*"+" { return symbol(PLUS); }
{WSPNL}*"?" { return symbol(QUESTION); }
{WSPNL}*"$" { return symbol(DOLLAR); }
{WSPNL}*"^" { bolUsed = true; return symbol(HAT); }
{WSPNL}*"." { return symbol(POINT); }
{WSPNL}*"[" { yybegin(CHARCLASS); return symbol(OPENCLASS); }
{WSPNL}*"/" { return symbol(LOOKAHEAD); }
{WSPNL}* "{" {WSP}* {Ident} {WSP}* "}" { return symbol_countUpdate(MACROUSE, makeMacroIdent()); }
{WSPNL}* "{" {WSP}* {Number} { yybegin(REPEATEXP); return symbol(REPEAT, new Integer(yytext().trim().substring(1).trim())); }
{WSPNL}+ "{" { actionText.setLength(0); yybegin(JAVA_CODE); action_line = yyline+1; return symbol(REGEXPEND); }
{NL} { if (macroDefinition) { yybegin(MACROS); } return symbol(REGEXPEND); }
{WSPNL}*"/*" { nextState = REGEXP; yybegin(COMMENT); }
{WSPNL}*"//"{NNL}* { }
{WSP}+ { }
<CHARCLASS> {
{WSPNL}*"[:jletter:]" { return symbol(JLETTERCLASS); }
{WSPNL}*"[:jletterdigit:]" { return symbol(JLETTERDIGITCLASS); }
{WSPNL}*"[:letter:]" { return symbol(LETTERCLASS); }
{WSPNL}*"[:digit:]" { return symbol(DIGITCLASS); }
{WSPNL}*"[:uppercase:]" { return symbol(UPPERCLASS); }
{WSPNL}*"[:lowercase:]" { return symbol(LOWERCLASS); }
{WSPNL}*"[:unicode_unassigned:]" { return symbol(UNICODE_UNASSIGNED); }
{WSPNL}*"[:unicode_uppercase_letter:]" { return symbol(UNICODE_UPPERCASE_LETTER); }
{WSPNL}*"[:unicode_lowercase_letter:]" { return symbol(UNICODE_LOWERCASE_LETTER); }
{WSPNL}*"[:unicode_titlecase_letter:]" { return symbol(UNICODE_TITLECASE_LETTER); }
{WSPNL}*"[:unicode_modifier_letter:]" { return symbol(UNICODE_MODIFIER_LETTER); }
{WSPNL}*"[:unicode_other_letter:]" { return symbol(UNICODE_OTHER_LETTER); }
{WSPNL}*"[:unicode_non_spacing_mark:]" { return symbol(UNICODE_NON_SPACING_MARK); }
{WSPNL}*"[:unicode_enclosing_mark:]" { return symbol(UNICODE_ENCLOSING_MARK); }
{WSPNL}*"[:unicode_combining_spacing_mark:]" { return symbol(UNICODE_COMBINING_SPACING_MARK); }
{WSPNL}*"[:unicode_decimal_digit_number:]" { return symbol(UNICODE_DECIMAL_DIGIT_NUMBER); }
{WSPNL}*"[:unicode_letter_number:]" { return symbol(UNICODE_LETTER_NUMBER); }
{WSPNL}*"[:unicode_other_number:]" { return symbol(UNICODE_OTHER_NUMBER); }
{WSPNL}*"[:unicode_space_separator:]" { return symbol(UNICODE_SPACE_SEPARATOR); }
{WSPNL}*"[:unicode_line_separator:]" { return symbol(UNICODE_LINE_SEPARATOR); }
{WSPNL}*"[:unicode_paragraph_separator:]" { return symbol(UNICODE_PARAGRAPH_SEPARATOR); }
{WSPNL}*"[:unicode_control:]" { return symbol(UNICODE_CONTROL); }
{WSPNL}*"[:unicode_format:]" { return symbol(UNICODE_FORMAT); }
{WSPNL}*"[:unicode_private_use:]" { return symbol(UNICODE_PRIVATE_USE); }
{WSPNL}*"[:unicode_surrogate:]" { return symbol(UNICODE_SURROGATE); }
{WSPNL}*"[:unicode_dash_punctuation:]" { return symbol(UNICODE_DASH_PUNCTUATION); }
{WSPNL}*"[:unicode_start_punctuation:]" { return symbol(UNICODE_START_PUNCTUATION); }
{WSPNL}*"[:unicode_end_punctuation:]" { return symbol(UNICODE_END_PUNCTUATION); }
{WSPNL}*"[:unicode_connector_punctuation:]" { return symbol(UNICODE_CONNECTOR_PUNCTUATION); }
{WSPNL}*"[:unicode_other_punctuation:]" { return symbol(UNICODE_OTHER_PUNCTUATION); }
{WSPNL}*"[:unicode_math_symbol:]" { return symbol(UNICODE_MATH_SYMBOL); }
{WSPNL}*"[:unicode_currency_symbol:]" { return symbol(UNICODE_CURRENCY_SYMBOL); }
{WSPNL}*"[:unicode_modifier_symbol:]" { return symbol(UNICODE_MODIFIER_SYMBOL); }
{WSPNL}*"[:unicode_other_symbol:]" { return symbol(UNICODE_OTHER_SYMBOL); }
{WSPNL}*"[:unicode_initial_quote_punctuation:]" { return symbol(UNICODE_INITIAL_QUOTE_PUNCTUATION); }
{WSPNL}*"[:unicode_final_quote_punctuation:]" { return symbol(UNICODE_FINAL_QUOTE_PUNCTUATION); }
}
. { return symbol(CHAR, new Character(yytext().charAt(0))); }
}
<EATWSPNL> {WSPNL}+ { yybegin(REGEXP); }
<REPEATEXP> {
"}" { yybegin(REGEXP); return symbol(RBRACE); }
"," {WSP}* {Number} { return symbol(REPEAT, new Integer(yytext().substring(1).trim())); }
{WSP}+ { }
<<EOF>> { throw new ScannerException(file,ErrorMessages.EOF_IN_REGEXP); }
}
<CHARCLASS> {
"{"{Ident}"}" { return symbol(MACROUSE, yytext().substring(1,yytext().length()-1)); }
"[" { balance++; return symbol(OPENCLASS); }
"]" { if (balance > 0) balance--; else yybegin(REGEXP); return symbol(CLOSECLASS); }
"^" { return symbol(HAT); }
"-" { return symbol(DASH); }
// this is a hack to keep JLex compatibilty with char class
// expressions like [+-]
"-]" { yypushback(1); yycolumn--; return symbol(CHAR, new Character(yytext().charAt(0))); }
\" { string.setLength(0); nextState = CHARCLASS; yybegin(STRING_CONTENT); }
. { return symbol(CHAR, new Character(yytext().charAt(0))); }
\n { throw new ScannerException(file,ErrorMessages.EOL_IN_CHARCLASS,yyline,yycolumn); }
<<EOF>> { throw new ScannerException(file,ErrorMessages.EOF_IN_REGEXP); }
}
<STRING_CONTENT> {
\" { yybegin(nextState); return symbol(STRING, string.toString()); }
\\\" { string.append('\"'); }
[^\"\\\u2028\u2029\u000A\u000B\u000C\u000D\u0085]+ { string.append(yytext()); }
{NL} { throw new ScannerException(file,ErrorMessages.UNTERMINATED_STR, yyline, yycolumn); }
{HexNumber} { string.append( (char) Integer.parseInt(yytext().substring(2,yytext().length()), 16)); }
{Unicode} { string.append( (char) Integer.parseInt(yytext().substring(2,yytext().length()), 16)); }
{OctNumber} { string.append( (char) Integer.parseInt(yytext().substring(1,yytext().length()), 8)); }
\\b { string.append('\b'); }
\\n { string.append('\n'); }
\\t { string.append('\t'); }
\\f { string.append('\f'); }
\\r { string.append('\r'); }
\\. { string.append(yytext().charAt(1)); }
<<EOF>> { throw new ScannerException(file,ErrorMessages.EOF_IN_STRING); }
}
<REGEXP, CHARCLASS> {
{HexNumber} { return symbol(CHAR, new Character( (char) Integer.parseInt(yytext().substring(2,yytext().length()), 16))); }
{Unicode} { return symbol(CHAR, new Character( (char) Integer.parseInt(yytext().substring(2,yytext().length()), 16))); }
{OctNumber} { return symbol(CHAR, new Character( (char) Integer.parseInt(yytext().substring(1,yytext().length()), 8))); }
\\b { return symbol(CHAR,new Character('\b')); }
\\n { return symbol(CHAR,new Character('\n')); }
\\t { return symbol(CHAR,new Character('\t')); }
\\f { return symbol(CHAR,new Character('\f')); }
\\r { return symbol(CHAR,new Character('\r')); }
\\. { return symbol(CHAR, new Character(yytext().charAt(1))); }
}
<JAVA_CODE> {
"{" { balance++; actionText.append('{'); }
"}" { if (balance > 0) {
balance--;
actionText.append('}');
}
else {
yybegin(REGEXPSTART);
Action a = new Action(actionText.toString(), action_line);
actions.addElement(a);
return symbol(ACTION, a);
}
}
{JavaCode} { actionText.append(yytext()); }
<<EOF>> { throw new ScannerException(file,ErrorMessages.EOF_IN_ACTION, action_line-1); }
}
<COMMENT> {
"/"+ "*" { commentbalance++; }
"*"+ "/" { if (commentbalance > 0)
commentbalance--;
else
yybegin(nextState);
}
{JFlexComment} { /* ignore */ }
<<EOF>> { throw new ScannerException(file,ErrorMessages.EOF_IN_COMMENT); }
}
. { throw new ScannerException(file,ErrorMessages.UNEXPECTED_CHAR, yyline, yycolumn); }
\n { throw new ScannerException(file,ErrorMessages.UNEXPECTED_NL, yyline, yycolumn); }
<<EOF>> { if ( yymoreStreams() ) {
file = (File) files.pop();
yypopStream();
}
else
return symbol(EOF); }
tools/lexer/jflex-1.4/src/JFlex/LexScan.java
-3404
View File
File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More