Skip to content

GitLab

Commit 2ad75270 authored by Arpad Lovassy's avatar Arpad Lovassy
Browse files

Parse tree logger: removed ...LexerLogUtil related dependencies and class generation script 


Signed-off-by: Arpad Lovassy's avatarzlovarp <arpad.lovassy@semcon.com>
parent 385eaf14
Hide whitespace changes
Inline Side-by-side
Tools/antlr4_compile.cmd
View file @ 2ad75270
......@@ -87,9 +87,5 @@
@java %ANTLR% VariantAttributeLexer.g4 -no-listener -no-visitor -encoding UTF-8 -package org.eclipse.titan.designer.parsers.variantattributeparser
@java %ANTLR% VariantAttributeParser.g4 -no-listener -no-visitor -encoding UTF-8 -package org.eclipse.titan.designer.parsers.variantattributeparser
@echo Generating ...LexerLogUtil.java files from ...Lexer.java files for resolving token names (OPTIONAL)
@cd %DIR%
@perl antlr4_generate_lexerlogutil.pl
@cd %CURDIR%
Tools/antlr4_compile.sh
View file @ 2ad75270
......@@ -84,7 +84,3 @@ cd $WORKSPACE_PATH/org.eclipse.titan.designer/src/org/eclipse/titan/designer/par
$ANTLR4 VariantAttributeLexer.g4 -no-listener -no-visitor -encoding UTF-8 -package org.eclipse.titan.designer.parsers.variantattributeparser
$ANTLR4 VariantAttributeParser.g4 -no-listener -no-visitor -encoding UTF-8 -package org.eclipse.titan.designer.parsers.variantattributeparser
# Generating ...LexerLogUtil.java files from ...Lexer.java files for resolving token names (OPTIONAL)
cd $WORKSPACE_PATH
$DIR/antlr4_generate_lexerlogutil.pl
Tools/antlr4_generate_lexerlogutil.pl deleted 100755 → 0
View file @ 385eaf14
#!/usr/bin/perl
###############################################################################
# Copyright (c) 2000-2021 Ericsson Telecom AB
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Eclipse Public License v2.0
# which accompanies this distribution, and is available at
# https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.html
#
# Contributors:
# Lovassy, Arpad
#
###############################################################################
###############################################################################
# Generates ...LexerLogUtil.java implements ILexerLogUtil
# for all ...Lexer.java under the current directory
# to determine token name from token index.
# This is used for logging purpose,
# see ParserLogger.java and TokenNameResolver.java
#
# Prerequisites: ANTLR lexer .g4 files must be compiled,
# because generated java files must exist.
#
# Example usage (to create ...LexerLogUtil.java for all the ...Lexer.java
# in titan.EclipsePlug-ins project):
# cd <titan.EclipsePlug-ins project root>
# Tools/antlr4_generate_lexerlogutil.pl
###############################################################################
# to read the whole file at once, not just line-by-line
# http://www.kichwa.com/quik_ref/spec_variables.html
undef $/;
$fileindex = 0;
#total number of changed files
$total = 0;
sub load
{
local $filename = shift; # 1st parameter
local $f = "F".$fileindex++;
if(opendir($f, $filename))
{
# $filename is a directory
while(local $newfilename = readdir($f))
{
if(!($newfilename =~ /^\.(\.|git)?$/)) # filter . .. .git
{
local $longfilename = $filename."/".$newfilename; # for Unix
#local $longfilename = $filename."\\".$newfilename; # for DOS/Windows
load($longfilename);
}
}
closedir($f);
}
else
{
# $filename is a file
if($filename =~ /^(.*\/)(Asn1|Cfg|Ttcn3|ExtensionAttribute|VariantAttribute|PreprocessorDirective)(Lexer)(\.java)$/)
{
$file_part1 = $1;
$file_part2 = $2;
$file_part3 = $3;
$file_part4 = "LogUtil";
$file_part5 = $4;
print("Source file: $filename\n");
open(IN, $filename);
$whole_file = <IN>;
close(IN);
$outfilename = $file_part1.$file_part2.$file_part3.$file_part4.$file_part5;
print("Output file: $outfilename\n");
open(OUT, ">$outfilename");
if ( $whole_file =~ /package\s+([a-zA-Z_][a-zA-Z_0-9]*(\.[a-zA-Z_][a-zA-Z_0-9]*)*);/gs ) {
$package = $1;
print "Package: $package\n";
} else {
print "ERROR: package missing";
}
$out =
"package $package;\n".
"\n".
"import org.eclipse.titan.common.parsers.ILexerLogUtil;\n".
"\n".
"public class $file_part2$file_part3$file_part4 implements ILexerLogUtil {\n".
"\n".
" public String getTokenName( int aIndex ) {\n".
" switch ( aIndex ) {\n";
if ( $whole_file =~ /public static final int\r?\n\s*(.*?);/gs ) {
$const_defs = $1; #comma separated constant definition: WS=1, LINE_COMMENT=2, ..., MACRO12=451
# print "\$const_defs == $const_defs\n";
my @list = split(/,\s*/, $const_defs);
foreach my $const_def (@list) {
# print "\$const_def == \"$const_def\"\n";
if ( $const_def =~ /(^[A-Za-z][A-Za-z0-9_]*)=([0-9]+)$/ ) {
$const_name = $1;
$const_value = $2;
# print "\$const_name == \"$const_name\"\n";
# print "\$const_value == \"$const_value\"\n";
$out .=
" case $const_value:\n".
" return \"$const_name\";\n";
}
else {
print "ERROR: $const_def does NOT match!\n";
}
}
}
$out .=
" case -1:\n".
" return \"EOF\";\n".
" default:\n".
" return \"\" + aIndex;\n".
" }\n".
" }\n".
"}\n".
"";
print OUT $out;
close(OUT);
print("DONE\n");
}
}
}
load("..");
org.eclipse.titan.common/src/org/eclipse/titan/common/parsers/ILexerLogUtil.java deleted 100644 → 0
View file @ 385eaf14
/******************************************************************************
* Copyright (c) 2000-2021 Ericsson Telecom AB
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v2.0
* which accompanies this distribution, and is available at
* https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.html
******************************************************************************/
package org.eclipse.titan.common.parsers;
/**
* USED FOR LOGGING PURPOSES
* Interface for token name resolving
* @see TokenNameResolver
* @author Arpad Lovassy
*/
public interface ILexerLogUtil {
/**
* @param aIndex token type index
* @return resolved token name
*/
public String getTokenName( int aIndex );
}
org.eclipse.titan.common/src/org/eclipse/titan/common/parsers/ParserLogger.java
View file @ 2ad75270
......@@ -15,13 +15,13 @@ import java.util.Date;
import java.util.List;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.NoViableAltException;
import org.antlr.v4.runtime.Parser;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.Vocabulary;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.TerminalNodeImpl;
import org.eclipse.titan.common.parsers.cfg.CfgInterval;
......@@ -32,6 +32,7 @@ import org.eclipse.titan.common.parsers.cfg.CfgInterval;
* @author Arpad Lovassy
*/
public final class ParserLogger {
/** date format in ISO 8601 */
private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
......@@ -47,17 +48,13 @@ public final class ParserLogger {
/**
* Logs a parse tree. (General version)
* Token name is resolved if ...LexerLogUtil.java is generated,
* otherwise only token type index is displayed
* @param aRoot parse tree
* @param aParser parser to get rule names
* @param aTokenNameResolver resolver to get token name
* @param aPrinter printer
* @param aDescription description of the parsing type, for the logging (for example: TTCN-3, Cfg, ASN.1)
*/
public static synchronized void log( final ParseTree aRoot,
final Parser aParser,
final TokenNameResolver aTokenNameResolver,
final IPrinter aPrinter,
final String aDescription ) {
if ( !aParser.getBuildParseTree() ) {
......@@ -76,21 +73,7 @@ public final class ParserLogger {
}
final CommonTokenStream commonTokenStream = (CommonTokenStream)tokenStream;
final List<Token> tokens = commonTokenStream.getTokens();
log( aRoot, aParser, tokens, aTokenNameResolver );
}
/**
* Logs a parse tree
* @param aRoot parse tree
* @param aParser parser to get rule names
* @param aTokens token list to get tokens by index (for getting tokens of a rule)
* @param aLexerLogUtil resolver to get token name by method 1, see TokenNameResolver
*/
public static void log( final ParseTree aRoot,
final Parser aParser,
final List<Token> aTokens,
final ILexerLogUtil aLexerLogUtil ) {
log( aRoot, aParser, aTokens, new TokenNameResolver( aLexerLogUtil ) );
log( aRoot, aParser, tokens );
}
/**
......@@ -98,40 +81,12 @@ public final class ParserLogger {
* @param aRoot parse tree
* @param aParser parser to get rule names
* @param aTokens token list to get tokens by index (for getting tokens of a rule)
* @param aLexer resolver to get token name by method 2, see TokenNameResolver
*/
public static void log( final ParseTree aRoot,
final Parser aParser,
final List<Token> aTokens,
final Lexer aLexer ) {
log( aRoot, aParser, aTokens, new TokenNameResolver( aLexer ) );
}
/**
* Logs a parse tree, token name is not resolved, only token type index is displayed
* @param aRoot parse tree
* @param aParser parser to get rule names
* @param aTokens token list to get tokens by index (for getting tokens of a rule)
*/
public static void log( final ParseTree aRoot,
final Parser aParser,
final List<Token> aTokens ) {
log( aRoot, aParser, aTokens, new TokenNameResolver() );
}
/**
* Logs a parse tree
* @param aRoot parse tree
* @param aParser parser to get rule names
* @param aTokens token list to get tokens by index (for getting tokens of a rule)
* @param aTokenNameResolver resolver to get token name
*/
private static void log( final ParseTree aRoot,
final Parser aParser,
final List<Token> aTokens,
final TokenNameResolver aTokenNameResolver ) {
try {
log( aRoot, aParser, aTokens, aTokenNameResolver, new ArrayList<Boolean>(), false, false );
log( aRoot, aParser, aTokens, new ArrayList<Boolean>(), false, false );
} catch( Exception e ) {
final StringWriter sw = new StringWriter();
final PrintWriter pw = new PrintWriter( sw );
......@@ -147,7 +102,6 @@ public final class ParserLogger {
* @param aRoot parse tree
* @param aParser parser to get rule name
* @param aTokens token list to get tokens by index (for getting tokens of a rule)
* @param aTokenNameResolver resolver to get token name
* @param aLevel a list, that tells, if tree section is drawn for that level (parent).
* If the parent of the given level is already the last child, tree sections are not drawn below.
* NOTE: indentation level is aLevel.size()
......@@ -157,7 +111,6 @@ public final class ParserLogger {
private static void log( final ParseTree aRoot,
final Parser aParser,
final List<Token> aTokens,
final TokenNameResolver aTokenNameResolver,
final List<Boolean> aLevel,
final boolean aParentOneChild,
final boolean aLastChild ) {
......@@ -172,7 +125,7 @@ public final class ParserLogger {
if ( aRoot instanceof ParserRuleContext ) {
final ParserRuleContext rule = (ParserRuleContext)aRoot;
final String ruleInfo = getRuleInfo( rule, aParser, aTokenNameResolver );
final String ruleInfo = getRuleInfo( rule, aParser );
if ( aParentOneChild ) {
printArrow( ruleInfo );
} else {
......@@ -182,7 +135,7 @@ public final class ParserLogger {
final int count = rule.getChildCount();
final boolean oneChild = count == 1 && rule.exception == null;
if ( !oneChild ) {
println( ": '" + getEscapedRuleText( rule, aTokens ) + "'" + getExceptionInfo( rule, aTokenNameResolver ) );
println( ": '" + getEscapedRuleText( rule, aTokens ) + "'" + getExceptionInfo( rule, aParser ) );
}
for( int i = 0; i < count; i++ ) {
......@@ -195,7 +148,7 @@ public final class ParserLogger {
}
level.add( true );
}
log( child, aParser, aTokens, aTokenNameResolver, level, oneChild, i == count - 1 );
log( child, aParser, aTokens, level, oneChild, i == count - 1 );
}
} else if ( aRoot instanceof TerminalNodeImpl ) {
final TerminalNodeImpl tn = (TerminalNodeImpl)aRoot;
......@@ -203,7 +156,7 @@ public final class ParserLogger {
println( ": '" + getEscapedTokenText( tn.getSymbol() ) + "'" );
}
printIndent( getTokenInfo( tn.getSymbol(), aTokenNameResolver ), aLevel );
printIndent( getTokenInfo( tn.getSymbol(), aParser ), aLevel );
if ( tn.parent == null ) {
print(", parent == null <-------------------------------------------------------------- ERROR");
}
......@@ -226,10 +179,10 @@ public final class ParserLogger {
/**
* Rule exception info in string format for logging purpose
* @param aRule rule
* @param aTokenNameResolver resolver to get token name
* @param aParser parser to get token name
* @return exception stack trace + some other info from the exception object
*/
private static String getExceptionInfo( final ParserRuleContext aRule, final TokenNameResolver aTokenNameResolver ) {
private static String getExceptionInfo( final ParserRuleContext aRule, final Parser aParser ) {
final RecognitionException e = aRule.exception;
if ( e == null ) {
return "";
......@@ -242,12 +195,12 @@ public final class ParserLogger {
sb.append(", ");
}
final int tokenType = expectedTokens.get( i );
sb.append( getTokenName( tokenType, aTokenNameResolver ) );
sb.append( getTokenName( tokenType, aParser ) );
}
sb.append(']');
if ( e instanceof NoViableAltException ) {
final NoViableAltException nvae = (NoViableAltException)e;
sb.append( ", start token: " + getTokenInfo( nvae.getStartToken(), aTokenNameResolver ) );
sb.append( ", start token: " + getTokenInfo( nvae.getStartToken(), aParser ) );
}
return sb.toString();
......@@ -257,10 +210,9 @@ public final class ParserLogger {
* Rule info in string format for logging purpose
* @param aRule rule
* @param aParser parser to get rule name
* @param aTokenNameResolver resolver to get token name (used only if rule is erroneous)
* @return rule name and exception and some other info if rule is erroneous
*/
private static String getRuleInfo( final ParserRuleContext aRule, final Parser aParser, final TokenNameResolver aTokenNameResolver ) {
private static String getRuleInfo( final ParserRuleContext aRule, final Parser aParser ) {
// only rule name
final String info = aParser.getRuleInvocationStack( aRule ).get( 0 );
return info;
......@@ -333,7 +285,7 @@ public final class ParserLogger {
/**
* Token info in string format for logging purpose
* @param aToken token
* @param aTokenNameResolver resolver to get token name
* @param aParser parser to get token name
* @return &lt;token name&gt;: '&lt;token text&gt;', @&lt;token index&gt;, &lt;line&gt;:&lt;column&gt;[, channel=&lt;channel&gt;]
* <br>where
* <br>&lt;token index&gt; starts from 0,
......@@ -341,10 +293,10 @@ public final class ParserLogger {
* <br>&lt;column&gt; starts from 0,
* <br>channel info is provided if &lt;channel&gt; > 0 (hidden channel)
*/
private static String getTokenInfo( final Token aToken, final TokenNameResolver aTokenNameResolver ) {
private static String getTokenInfo( final Token aToken, final Parser aParser ) {
final StringBuilder sb = new StringBuilder();
final int tokenType = aToken.getType();
final String tokenName = getTokenName( tokenType, aTokenNameResolver );
final String tokenName = getTokenName( tokenType, aParser );
sb.append( tokenName );
sb.append( ": " );
......@@ -390,22 +342,13 @@ public final class ParserLogger {
/**
* @param aTokenType token type index
* @param aTokenNameResolver token name resolver
* @param aParser parser to resolve token name
* @return resolved token name
*/
private static String getTokenName( final int aTokenType, TokenNameResolver aTokenNameResolver ) {
/*
TODO: use Vocabulary interface when ANTLR 4.5 is used
See https://github.com/antlr/antlr4/pull/712
Vocabulary vocabulary = recognizer.getVocabulary();
private static String getTokenName( final int aTokenType, final Parser aParser ) {
final Vocabulary vocabulary = aParser.getVocabulary();
final String symbolicName = vocabulary.getSymbolicName( aTokenType );
return symbolicName;
*/
if ( aTokenNameResolver == null ) {
aTokenNameResolver = new TokenNameResolver();
}
return aTokenNameResolver.getTokenName( aTokenType );
}
/**
......
org.eclipse.titan.common/src/org/eclipse/titan/common/parsers/TokenNameResolver.java deleted 100644 → 0
View file @ 385eaf14
/******************************************************************************
* Copyright (c) 2000-2021 Ericsson Telecom AB
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v2.0
* which accompanies this distribution, and is available at
* https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.html
******************************************************************************/
package org.eclipse.titan.common.parsers;
import org.antlr.v4.runtime.Lexer;
/**
* USED FOR LOGGING PURPOSES.<br>
* Utility class to get the token name from token index for logging purpose.<br>
* Token name can be calculated in 2 ways:
* <ol>
* <li> Generate ILexerLogUtil for each lexer from ...Lexer.java,
* see <titan.EclipsePlug-ins root>/Tools/antlr4_generate_logutil.pl<br>
* This is preferred, it is accurate
* <li> Get token name like this: aTokenType == -1 ? "EOF" : aLexer.getRuleNames()[ aTokenType - 1 ]<br>
* There are 2 problems with it:
* <ol>
* <li> fragments are not tokens, but they are listed in ruleNames[], for example: CfgLexer FR_NUMBER1
* <li> tokens, which are defined, but has no rule are not listed in ruleNames[], for example: Ttcn3Lexer MACRO
* </ol>
* so after these cases the lexer rule names are in wrong position, it is a bug in ANTLR 4.4<br>
* But it can be still used in general cases if no ILexerLogUtil is generated
* </ol>
* @author Arpad Lovassy
*/
public class TokenNameResolver {
/**
* token name resolver for case 1
*/
private ILexerLogUtil mLexerLogUtil = null;
/**
* token name resolver for case 2
*/
private Lexer mLexer = null;
/**
* Default constructor, in this case no resolving is made, token index is displayed
*/
public TokenNameResolver() {
}
/**
* Constructor for case 1
* @param aLexerLogUtil token name resolver for case 1
*/
public TokenNameResolver( final ILexerLogUtil aLexerLogUtil ) {
mLexerLogUtil = aLexerLogUtil;
}
/**
* Constructor for case 2
* @param aLexerLogUtil token name resolver for case 2
*/
public TokenNameResolver( final Lexer aLexer ) {
mLexer = aLexer;
}
/**
* @param aTokenType token type, possible values: -1 (EOF), otherwise token types start from 1
* @return resolved token name
*/
public String getTokenName( final int aTokenType ) {
if ( mLexerLogUtil != null ) {
return mLexerLogUtil.getTokenName( aTokenType );
} else if ( mLexer != null ) {
return aTokenType == -1 ? "EOF" : mLexer.getRuleNames()[ aTokenType - 1 ];
} else {
return "" + aTokenType;
}
}
}
org.eclipse.titan.designer/src/org/eclipse/titan/designer/parsers/ParserUtilities.java
View file @ 2ad75270
......@@ -17,10 +17,8 @@ import org.antlr.v4.runtime.TokenStream;
import org.antlr.v4.runtime.tree.ParseTree;
import org.eclipse.core.resources.IFile;
import org.eclipse.titan.common.logging.ErrorReporter;
import org.eclipse.titan.common.parsers.ILexerLogUtil;
import org.eclipse.titan.common.parsers.IPrinter;
import org.eclipse.titan.common.parsers.ParserLogger;
import org.eclipse.titan.common.parsers.TokenNameResolver;
import org.eclipse.titan.common.parsers.cfg.CfgParser;
import org.eclipse.titan.designer.AST.Location;
import org.eclipse.titan.designer.parsers.asn1parser.Asn1Parser;
......@@ -68,8 +66,6 @@ public class ParserUtilities {
/**
* Logs a parse tree.
* Token name is resolved if ...LexerLogUtil.java is generated,
* otherwise only token type index is displayed
* @param aRoot parse tree
* @param aParser parser to get rule names
*/
......@@ -80,10 +76,8 @@ public class ParserUtilities {
return;
}
final String description = getDescription( aParser );
final String lexerLogUtilClassName = getLexerLogUtilClassName( aParser );
final IPrinter p = TitanDebugConsolePrinter.INSTANCE;
final TokenNameResolver tokenNameResolver = getTokenNameResolver( lexerLogUtilClassName, p );
ParserLogger.log( aRoot, aParser, tokenNameResolver, p, description );
ParserLogger.log( aRoot, aParser, p, description );
}
......@@ -111,48 +105,6 @@ public class ParserUtilities {
}
}
/**
* @param aParser parser instance
* @return gets the ...LexerLogUtil class name for the parser instance for token name resolving
*/
private static String getLexerLogUtilClassName( final Parser aParser ) {
if ( aParser == null ) {
return null;
} else if ( aParser instanceof Ttcn3Parser ) {
return "org.eclipse.titan.designer.parsers.ttcn3parser.Ttcn3LexerLogUtil";
} else if ( aParser instanceof Ttcn3Reparser ) {
return "org.eclipse.titan.designer.parsers.ttcn3parser.Ttcn3LexerLogUtil";
} else if ( aParser instanceof Asn1Parser ) {
return "org.eclipse.titan.designer.parsers.asn1parser.Asn1LexerLogUtil";
} else if ( aParser instanceof ExtensionAttributeParser ) {
return "org.eclipse.titan.designer.parsers.extensionattributeparser.ExtensionAttributeLexerLogUtil";
} else if ( aParser instanceof PreprocessorDirectiveParser ) {
return "org.eclipse.titan.designer.parsers.ttcn3parser.PreprocessorDirectiveLexerLogUtil";
} else if ( aParser instanceof CfgParser ) {
return "org.eclipse.titan.common.parsers.cfg.CfgLexerLogUtil";
} else {
return null;
}
}
/**
* @param aLexerLogUtilClassName the full class name of the ...LexerLogUtil class. It is instantiated dynamically if exists.
* @param aPrinter printer for error message
* @return object to resolve lexer token names from token index
*/
private static TokenNameResolver getTokenNameResolver( final String aLexerLogUtilClassName, final IPrinter aPrinter ) {
try {
// ...LexerLogUtil is generated, it may not exist
final Class<?> c = Class.forName( aLexerLogUtilClassName );
final ILexerLogUtil o = (ILexerLogUtil)c.newInstance();
return new TokenNameResolver( o );
} catch ( final Exception e ) {
aPrinter.println("ERROR: File does NOT exist: " + aLexerLogUtilClassName);
aPrinter.println(" Run <titan.EclipsePlug-ins root>/Tools/antlr4_generate_lexerlogutil.pl to generate");
return new TokenNameResolver();
}
}
/**
* Get comments before a token (TTCN-3)
* @param aToken the token, this will NOT be printed
......
org.eclipse.titan.designer/src/org/eclipse/titan/designer/parsers/asn1parser/.gitignore
View file @ 2ad75270
/Asn1Lexer.java
/Asn1Lexer.tokens
/Asn1LexerLogUtil.java
/Asn1Parser.java
/Asn1Parser.tokens
/Asn1ParserBaseListener.java
......
org.eclipse.titan.designer/src/org/eclipse/titan/designer/parsers/extensionattributeparser/.gitignore
View file @ 2ad75270
......@@ -2,7 +2,6 @@
/ExtensionAttributeLexer.tokens
/ExtensionAttributeParser.java
/ExtensionAttributeParser.tokens
/ExtensionAttributeLexerLogUtil.java