Commit 1fb9fa44 authored by earplov's avatar earplov
Browse files

bugfix: remove unnecessary CommonTokenStream dependency, and just token list is used instead


Signed-off-by: default avatarearplov <arpad.lovassy@ericsson.com>
parent 6c14a018
...@@ -166,8 +166,8 @@ public final class CfgAnalyzer { ...@@ -166,8 +166,8 @@ public final class CfgAnalyzer {
// pr_PatternChunk[StringBuilder builder, boolean[] uni]: // pr_PatternChunk[StringBuilder builder, boolean[] uni]:
// $builder.append($v.text); <-- exception is thrown here: java.lang.UnsupportedOperationException: interval 85..85 not in token buffer window: 86..341 // $builder.append($v.text); <-- exception is thrown here: java.lang.UnsupportedOperationException: interval 85..85 not in token buffer window: 86..341
// 2. Changed from BufferedTokenStream to CommonTokenStream, otherwise tokens with "-> channel(HIDDEN)" are not filtered out in lexer. // 2. Changed from BufferedTokenStream to CommonTokenStream, otherwise tokens with "-> channel(HIDDEN)" are not filtered out in lexer.
final CommonTokenStream tokens = new CommonTokenStream( lexer ); final CommonTokenStream tokenStream = new CommonTokenStream( lexer );
final CfgParser parser = new CfgParser(tokens); final CfgParser parser = new CfgParser( tokenStream );
//parser tree is built by default //parser tree is built by default
parserListener = new TitanListener(); parserListener = new TitanListener();
parser.removeErrorListeners(); // remove ConsoleErrorListener parser.removeErrorListeners(); // remove ConsoleErrorListener
...@@ -179,7 +179,7 @@ public final class CfgAnalyzer { ...@@ -179,7 +179,7 @@ public final class CfgAnalyzer {
// manually add the result parse tree, and its corresponding token stream, // manually add the result parse tree, and its corresponding token stream,
// because they logically belong to here // because they logically belong to here
mCfgParseResult.setParseTreeRoot( parseTreeRoot ); mCfgParseResult.setParseTreeRoot( parseTreeRoot );
mCfgParseResult.setTokenStream( tokens ); mCfgParseResult.setTokens( tokenStream.getTokens() );
// fill handlers // fill handlers
moduleParametersHandler = parser.getModuleParametersHandler(); moduleParametersHandler = parser.getModuleParametersHandler();
......
...@@ -12,8 +12,8 @@ import java.util.HashMap; ...@@ -12,8 +12,8 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.eclipse.titan.common.parsers.TITANMarker; import org.eclipse.titan.common.parsers.TITANMarker;
/** /**
...@@ -30,7 +30,7 @@ public class CfgParseResult { ...@@ -30,7 +30,7 @@ public class CfgParseResult {
* It contains all the tokens from all the channels, so it contains also the hidden tokens as well. * It contains all the tokens from all the channels, so it contains also the hidden tokens as well.
* A token is hidden, if token channel > 0 * A token is hidden, if token channel > 0
*/ */
private CommonTokenStream mTokenStream = null; private List<Token> mTokens = null;
/** /**
* list of syntactic warnings collected during parsing * list of syntactic warnings collected during parsing
...@@ -105,12 +105,12 @@ public class CfgParseResult { ...@@ -105,12 +105,12 @@ public class CfgParseResult {
this.mParseTreeRoot = aParseTreeRoot; this.mParseTreeRoot = aParseTreeRoot;
} }
public CommonTokenStream getTokenStream() { public List<Token> getTokens() {
return mTokenStream; return mTokens;
} }
public void setTokenStream(CommonTokenStream aTokenStream) { public void setTokens( List<Token> aTokens ) {
this.mTokenStream = aTokenStream; this.mTokens = aTokens;
} }
public List<TITANMarker> getWarnings() { public List<TITANMarker> getWarnings() {
......
...@@ -15,7 +15,6 @@ import java.util.Map.Entry; ...@@ -15,7 +15,6 @@ import java.util.Map.Entry;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.ParseTree;
...@@ -160,15 +159,15 @@ public class CfgParseTreePrinter { ...@@ -160,15 +159,15 @@ public class CfgParseTreePrinter {
* @param aParseTreeRoot root of the parse tree to print * @param aParseTreeRoot root of the parse tree to print
* @param aPrintHiddenBefore true to print hidden tokens before the parse tree * @param aPrintHiddenBefore true to print hidden tokens before the parse tree
* (NOTE: hidden tokens in the parse tree will be printed) * (NOTE: hidden tokens in the parse tree will be printed)
* @param aTokenStream token stream to get the tokens from (all, hidden and not hidden also) * @param aTokens token list from the lexer (all, hidden and not hidden also)
* @return output parse tree text * @return output parse tree text
*/ */
public static String toStringWithHidden( final ParseTree aParseTreeRoot, public static String toStringWithHidden( final ParseTree aParseTreeRoot,
final CommonTokenStream aTokenStream, final List<Token> aTokens,
final boolean aPrintHiddenBefore ) { final boolean aPrintHiddenBefore ) {
final StringBuilder sb = new StringBuilder(); final StringBuilder sb = new StringBuilder();
CfgParseTreePrinter printer = new CfgParseTreePrinter( sb ); CfgParseTreePrinter printer = new CfgParseTreePrinter( sb );
printer.print( aParseTreeRoot, aTokenStream, aPrintHiddenBefore, ResolveMode.NO_RESOLVING, null ); printer.print( aParseTreeRoot, aTokens, aPrintHiddenBefore, ResolveMode.NO_RESOLVING, null );
// there are no hidden tokens after the last token // there are no hidden tokens after the last token
return sb.toString(); return sb.toString();
...@@ -206,7 +205,7 @@ public class CfgParseTreePrinter { ...@@ -206,7 +205,7 @@ public class CfgParseTreePrinter {
aCfgParseResults, aDefinitions, aEnvVariables, filesToResolve ); aCfgParseResults, aDefinitions, aEnvVariables, filesToResolve );
for ( Entry<Path, CfgParseResult> entry : aCfgParseResults.entrySet() ) { for ( Entry<Path, CfgParseResult> entry : aCfgParseResults.entrySet() ) {
printer.printResolved( entry.getKey(), entry.getValue().getParseTreeRoot(), printer.printResolved( entry.getKey(), entry.getValue().getParseTreeRoot(),
entry.getValue().getTokenStream(), aResolveMode ); entry.getValue().getTokens(), aResolveMode );
} }
break; break;
} }
...@@ -219,7 +218,7 @@ public class CfgParseTreePrinter { ...@@ -219,7 +218,7 @@ public class CfgParseTreePrinter {
CfgParseTreePrinter printer = new CfgParseTreePrinter( aSb, aDisallowedNodes, CfgParseTreePrinter printer = new CfgParseTreePrinter( aSb, aDisallowedNodes,
aCfgParseResults, aDefinitions, aEnvVariables, filesToResolve ); aCfgParseResults, aDefinitions, aEnvVariables, filesToResolve );
printer.printResolved( entry.getKey(), entry.getValue().getParseTreeRoot(), printer.printResolved( entry.getKey(), entry.getValue().getParseTreeRoot(),
entry.getValue().getTokenStream(), aResolveMode ); entry.getValue().getTokens(), aResolveMode );
break; break;
} }
case NO_RESOLVING: case NO_RESOLVING:
...@@ -242,11 +241,11 @@ public class CfgParseTreePrinter { ...@@ -242,11 +241,11 @@ public class CfgParseTreePrinter {
*/ */
private void printResolved( final Path aFile, private void printResolved( final Path aFile,
final ParseTree aParseTreeRoot, final ParseTree aParseTreeRoot,
final CommonTokenStream aTokenStream, final List<Token> aTokens,
final ResolveMode aResolveMode ) { final ResolveMode aResolveMode ) {
if ( mFilesToResolve.contains( aFile ) ) { if ( mFilesToResolve.contains( aFile ) ) {
mSb.append( INCLUDED_BEGIN ).append( aFile.toOSString()).append('\n'); mSb.append( INCLUDED_BEGIN ).append( aFile.toOSString()).append('\n');
print( aParseTreeRoot, aTokenStream, true, aResolveMode, aFile ); print( aParseTreeRoot, aTokens, true, aResolveMode, aFile );
mSb.append( INCLUDED_END ).append( aFile.toOSString()).append('\n'); mSb.append( INCLUDED_END ).append( aFile.toOSString()).append('\n');
mFilesToResolve.remove( aFile ); mFilesToResolve.remove( aFile );
} }
...@@ -263,7 +262,7 @@ public class CfgParseTreePrinter { ...@@ -263,7 +262,7 @@ public class CfgParseTreePrinter {
* needed only if aResolveMode != NO_RESOLVING, in case of [ORDERED_INCLUDE] * needed only if aResolveMode != NO_RESOLVING, in case of [ORDERED_INCLUDE]
*/ */
private void print( final ParseTree aParseTree, private void print( final ParseTree aParseTree,
final CommonTokenStream aTokenStream, final List<Token> aTokens,
final boolean aPrintHiddenBefore, final boolean aPrintHiddenBefore,
final ResolveMode aResolveMode, final ResolveMode aResolveMode,
final Path aFile ) { final Path aFile ) {
...@@ -278,15 +277,15 @@ public class CfgParseTreePrinter { ...@@ -278,15 +277,15 @@ public class CfgParseTreePrinter {
} }
if ( aPrintHiddenBefore && rule.getChildCount() > 0 && rule.getChild( 0 ) instanceof AddedParseTree ) { if ( aPrintHiddenBefore && rule.getChildCount() > 0 && rule.getChild( 0 ) instanceof AddedParseTree ) {
// special case: if AddedParseTree is the 1st in the rule, it has no information // special case: if AddedParseTree is the 1st in the rule, it has no information
// about the hidden tokens, as it has no position in the token stream, but the rule may have // about the hidden tokens, as it has no position in the token list, but the rule may have
printHiddenTokensBefore( rule, aTokenStream ); printHiddenTokensBefore( rule, aTokens );
} }
} }
else if ( aParseTree instanceof TerminalNodeImpl ) { else if ( aParseTree instanceof TerminalNodeImpl ) {
final TerminalNodeImpl tn = (TerminalNodeImpl)aParseTree; final TerminalNodeImpl tn = (TerminalNodeImpl)aParseTree;
final Token token = tn.getSymbol(); final Token token = tn.getSymbol();
if ( mDisallowedNodes == null || !mDisallowedNodes.contains( token.getType() ) ) { if ( mDisallowedNodes == null || !mDisallowedNodes.contains( token.getType() ) ) {
printToken( token, aTokenStream, aPrintHiddenBefore, aResolveMode, aFile ); printToken( token, aTokens, aPrintHiddenBefore, aResolveMode, aFile );
} }
} }
else if ( aParseTree instanceof AddedParseTree ) { else if ( aParseTree instanceof AddedParseTree ) {
...@@ -302,7 +301,7 @@ public class CfgParseTreePrinter { ...@@ -302,7 +301,7 @@ public class CfgParseTreePrinter {
if ( child == aParseTree ) { if ( child == aParseTree ) {
ErrorReporter.INTERNAL_ERROR("ConfigTreeNodeUtilities.print(): child == aParseTree"); ErrorReporter.INTERNAL_ERROR("ConfigTreeNodeUtilities.print(): child == aParseTree");
} else { } else {
print( child, aTokenStream, aPrintHiddenBefore || i > 0, aResolveMode, aFile ); print( child, aTokens, aPrintHiddenBefore || i > 0, aResolveMode, aFile );
} }
} }
} }
...@@ -316,7 +315,7 @@ public class CfgParseTreePrinter { ...@@ -316,7 +315,7 @@ public class CfgParseTreePrinter {
* needed only if aResolveMode != NO_RESOLVING, in case of [ORDERED_INCLUDE] * needed only if aResolveMode != NO_RESOLVING, in case of [ORDERED_INCLUDE]
*/ */
private void printToken( final Token aToken, private void printToken( final Token aToken,
final CommonTokenStream aTokenStream, final List<Token> aTokens,
final boolean aPrintHiddenBefore, final boolean aPrintHiddenBefore,
final ResolveMode aResolveMode, final ResolveMode aResolveMode,
final Path aFile ) { final Path aFile ) {
...@@ -324,8 +323,8 @@ public class CfgParseTreePrinter { ...@@ -324,8 +323,8 @@ public class CfgParseTreePrinter {
if ( tokenIndex > -1 && aPrintHiddenBefore ) { if ( tokenIndex > -1 && aPrintHiddenBefore ) {
// Token has no index if tokenIndex == -1. // Token has no index if tokenIndex == -1.
// If a token is added to the parse tree after parse time, token start index in unknown (-1), // If a token is added to the parse tree after parse time, token start index in unknown (-1),
// because token has no index in the token stream. // because token has no index in the token list.
printHiddenTokensBefore( aToken, aTokenStream ); printHiddenTokensBefore( aToken, aTokens );
} }
// the only non-hidden token // the only non-hidden token
...@@ -342,20 +341,20 @@ public class CfgParseTreePrinter { ...@@ -342,20 +341,20 @@ public class CfgParseTreePrinter {
* @param aToken the token, this will NOT be printed * @param aToken the token, this will NOT be printed
*/ */
private void printHiddenTokensBefore( final Token aToken, private void printHiddenTokensBefore( final Token aToken,
final CommonTokenStream aTokenStream ) { final List<Token> aTokens ) {
final int tokenIndex = aToken.getTokenIndex(); final int tokenIndex = aToken.getTokenIndex();
if ( tokenIndex == -1 ) { if ( tokenIndex == -1 ) {
// Token has no index. // Token has no index.
// If a token is added to the parse tree after parse time, token start index in unknown (-1), // If a token is added to the parse tree after parse time, token start index in unknown (-1),
// because token has no index in the token stream. // because token has no index in the token list.
return; return;
} }
int startHiddenIndex = tokenIndex; int startHiddenIndex = tokenIndex;
while ( isHiddenToken( startHiddenIndex - 1, aTokenStream ) ) { while ( isHiddenToken( startHiddenIndex - 1, aTokens ) ) {
startHiddenIndex--; startHiddenIndex--;
} }
for ( int i = startHiddenIndex; i < tokenIndex; i++ ) { for ( int i = startHiddenIndex; i < tokenIndex; i++ ) {
final Token t = aTokenStream.get( i ); final Token t = aTokens.get( i );
final String tokenText = t.getText(); final String tokenText = t.getText();
mSb.append( tokenText != null ? tokenText : "" ); mSb.append( tokenText != null ? tokenText : "" );
} }
...@@ -365,25 +364,26 @@ public class CfgParseTreePrinter { ...@@ -365,25 +364,26 @@ public class CfgParseTreePrinter {
* Builds hidden tokens before the rule * Builds hidden tokens before the rule
* @param aRule the rule, this will NOT be printed * @param aRule the rule, this will NOT be printed
*/ */
private void printHiddenTokensBefore( final ParserRuleContext aRule, final CommonTokenStream aTokenStream ) { private void printHiddenTokensBefore( final ParserRuleContext aRule, final List<Token> aTokens ) {
Token startToken = aRule.start; Token startToken = aRule.start;
if ( startToken == null ) { if ( startToken == null ) {
return; return;
} }
printHiddenTokensBefore( startToken, aTokenStream ); printHiddenTokensBefore( startToken, aTokens );
} }
/** /**
* @param aIndex token index to check * @param aIndex token index to check
* @param aTokens token list from the lexer (all, hidden and not hidden also)
* @return true, iff token index is valid AND token is hidden * @return true, iff token index is valid AND token is hidden
*/ */
private boolean isHiddenToken( final int aIndex, final CommonTokenStream aTokenStream ) { private static boolean isHiddenToken( final int aIndex, final List<Token> aTokens ) {
if ( aTokenStream == null ) { if ( aTokens == null ) {
ErrorReporter.INTERNAL_ERROR("ConfigTreeNodeUtilities.isHiddenToken(): aTokenStream == null"); ErrorReporter.INTERNAL_ERROR("ConfigTreeNodeUtilities.isHiddenToken(): aTokens == null");
return false; return false;
} }
return aIndex >= 0 && aIndex < aTokenStream.size() && aTokenStream.get( aIndex ).getChannel() > 0; return aIndex >= 0 && aIndex < aTokens.size() && aTokens.get( aIndex ).getChannel() > 0;
} }
/** /**
...@@ -444,7 +444,7 @@ public class CfgParseTreePrinter { ...@@ -444,7 +444,7 @@ public class CfgParseTreePrinter {
final Path absolutePath = getAbsolutePath( aFile, filename ); final Path absolutePath = getAbsolutePath( aFile, filename );
final CfgParseResult cfgParseResult = mCfgParseResults.get( absolutePath ); final CfgParseResult cfgParseResult = mCfgParseResults.get( absolutePath );
if ( cfgParseResult != null ) { if ( cfgParseResult != null ) {
printResolved( absolutePath, cfgParseResult.getParseTreeRoot(), cfgParseResult.getTokenStream(), aResolveMode ); printResolved( absolutePath, cfgParseResult.getParseTreeRoot(), cfgParseResult.getTokens(), aResolveMode );
} else { } else {
// include file is missing from mCfgParseResults, so the included cfg file is not parsed // include file is missing from mCfgParseResults, so the included cfg file is not parsed
// in ConfigFileHandler.readFromFile() // in ConfigFileHandler.readFromFile()
......
...@@ -293,23 +293,23 @@ public final class ConfigTreeNodeUtilities { ...@@ -293,23 +293,23 @@ public final class ConfigTreeNodeUtilities {
/** /**
* Builds parse tree text including hidden tokens (also before the rule) * Builds parse tree text including hidden tokens (also before the rule)
* @param aParseTreeRoot root of the parse tree to print * @param aParseTreeRoot root of the parse tree to print
* @param aTokenStream token stream to get the tokens from (all, hidden and not hidden also) * @param aTokens token list from the lexer (all, hidden and not hidden also)
* @return output parse tree text * @return output parse tree text
*/ */
public static String toStringWithHiddenBefore( final ParseTree aParseTreeRoot, public static String toStringWithHiddenBefore( final ParseTree aParseTreeRoot,
final CommonTokenStream aTokenStream ) { final List<Token> aTokens ) {
return CfgParseTreePrinter.toStringWithHidden( aParseTreeRoot, aTokenStream, true ); return CfgParseTreePrinter.toStringWithHidden( aParseTreeRoot, aTokens, true );
} }
//TODO: remove, call directly //TODO: remove, call directly
/** /**
* Builds parse tree text including hidden tokens (but not before the rule) * Builds parse tree text including hidden tokens (but not before the rule)
* @param aParseTreeRoot root of the parse tree to print * @param aParseTreeRoot root of the parse tree to print
* @param aTokenStream token stream to get the tokens from (all, hidden and not hidden also) * @param aTokens token list from the lexer (all, hidden and not hidden also)
* @return output parse tree text * @return output parse tree text
*/ */
public static String toStringWithHidden( final ParseTree aParseTreeRoot, public static String toStringWithHidden( final ParseTree aParseTreeRoot,
final CommonTokenStream aTokenStream ) { final List<Token> aTokens ) {
return CfgParseTreePrinter.toStringWithHidden( aParseTreeRoot, aTokenStream, false ); return CfgParseTreePrinter.toStringWithHidden( aParseTreeRoot, aTokens, false );
} }
} }
...@@ -7,8 +7,10 @@ ...@@ -7,8 +7,10 @@
******************************************************************************/ ******************************************************************************/
package org.eclipse.titan.designer.editors.configeditor; package org.eclipse.titan.designer.editors.configeditor;
import org.antlr.v4.runtime.CommonTokenStream; import java.util.List;
import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.eclipse.core.resources.IMarker; import org.eclipse.core.resources.IMarker;
import org.eclipse.core.resources.IResourceChangeEvent; import org.eclipse.core.resources.IResourceChangeEvent;
import org.eclipse.core.resources.IResourceChangeListener; import org.eclipse.core.resources.IResourceChangeListener;
...@@ -56,7 +58,7 @@ public final class ConfigEditor extends FormEditor implements IResourceChangeLis ...@@ -56,7 +58,7 @@ public final class ConfigEditor extends FormEditor implements IResourceChangeLis
private LoggingPage mLoggingEditor; private LoggingPage mLoggingEditor;
private ParserRuleContext mParseTreeRoot; private ParserRuleContext mParseTreeRoot;
private CommonTokenStream mTokenStream; private List<Token> mTokens;
public ConfigEditor() { public ConfigEditor() {
editor = new ConfigTextEditor(this); editor = new ConfigTextEditor(this);
...@@ -224,18 +226,18 @@ public final class ConfigEditor extends FormEditor implements IResourceChangeLis ...@@ -224,18 +226,18 @@ public final class ConfigEditor extends FormEditor implements IResourceChangeLis
mParseTreeRoot = aParseTreeRoot; mParseTreeRoot = aParseTreeRoot;
} }
public CommonTokenStream getTokenStream() { public List<Token> getTokens() {
return mTokenStream; return mTokens;
} }
public void setTokenStream(final CommonTokenStream aTokenStream) { public void setTokens(final List<Token> aTokens) {
mTokenStream = aTokenStream; mTokens = aTokens;
} }
private void updateTextualPage() { private void updateTextualPage() {
if (mParseTreeRoot != null && mTokenStream != null) { if (mParseTreeRoot != null && mTokens != null) {
String original = editor.getDocument().get(); String original = editor.getDocument().get();
String content = ConfigTreeNodeUtilities.toStringWithHiddenBefore( mParseTreeRoot, mTokenStream ); String content = ConfigTreeNodeUtilities.toStringWithHiddenBefore( mParseTreeRoot, mTokens );
if (!content.equals(original)) { if (!content.equals(original)) {
editor.getDocument().set(content); editor.getDocument().set(content);
......
...@@ -240,7 +240,7 @@ public final class ModuleParameterSectionPage extends FormPage { ...@@ -240,7 +240,7 @@ public final class ModuleParameterSectionPage extends FormPage {
if (moduleParameter != null) { if (moduleParameter != null) {
if (moduleParameter.getValue() != null) { if (moduleParameter.getValue() != null) {
final String moduleParamValueText = ConfigTreeNodeUtilities.toStringWithHidden( final String moduleParamValueText = ConfigTreeNodeUtilities.toStringWithHidden(
moduleParameter.getValue(), editor.getTokenStream() ); moduleParameter.getValue(), editor.getTokens() );
parameterValueText.setText( moduleParamValueText ); parameterValueText.setText( moduleParamValueText );
} }
......
...@@ -388,7 +388,7 @@ public final class ProjectConfigurationParser { ...@@ -388,7 +388,7 @@ public final class ProjectConfigurationParser {
ConfigEditor parentEditor = editor.getParentEditor(); ConfigEditor parentEditor = editor.getParentEditor();
if ( errorsStored == null || errorsStored.isEmpty() ) { if ( errorsStored == null || errorsStored.isEmpty() ) {
parentEditor.setParseTreeRoot(cfgParseResult.getParseTreeRoot()); parentEditor.setParseTreeRoot(cfgParseResult.getParseTreeRoot());
parentEditor.setTokenStream(cfgParseResult.getTokenStream()); parentEditor.setTokens(cfgParseResult.getTokens());
parentEditor.refresh(cfgAnalyzer); parentEditor.refresh(cfgAnalyzer);
parentEditor.setErrorMessage(null); parentEditor.setErrorMessage(null);
} else { } else {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment