Commit 30be7332 authored by Árpád Lovassy's avatar Árpád Lovassy Committed by GitHub
Browse files

Merge pull request #91 from alovassy/master

bugfix: remove unnecessary CommonTokenStream dependency, and just tok…
parents 6c14a018 1fb9fa44
......@@ -166,8 +166,8 @@ public final class CfgAnalyzer {
// pr_PatternChunk[StringBuilder builder, boolean[] uni]:
// $builder.append($v.text); <-- exception is thrown here: java.lang.UnsupportedOperationException: interval 85..85 not in token buffer window: 86..341
// 2. Changed from BufferedTokenStream to CommonTokenStream, otherwise tokens with "-> channel(HIDDEN)" are not filtered out in lexer.
final CommonTokenStream tokens = new CommonTokenStream( lexer );
final CfgParser parser = new CfgParser(tokens);
final CommonTokenStream tokenStream = new CommonTokenStream( lexer );
final CfgParser parser = new CfgParser( tokenStream );
//parser tree is built by default
parserListener = new TitanListener();
parser.removeErrorListeners(); // remove ConsoleErrorListener
......@@ -179,7 +179,7 @@ public final class CfgAnalyzer {
// manually add the result parse tree, and its corresponding token stream,
// because they logically belong to here
mCfgParseResult.setParseTreeRoot( parseTreeRoot );
mCfgParseResult.setTokenStream( tokens );
mCfgParseResult.setTokens( tokenStream.getTokens() );
// fill handlers
moduleParametersHandler = parser.getModuleParametersHandler();
......
......@@ -12,8 +12,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.eclipse.titan.common.parsers.TITANMarker;
/**
......@@ -30,7 +30,7 @@ public class CfgParseResult {
* It contains all the tokens from all the channels, so it contains also the hidden tokens as well.
* A token is hidden, if token channel > 0
*/
private CommonTokenStream mTokenStream = null;
private List<Token> mTokens = null;
/**
* list of syntactic warnings collected during parsing
......@@ -105,12 +105,12 @@ public class CfgParseResult {
this.mParseTreeRoot = aParseTreeRoot;
}
public CommonTokenStream getTokenStream() {
return mTokenStream;
public List<Token> getTokens() {
return mTokens;
}
public void setTokenStream(CommonTokenStream aTokenStream) {
this.mTokenStream = aTokenStream;
public void setTokens( List<Token> aTokens ) {
this.mTokens = aTokens;
}
public List<TITANMarker> getWarnings() {
......
......@@ -15,7 +15,6 @@ import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.tree.ParseTree;
......@@ -160,15 +159,15 @@ public class CfgParseTreePrinter {
* @param aParseTreeRoot root of the parse tree to print
* @param aPrintHiddenBefore true to print hidden tokens before the parse tree
* (NOTE: hidden tokens in the parse tree will be printed)
* @param aTokenStream token stream to get the tokens from (all, hidden and not hidden also)
* @param aTokens token list from the lexer (all, hidden and not hidden also)
* @return output parse tree text
*/
public static String toStringWithHidden( final ParseTree aParseTreeRoot,
final CommonTokenStream aTokenStream,
final List<Token> aTokens,
final boolean aPrintHiddenBefore ) {
final StringBuilder sb = new StringBuilder();
CfgParseTreePrinter printer = new CfgParseTreePrinter( sb );
printer.print( aParseTreeRoot, aTokenStream, aPrintHiddenBefore, ResolveMode.NO_RESOLVING, null );
printer.print( aParseTreeRoot, aTokens, aPrintHiddenBefore, ResolveMode.NO_RESOLVING, null );
// there are no hidden tokens after the last token
return sb.toString();
......@@ -206,7 +205,7 @@ public class CfgParseTreePrinter {
aCfgParseResults, aDefinitions, aEnvVariables, filesToResolve );
for ( Entry<Path, CfgParseResult> entry : aCfgParseResults.entrySet() ) {
printer.printResolved( entry.getKey(), entry.getValue().getParseTreeRoot(),
entry.getValue().getTokenStream(), aResolveMode );
entry.getValue().getTokens(), aResolveMode );
}
break;
}
......@@ -219,7 +218,7 @@ public class CfgParseTreePrinter {
CfgParseTreePrinter printer = new CfgParseTreePrinter( aSb, aDisallowedNodes,
aCfgParseResults, aDefinitions, aEnvVariables, filesToResolve );
printer.printResolved( entry.getKey(), entry.getValue().getParseTreeRoot(),
entry.getValue().getTokenStream(), aResolveMode );
entry.getValue().getTokens(), aResolveMode );
break;
}
case NO_RESOLVING:
......@@ -242,11 +241,11 @@ public class CfgParseTreePrinter {
*/
private void printResolved( final Path aFile,
final ParseTree aParseTreeRoot,
final CommonTokenStream aTokenStream,
final List<Token> aTokens,
final ResolveMode aResolveMode ) {
if ( mFilesToResolve.contains( aFile ) ) {
mSb.append( INCLUDED_BEGIN ).append( aFile.toOSString()).append('\n');
print( aParseTreeRoot, aTokenStream, true, aResolveMode, aFile );
print( aParseTreeRoot, aTokens, true, aResolveMode, aFile );
mSb.append( INCLUDED_END ).append( aFile.toOSString()).append('\n');
mFilesToResolve.remove( aFile );
}
......@@ -263,7 +262,7 @@ public class CfgParseTreePrinter {
* needed only if aResolveMode != NO_RESOLVING, in case of [ORDERED_INCLUDE]
*/
private void print( final ParseTree aParseTree,
final CommonTokenStream aTokenStream,
final List<Token> aTokens,
final boolean aPrintHiddenBefore,
final ResolveMode aResolveMode,
final Path aFile ) {
......@@ -278,15 +277,15 @@ public class CfgParseTreePrinter {
}
if ( aPrintHiddenBefore && rule.getChildCount() > 0 && rule.getChild( 0 ) instanceof AddedParseTree ) {
// special case: if AddedParseTree is the 1st in the rule, it has no information
// about the hidden tokens, as it has no position in the token stream, but the rule may have
printHiddenTokensBefore( rule, aTokenStream );
// about the hidden tokens, as it has no position in the token list, but the rule may have
printHiddenTokensBefore( rule, aTokens );
}
}
else if ( aParseTree instanceof TerminalNodeImpl ) {
final TerminalNodeImpl tn = (TerminalNodeImpl)aParseTree;
final Token token = tn.getSymbol();
if ( mDisallowedNodes == null || !mDisallowedNodes.contains( token.getType() ) ) {
printToken( token, aTokenStream, aPrintHiddenBefore, aResolveMode, aFile );
printToken( token, aTokens, aPrintHiddenBefore, aResolveMode, aFile );
}
}
else if ( aParseTree instanceof AddedParseTree ) {
......@@ -302,7 +301,7 @@ public class CfgParseTreePrinter {
if ( child == aParseTree ) {
ErrorReporter.INTERNAL_ERROR("ConfigTreeNodeUtilities.print(): child == aParseTree");
} else {
print( child, aTokenStream, aPrintHiddenBefore || i > 0, aResolveMode, aFile );
print( child, aTokens, aPrintHiddenBefore || i > 0, aResolveMode, aFile );
}
}
}
......@@ -316,7 +315,7 @@ public class CfgParseTreePrinter {
* needed only if aResolveMode != NO_RESOLVING, in case of [ORDERED_INCLUDE]
*/
private void printToken( final Token aToken,
final CommonTokenStream aTokenStream,
final List<Token> aTokens,
final boolean aPrintHiddenBefore,
final ResolveMode aResolveMode,
final Path aFile ) {
......@@ -324,8 +323,8 @@ public class CfgParseTreePrinter {
if ( tokenIndex > -1 && aPrintHiddenBefore ) {
// Token has no index if tokenIndex == -1.
// If a token is added to the parse tree after parse time, token start index in unknown (-1),
// because token has no index in the token stream.
printHiddenTokensBefore( aToken, aTokenStream );
// because token has no index in the token list.
printHiddenTokensBefore( aToken, aTokens );
}
// the only non-hidden token
......@@ -342,20 +341,20 @@ public class CfgParseTreePrinter {
* @param aToken the token, this will NOT be printed
*/
private void printHiddenTokensBefore( final Token aToken,
final CommonTokenStream aTokenStream ) {
final List<Token> aTokens ) {
final int tokenIndex = aToken.getTokenIndex();
if ( tokenIndex == -1 ) {
// Token has no index.
// If a token is added to the parse tree after parse time, token start index in unknown (-1),
// because token has no index in the token stream.
// because token has no index in the token list.
return;
}
int startHiddenIndex = tokenIndex;
while ( isHiddenToken( startHiddenIndex - 1, aTokenStream ) ) {
while ( isHiddenToken( startHiddenIndex - 1, aTokens ) ) {
startHiddenIndex--;
}
for ( int i = startHiddenIndex; i < tokenIndex; i++ ) {
final Token t = aTokenStream.get( i );
final Token t = aTokens.get( i );
final String tokenText = t.getText();
mSb.append( tokenText != null ? tokenText : "" );
}
......@@ -365,25 +364,26 @@ public class CfgParseTreePrinter {
* Builds hidden tokens before the rule
* @param aRule the rule, this will NOT be printed
*/
private void printHiddenTokensBefore( final ParserRuleContext aRule, final CommonTokenStream aTokenStream ) {
private void printHiddenTokensBefore( final ParserRuleContext aRule, final List<Token> aTokens ) {
Token startToken = aRule.start;
if ( startToken == null ) {
return;
}
printHiddenTokensBefore( startToken, aTokenStream );
printHiddenTokensBefore( startToken, aTokens );
}
/**
* @param aIndex token index to check
* @param aTokens token list from the lexer (all, hidden and not hidden also)
* @return true, iff token index is valid AND token is hidden
*/
private boolean isHiddenToken( final int aIndex, final CommonTokenStream aTokenStream ) {
if ( aTokenStream == null ) {
ErrorReporter.INTERNAL_ERROR("ConfigTreeNodeUtilities.isHiddenToken(): aTokenStream == null");
private static boolean isHiddenToken( final int aIndex, final List<Token> aTokens ) {
if ( aTokens == null ) {
ErrorReporter.INTERNAL_ERROR("ConfigTreeNodeUtilities.isHiddenToken(): aTokens == null");
return false;
}
return aIndex >= 0 && aIndex < aTokenStream.size() && aTokenStream.get( aIndex ).getChannel() > 0;
return aIndex >= 0 && aIndex < aTokens.size() && aTokens.get( aIndex ).getChannel() > 0;
}
/**
......@@ -444,7 +444,7 @@ public class CfgParseTreePrinter {
final Path absolutePath = getAbsolutePath( aFile, filename );
final CfgParseResult cfgParseResult = mCfgParseResults.get( absolutePath );
if ( cfgParseResult != null ) {
printResolved( absolutePath, cfgParseResult.getParseTreeRoot(), cfgParseResult.getTokenStream(), aResolveMode );
printResolved( absolutePath, cfgParseResult.getParseTreeRoot(), cfgParseResult.getTokens(), aResolveMode );
} else {
// include file is missing from mCfgParseResults, so the included cfg file is not parsed
// in ConfigFileHandler.readFromFile()
......
......@@ -293,23 +293,23 @@ public final class ConfigTreeNodeUtilities {
/**
* Builds parse tree text including hidden tokens (also before the rule)
* @param aParseTreeRoot root of the parse tree to print
* @param aTokenStream token stream to get the tokens from (all, hidden and not hidden also)
* @param aTokens token list from the lexer (all, hidden and not hidden also)
* @return output parse tree text
*/
public static String toStringWithHiddenBefore( final ParseTree aParseTreeRoot,
final CommonTokenStream aTokenStream ) {
return CfgParseTreePrinter.toStringWithHidden( aParseTreeRoot, aTokenStream, true );
final List<Token> aTokens ) {
return CfgParseTreePrinter.toStringWithHidden( aParseTreeRoot, aTokens, true );
}
//TODO: remove, call directly
/**
* Builds parse tree text including hidden tokens (but not before the rule)
* @param aParseTreeRoot root of the parse tree to print
* @param aTokenStream token stream to get the tokens from (all, hidden and not hidden also)
* @param aTokens token list from the lexer (all, hidden and not hidden also)
* @return output parse tree text
*/
public static String toStringWithHidden( final ParseTree aParseTreeRoot,
final CommonTokenStream aTokenStream ) {
return CfgParseTreePrinter.toStringWithHidden( aParseTreeRoot, aTokenStream, false );
final List<Token> aTokens ) {
return CfgParseTreePrinter.toStringWithHidden( aParseTreeRoot, aTokens, false );
}
}
......@@ -7,8 +7,10 @@
******************************************************************************/
package org.eclipse.titan.designer.editors.configeditor;
import org.antlr.v4.runtime.CommonTokenStream;
import java.util.List;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.Token;
import org.eclipse.core.resources.IMarker;
import org.eclipse.core.resources.IResourceChangeEvent;
import org.eclipse.core.resources.IResourceChangeListener;
......@@ -56,7 +58,7 @@ public final class ConfigEditor extends FormEditor implements IResourceChangeLis
private LoggingPage mLoggingEditor;
private ParserRuleContext mParseTreeRoot;
private CommonTokenStream mTokenStream;
private List<Token> mTokens;
public ConfigEditor() {
editor = new ConfigTextEditor(this);
......@@ -224,18 +226,18 @@ public final class ConfigEditor extends FormEditor implements IResourceChangeLis
mParseTreeRoot = aParseTreeRoot;
}
public CommonTokenStream getTokenStream() {
return mTokenStream;
public List<Token> getTokens() {
return mTokens;
}
public void setTokenStream(final CommonTokenStream aTokenStream) {
mTokenStream = aTokenStream;
public void setTokens(final List<Token> aTokens) {
mTokens = aTokens;
}
private void updateTextualPage() {
if (mParseTreeRoot != null && mTokenStream != null) {
if (mParseTreeRoot != null && mTokens != null) {
String original = editor.getDocument().get();
String content = ConfigTreeNodeUtilities.toStringWithHiddenBefore( mParseTreeRoot, mTokenStream );
String content = ConfigTreeNodeUtilities.toStringWithHiddenBefore( mParseTreeRoot, mTokens );
if (!content.equals(original)) {
editor.getDocument().set(content);
......
......@@ -240,7 +240,7 @@ public final class ModuleParameterSectionPage extends FormPage {
if (moduleParameter != null) {
if (moduleParameter.getValue() != null) {
final String moduleParamValueText = ConfigTreeNodeUtilities.toStringWithHidden(
moduleParameter.getValue(), editor.getTokenStream() );
moduleParameter.getValue(), editor.getTokens() );
parameterValueText.setText( moduleParamValueText );
}
......
......@@ -388,7 +388,7 @@ public final class ProjectConfigurationParser {
ConfigEditor parentEditor = editor.getParentEditor();
if ( errorsStored == null || errorsStored.isEmpty() ) {
parentEditor.setParseTreeRoot(cfgParseResult.getParseTreeRoot());
parentEditor.setTokenStream(cfgParseResult.getTokenStream());
parentEditor.setTokens(cfgParseResult.getTokens());
parentEditor.refresh(cfgAnalyzer);
parentEditor.setErrorMessage(null);
} else {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment