diff --git a/org.eclipse.xtend.core.tests/src/org/eclipse/xtend/core/tests/parsing/FlexTokenRegionProviderTest.java b/org.eclipse.xtend.core.tests/src/org/eclipse/xtend/core/tests/parsing/FlexTokenRegionProviderTest.java index 5b26c5f4b2d..f3da20d33dc 100644 --- a/org.eclipse.xtend.core.tests/src/org/eclipse/xtend/core/tests/parsing/FlexTokenRegionProviderTest.java +++ b/org.eclipse.xtend.core.tests/src/org/eclipse/xtend/core/tests/parsing/FlexTokenRegionProviderTest.java @@ -118,7 +118,7 @@ protected List getTokens(String model) { do { currentToken = (CommonToken) tokenSource.nextToken(); tokens.add(currentToken); - } while(currentToken != Token.EOF_TOKEN); + } while (currentToken.getType() != Token.EOF); return tokens; } } diff --git a/org.eclipse.xtend.core.tests/src/org/eclipse/xtend/core/tests/smoke/AbstractSmokeTest.java b/org.eclipse.xtend.core.tests/src/org/eclipse/xtend/core/tests/smoke/AbstractSmokeTest.java index 854b2e5cff3..79919d74fb7 100644 --- a/org.eclipse.xtend.core.tests/src/org/eclipse/xtend/core/tests/smoke/AbstractSmokeTest.java +++ b/org.eclipse.xtend.core.tests/src/org/eclipse/xtend/core/tests/smoke/AbstractSmokeTest.java @@ -167,7 +167,7 @@ protected LazyLinkingResource createResource(final String model) throws IOExcept Lexer lexer = lexerProvider.get(); lexer.setCharStream(new ANTLRStringStream(string)); Token token = lexer.nextToken(); - while(token != Token.EOF_TOKEN) { + while (token.getType() != Token.EOF) { tokenList.add((CommonToken) token); token = lexer.nextToken(); } @@ -233,7 +233,7 @@ protected LazyLinkingResource createResource(final String model) throws IOExcept Lexer lexer = lexerProvider.get(); lexer.setCharStream(new ANTLRStringStream(string)); Token token = lexer.nextToken(); - while(token != Token.EOF_TOKEN) { + while (token.getType() != Token.EOF) { tokenList.add((CommonToken) token); token = lexer.nextToken(); } diff --git a/org.eclipse.xtend.core.tests/src/org/eclipse/xtend/core/tests/smoke/SmokeTest.java b/org.eclipse.xtend.core.tests/src/org/eclipse/xtend/core/tests/smoke/SmokeTest.java index 6ce3755394f..f12413cbcc6 100644 --- a/org.eclipse.xtend.core.tests/src/org/eclipse/xtend/core/tests/smoke/SmokeTest.java +++ b/org.eclipse.xtend.core.tests/src/org/eclipse/xtend/core/tests/smoke/SmokeTest.java @@ -115,7 +115,7 @@ public Lexer get() { Lexer lexer = lexerProvider.get(); lexer.setCharStream(new ANTLRStringStream(string)); Token token = lexer.nextToken(); - while(token != Token.EOF_TOKEN) { + while (token.getType() != Token.EOF) { tokenList.add((CommonToken) token); token = lexer.nextToken(); } diff --git a/org.eclipse.xtend.core/src/org/eclipse/xtend/core/parser/antlr/internal/FlexTokenSource.java b/org.eclipse.xtend.core/src/org/eclipse/xtend/core/parser/antlr/internal/FlexTokenSource.java index fd24e04e984..38b3a386aa5 100644 --- a/org.eclipse.xtend.core/src/org/eclipse/xtend/core/parser/antlr/internal/FlexTokenSource.java +++ b/org.eclipse.xtend.core/src/org/eclipse/xtend/core/parser/antlr/internal/FlexTokenSource.java @@ -39,7 +39,7 @@ public Token nextToken() { try { int type = flexer.advance(); if (type == Token.EOF) { - return Token.EOF_TOKEN; + return new CommonToken(Token.EOF); } int length = flexer.getTokenLength(); final String tokenText = flexer.getTokenText(); diff --git a/org.eclipse.xtend.ide.common/src/org/eclipse/xtend/ide/common/contentassist/antlr/internal/ContentAssistTokenSource.java b/org.eclipse.xtend.ide.common/src/org/eclipse/xtend/ide/common/contentassist/antlr/internal/ContentAssistTokenSource.java index 02dc17e518c..a27fa70130d 100644 --- a/org.eclipse.xtend.ide.common/src/org/eclipse/xtend/ide/common/contentassist/antlr/internal/ContentAssistTokenSource.java +++ b/org.eclipse.xtend.ide.common/src/org/eclipse/xtend/ide/common/contentassist/antlr/internal/ContentAssistTokenSource.java @@ -39,7 +39,7 @@ public Token nextToken() { try { int type = flexer.advance(); if (type == Token.EOF) { - return Token.EOF_TOKEN; + return new CommonToken(Token.EOF); } int length = flexer.getTokenLength(); CommonToken result = new CommonTokenWithoutText(type, Token.DEFAULT_CHANNEL, offset, length); diff --git a/org.eclipse.xtend.ide/src/org/eclipse/xtend/ide/codetemplates/ui/highlighting/FlexerBasedTemplateBodyHighlighter.xtend b/org.eclipse.xtend.ide/src/org/eclipse/xtend/ide/codetemplates/ui/highlighting/FlexerBasedTemplateBodyHighlighter.xtend index 1b5cf515568..8d932a021fa 100644 --- a/org.eclipse.xtend.ide/src/org/eclipse/xtend/ide/codetemplates/ui/highlighting/FlexerBasedTemplateBodyHighlighter.xtend +++ b/org.eclipse.xtend.ide/src/org/eclipse/xtend/ide/codetemplates/ui/highlighting/FlexerBasedTemplateBodyHighlighter.xtend @@ -32,7 +32,7 @@ class FlexerBasedTemplateBodyHighlighter extends TemplateBodyHighlighter { override doProvideHighlightingFor(String body, IHighlightedPositionAcceptor acceptor) { val tokenSource = createTokenSource(new StringReader(body)) var token = tokenSource.nextToken - while (token != Token.EOF_TOKEN) { + while (token.type !== Token.EOF) { val id = token.type.id val offset = token.offset val length = token.length diff --git a/org.eclipse.xtend.ide/xtend-gen/org/eclipse/xtend/ide/codetemplates/ui/highlighting/FlexerBasedTemplateBodyHighlighter.java b/org.eclipse.xtend.ide/xtend-gen/org/eclipse/xtend/ide/codetemplates/ui/highlighting/FlexerBasedTemplateBodyHighlighter.java index 5639829ca0f..c7c589d21cf 100644 --- a/org.eclipse.xtend.ide/xtend-gen/org/eclipse/xtend/ide/codetemplates/ui/highlighting/FlexerBasedTemplateBodyHighlighter.java +++ b/org.eclipse.xtend.ide/xtend-gen/org/eclipse/xtend/ide/codetemplates/ui/highlighting/FlexerBasedTemplateBodyHighlighter.java @@ -10,7 +10,6 @@ import com.google.inject.Inject; import java.io.StringReader; -import java.util.Objects; import org.antlr.runtime.Token; import org.eclipse.xtend.core.parser.antlr.internal.FlexTokenSource; import org.eclipse.xtend.core.parser.antlr.internal.FlexerFactory; @@ -38,7 +37,7 @@ public void doProvideHighlightingFor(final String body, final IHighlightedPositi StringReader _stringReader = new StringReader(body); final FlexTokenSource tokenSource = this._flexerFactory.createTokenSource(_stringReader); Token token = tokenSource.nextToken(); - while ((!Objects.equals(token, Token.EOF_TOKEN))) { + while ((token.getType() != Token.EOF)) { { final String id = this._abstractAntlrTokenToAttributeIdMapper.getId(token.getType()); final int offset = TokenTool.getOffset(token); diff --git a/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/AntlrProposalConflictHelper.java b/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/AntlrProposalConflictHelper.java index 91c6e196281..0bcc8f740ea 100644 --- a/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/AntlrProposalConflictHelper.java +++ b/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/AntlrProposalConflictHelper.java @@ -65,16 +65,14 @@ public boolean existsConflict(String lastCompleteText, String proposal, ContentA if (!equalTokenSequence(getProposalLexer(), getCombinedLexer())) return true; Token lastToken = getProposalLexer().nextToken(); - if (!lastToken.equals(Token.EOF_TOKEN)) - return true; - return false; + return lastToken.getType() != Token.EOF; } protected boolean equalTokenSequence(TokenSource first, TokenSource second) { Token token = null; - while (!(token = first.nextToken()).equals(Token.EOF_TOKEN)) { + while ((token = first.nextToken()).getType() != Token.EOF) { Token otherToken = second.nextToken(); - if (otherToken.equals(Token.EOF_TOKEN)) { + if (otherToken.getType() == Token.EOF) { return false; } if (!token.getText().equals(otherToken.getText())) { diff --git a/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/ContentAssistContextFactory.java b/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/ContentAssistContextFactory.java index dbc92ea469a..ab939ebb927 100644 --- a/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/ContentAssistContextFactory.java +++ b/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/ContentAssistContextFactory.java @@ -285,10 +285,10 @@ protected void createContextsForLastCompleteNode(EObject previousModel, boolean if (!Strings.isEmpty(currentNodePrefix) && !currentNode.getText().equals(currentNodePrefix)) { lexer.setCharStream(new ANTLRStringStream(currentNodePrefix)); Token token = lexer.nextToken(); - if (token == Token.EOF_TOKEN) { // error case - nothing could be parsed + if (token.getType() == Token.EOF) { // error case - nothing could be parsed return; } - while(token != Token.EOF_TOKEN) { + while(token.getType() != Token.EOF) { if (isErrorToken(token)) return; token = lexer.nextToken(); diff --git a/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/LookAheadBasedTokenSource.java b/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/LookAheadBasedTokenSource.java index c165abeb3f8..9f3a6f2385e 100644 --- a/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/LookAheadBasedTokenSource.java +++ b/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/LookAheadBasedTokenSource.java @@ -10,6 +10,7 @@ import java.util.Iterator; +import org.antlr.runtime.CommonToken; import org.antlr.runtime.Token; import org.antlr.runtime.TokenSource; @@ -30,7 +31,7 @@ public Token nextToken() { ILookAheadTerminal lookAhead = iter.next(); return lookAhead.getToken(); } - return Token.EOF_TOKEN; + return new CommonToken(Token.EOF); } @Override diff --git a/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/PartialContentAssistContextFactory.java b/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/PartialContentAssistContextFactory.java index a645ce84f52..77832dcd245 100644 --- a/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/PartialContentAssistContextFactory.java +++ b/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/PartialContentAssistContextFactory.java @@ -15,6 +15,7 @@ import org.eclipse.emf.ecore.EObject; import org.eclipse.xtext.nodemodel.ILeafNode; import org.eclipse.xtext.nodemodel.INode; +import org.eclipse.xtext.parser.antlr.Lexer; import org.eclipse.xtext.util.Strings; /** @@ -27,10 +28,10 @@ protected void createContextsForLastCompleteNode(EObject previousModel, boolean if (!Strings.isEmpty(currentNodePrefix) && !currentNode.getText().equals(currentNodePrefix)) { lexer.setCharStream(new ANTLRStringStream(currentNodePrefix)); Token token = lexer.nextToken(); - if (token == Token.EOF_TOKEN) { + if (token.getType() == Token.EOF) { return; } - while (token != Token.EOF_TOKEN) { + while (token.getType() != Token.EOF) { if (isErrorToken(token)) { return; } diff --git a/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/internal/DFA.java b/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/internal/DFA.java index 1a85a23680d..fa6ecf4ac87 100644 --- a/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/internal/DFA.java +++ b/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/internal/DFA.java @@ -40,7 +40,7 @@ protected IUnorderedGroupHelper getUnorderedGroupHelper() { @Override protected void error(NoViableAltException nvae) { - if (nvae.token == Token.EOF_TOKEN) { + if (nvae.token.getType() == Token.EOF) { int lookAheadAddOn = getRecognizer().lookAheadAddOn; int lookAhead = ((XtextTokenStream)nvae.input).getCurrentLookAhead(); if ((lookAhead >= lookAheadAddOn && lookAheadAddOn > 0) || (lookAhead == 0 && lookAheadAddOn > 0) || lookAhead == -1) diff --git a/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/internal/Lexer.java b/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/internal/Lexer.java index 9eedfe4dd42..4636c79046a 100644 --- a/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/internal/Lexer.java +++ b/org.eclipse.xtext.ide/src/org/eclipse/xtext/ide/editor/contentassist/antlr/internal/Lexer.java @@ -9,6 +9,7 @@ package org.eclipse.xtext.ide.editor.contentassist.antlr.internal; import org.antlr.runtime.CharStream; +import org.antlr.runtime.CommonToken; import org.antlr.runtime.FailedPredicateException; import org.antlr.runtime.NoViableAltException; import org.antlr.runtime.RecognitionException; @@ -22,7 +23,7 @@ public abstract class Lexer extends org.antlr.runtime.Lexer { private static final Logger logger = Logger.getLogger(Lexer.class); - + protected Lexer(CharStream input) { super(input); } @@ -45,7 +46,7 @@ public Token nextToken() { state.tokenStartLine = input.getLine(); state.text = null; if ( input.LA(1)==CharStream.EOF ) { - return Token.EOF_TOKEN; + return new CommonToken(Token.EOF); } try { mTokens(); diff --git a/org.eclipse.xtext.testing/src/org/eclipse/xtext/testing/smoketest/AbstractSmokeTest.java b/org.eclipse.xtext.testing/src/org/eclipse/xtext/testing/smoketest/AbstractSmokeTest.java index 5bc0537862a..098c9b8388a 100644 --- a/org.eclipse.xtext.testing/src/org/eclipse/xtext/testing/smoketest/AbstractSmokeTest.java +++ b/org.eclipse.xtext.testing/src/org/eclipse/xtext/testing/smoketest/AbstractSmokeTest.java @@ -26,9 +26,9 @@ /** * Base class for smoke tests. - * + * * Clients may want to use the {@link XtextSmokeTestRunner} instead. - * + * * @author Sebastian Zarnekow - Initial contribution and API * @author Sven Efftinge */ @@ -83,7 +83,7 @@ public void testSkipTokensInBetween() throws Exception { Lexer lexer = lexerProvider.get(); lexer.setCharStream(new ANTLRStringStream(string)); Token token = lexer.nextToken(); - while (token != Token.EOF_TOKEN) { + while (token.getType() != Token.EOF) { tokenList.add((CommonToken) token); token = lexer.nextToken(); } @@ -157,7 +157,7 @@ public void testSkipTokensInBetweenWithoutResourceSet() throws Exception { Lexer lexer = lexerProvider.get(); lexer.setCharStream(new ANTLRStringStream(string)); Token token = lexer.nextToken(); - while (token != Token.EOF_TOKEN) { + while (token.getType() != Token.EOF) { tokenList.add((CommonToken) token); token = lexer.nextToken(); } diff --git a/org.eclipse.xtext.tests/src/org/eclipse/xtext/lexer/LexerErrorTest.java b/org.eclipse.xtext.tests/src/org/eclipse/xtext/lexer/LexerErrorTest.java index aff9fc45d31..f472f8796b3 100755 --- a/org.eclipse.xtext.tests/src/org/eclipse/xtext/lexer/LexerErrorTest.java +++ b/org.eclipse.xtext.tests/src/org/eclipse/xtext/lexer/LexerErrorTest.java @@ -12,8 +12,8 @@ import java.util.List; import org.antlr.runtime.ANTLRStringStream; -import org.antlr.runtime.CommonToken; import org.antlr.runtime.CommonTokenStream; +import org.antlr.runtime.Token; import org.eclipse.xtext.testlanguages.parser.antlr.internal.InternalSimpleExpressionsTestLanguageLexer; import org.eclipse.xtext.testlanguages.parser.antlr.internal.InternalSimpleExpressionsTestLanguageParser; import org.junit.Assert; @@ -27,21 +27,21 @@ public class LexerErrorTest extends Assert { lexer.setCharStream(new ANTLRStringStream(model)); CommonTokenStream stream = new CommonTokenStream(lexer); @SuppressWarnings("unchecked") - List tokens = stream.getTokens(); + List tokens = stream.getTokens(); assertEquals(tokens.toString(), 3, tokens.size()); assertEquals("a", tokens.get(0).getText()); assertEquals(" ", tokens.get(1).getText()); assertEquals("/* incomplete comment *", tokens.get(2).getText()); assertEquals(0, tokens.get(2).getType()); } - + @Test public void testLexerError_02() throws Exception { String model = "a 'incomplete string"; InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer(); lexer.setCharStream(new ANTLRStringStream(model)); CommonTokenStream stream = new CommonTokenStream(lexer); @SuppressWarnings("unchecked") - List tokens = stream.getTokens(); + List tokens = stream.getTokens(); assertEquals(tokens.toString(), 3, tokens.size()); assertEquals("a", tokens.get(0).getText()); assertEquals(" ", tokens.get(1).getText()); @@ -55,49 +55,49 @@ public class LexerErrorTest extends Assert { lexer.setCharStream(new ANTLRStringStream(model)); CommonTokenStream stream = new CommonTokenStream(lexer); @SuppressWarnings("unchecked") - List tokens = stream.getTokens(); + List tokens = stream.getTokens(); assertEquals(tokens.toString(), 3, tokens.size()); assertEquals("a", tokens.get(0).getText()); assertEquals(" ", tokens.get(1).getText()); assertEquals("'\\ incomplete string with bad escape sequence", tokens.get(2).getText()); assertEquals(0, tokens.get(2).getType()); } - + @Test public void testLexerError_04() throws Exception { String model = "a 'incomplete string with bad escape sequence \\"; InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer(); lexer.setCharStream(new ANTLRStringStream(model)); CommonTokenStream stream = new CommonTokenStream(lexer); @SuppressWarnings("unchecked") - List tokens = stream.getTokens(); + List tokens = stream.getTokens(); assertEquals(tokens.toString(), 3, tokens.size()); assertEquals("a", tokens.get(0).getText()); assertEquals(" ", tokens.get(1).getText()); assertEquals("'incomplete string with bad escape sequence \\", tokens.get(2).getText()); assertEquals(0, tokens.get(2).getType()); } - + @Test public void testLexerError_05() throws Exception { String model = "a 'incomplete string \\'"; InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer(); lexer.setCharStream(new ANTLRStringStream(model)); CommonTokenStream stream = new CommonTokenStream(lexer); @SuppressWarnings("unchecked") - List tokens = stream.getTokens(); + List tokens = stream.getTokens(); assertEquals(tokens.toString(), 3, tokens.size()); assertEquals("a", tokens.get(0).getText()); assertEquals(" ", tokens.get(1).getText()); assertEquals("'incomplete string \\'", tokens.get(2).getText()); assertEquals(0, tokens.get(2).getType()); } - + @Test public void testLexerError_06() throws Exception { String model = "a '"; InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer(); lexer.setCharStream(new ANTLRStringStream(model)); CommonTokenStream stream = new CommonTokenStream(lexer); @SuppressWarnings("unchecked") - List tokens = stream.getTokens(); + List tokens = stream.getTokens(); assertEquals(tokens.toString(), 3, tokens.size()); assertEquals("a", tokens.get(0).getText()); assertEquals(" ", tokens.get(1).getText()); diff --git a/org.eclipse.xtext.tests/src/org/eclipse/xtext/parser/TokenRegionProviderTest.java b/org.eclipse.xtext.tests/src/org/eclipse/xtext/parser/TokenRegionProviderTest.java index 7c6e779b235..3bdec57baa4 100644 --- a/org.eclipse.xtext.tests/src/org/eclipse/xtext/parser/TokenRegionProviderTest.java +++ b/org.eclipse.xtext.tests/src/org/eclipse/xtext/parser/TokenRegionProviderTest.java @@ -132,7 +132,7 @@ protected List getTokens(String model) { currentToken = (CommonToken) lexer.nextToken(); // System.out.println(currentToken); tokens.add(currentToken); - } while(currentToken != Token.EOF_TOKEN); + } while (currentToken.getType() != Token.EOF); return tokens; } } diff --git a/org.eclipse.xtext.tests/src/org/eclipse/xtext/parser/antlr/XtextTokenStreamTest.java b/org.eclipse.xtext.tests/src/org/eclipse/xtext/parser/antlr/XtextTokenStreamTest.java index 11e2c2f6def..1e9cc9ec3a7 100644 --- a/org.eclipse.xtext.tests/src/org/eclipse/xtext/parser/antlr/XtextTokenStreamTest.java +++ b/org.eclipse.xtext.tests/src/org/eclipse/xtext/parser/antlr/XtextTokenStreamTest.java @@ -80,7 +80,7 @@ protected void doTestLookahead(int lookAhead, int tokenCount) { @Override public Token nextToken() { if (tokenCount == 0) - return Token.EOF_TOKEN; + return new CommonToken(Token.EOF); return new CommonToken(tokenCount--, "Text"); } diff --git a/org.eclipse.xtext.ui.codetemplates.ui/src/org/eclipse/xtext/ui/codetemplates/ui/highlighting/TemplateBodyHighlighter.java b/org.eclipse.xtext.ui.codetemplates.ui/src/org/eclipse/xtext/ui/codetemplates/ui/highlighting/TemplateBodyHighlighter.java index cd77ac9ea6f..1741d3c88ab 100644 --- a/org.eclipse.xtext.ui.codetemplates.ui/src/org/eclipse/xtext/ui/codetemplates/ui/highlighting/TemplateBodyHighlighter.java +++ b/org.eclipse.xtext.ui.codetemplates.ui/src/org/eclipse/xtext/ui/codetemplates/ui/highlighting/TemplateBodyHighlighter.java @@ -44,7 +44,7 @@ public void provideHighlightingFor(String body, org.eclipse.xtext.ide.editor.syn protected void doProvideHighlightingFor(String body, org.eclipse.xtext.ide.editor.syntaxcoloring.IHighlightedPositionAcceptor acceptor) { lexer.setCharStream(new ANTLRStringStream(body)); Token token = lexer.nextToken(); - while(token != Token.EOF_TOKEN) { + while (token.getType() != Token.EOF) { String id = tokenIdMapper.getId(token.getType()); int offset = TokenTool.getOffset(token); int length = TokenTool.getLength(token); diff --git a/org.eclipse.xtext.ui.tests/tests/org/eclipse/xtext/ui/tests/editor/model/IXtextDocumentAdapterTest.java b/org.eclipse.xtext.ui.tests/tests/org/eclipse/xtext/ui/tests/editor/model/IXtextDocumentAdapterTest.java index 17d9bc75bb8..56aac568df5 100644 --- a/org.eclipse.xtext.ui.tests/tests/org/eclipse/xtext/ui/tests/editor/model/IXtextDocumentAdapterTest.java +++ b/org.eclipse.xtext.ui.tests/tests/org/eclipse/xtext/ui/tests/editor/model/IXtextDocumentAdapterTest.java @@ -18,6 +18,7 @@ import java.io.ByteArrayInputStream; import java.nio.charset.StandardCharsets; +import org.antlr.runtime.CommonToken; import org.antlr.runtime.Token; import org.antlr.runtime.TokenSource; import org.eclipse.core.resources.IFile; @@ -54,7 +55,7 @@ protected TokenSource createTokenSource(String string) { return new org.antlr.runtime.TokenSource() { @Override public Token nextToken() { - return Token.EOF_TOKEN; + return new CommonToken(Token.EOF); } @Override diff --git a/org.eclipse.xtext.ui.tests/tests/org/eclipse/xtext/ui/tests/xmleditor/AbstractLexerTest.java b/org.eclipse.xtext.ui.tests/tests/org/eclipse/xtext/ui/tests/xmleditor/AbstractLexerTest.java index 0b3536f38eb..3b795ddb5ea 100644 --- a/org.eclipse.xtext.ui.tests/tests/org/eclipse/xtext/ui/tests/xmleditor/AbstractLexerTest.java +++ b/org.eclipse.xtext.ui.tests/tests/org/eclipse/xtext/ui/tests/xmleditor/AbstractLexerTest.java @@ -3,7 +3,7 @@ * This program and the accompanying materials are made available under the * terms of the Eclipse Public License 2.0 which is available at * http://www.eclipse.org/legal/epl-2.0. - * + * * SPDX-License-Identifier: EPL-2.0 */ package org.eclipse.xtext.ui.tests.xmleditor; @@ -52,7 +52,7 @@ private String lexerResult(Lexer lexer, Map tokenNames, CharSeq List result = new ArrayList<>(); while (true) { Token token = lexer.nextToken(); - if (token == Token.EOF_TOKEN) { + if (token.getType() == Token.EOF) { return Joiner.on(System.lineSeparator()).join(result); } Object nameOrType = null; diff --git a/org.eclipse.xtext.ui/deprecated/org/eclipse/xtext/ui/editor/contentassist/antlr/AntlrProposalConflictHelper.java b/org.eclipse.xtext.ui/deprecated/org/eclipse/xtext/ui/editor/contentassist/antlr/AntlrProposalConflictHelper.java index 1bb716a8fb5..ffce0da6496 100644 --- a/org.eclipse.xtext.ui/deprecated/org/eclipse/xtext/ui/editor/contentassist/antlr/AntlrProposalConflictHelper.java +++ b/org.eclipse.xtext.ui/deprecated/org/eclipse/xtext/ui/editor/contentassist/antlr/AntlrProposalConflictHelper.java @@ -25,7 +25,7 @@ * with the input in the document.

*

A proposal is considered to be conflicting if the lexer would not produce * two distinct tokens for the previous sibling and the proposal itself but consume - * parts of the proposal as part of the first token. + * parts of the proposal as part of the first token. * Example:

* * === Grammar:
@@ -36,8 +36,8 @@ *

where | denotes the cursor position. A valid follow * token for the parser is the ID of the cross reference. However, since * MyIdSomethingElse would be consumed as a single ID by the lexer, - * the proposal SomethingElse is invalid.

- * + * the proposal SomethingElse is invalid.

+ * * @author Sebastian Zarnekow - Initial contribution and API */ public class AntlrProposalConflictHelper extends ProposalConflictHelper { @@ -49,7 +49,7 @@ public class AntlrProposalConflictHelper extends ProposalConflictHelper { @Inject @Named(LexerBindings.RUNTIME) private Lexer lastCompleteLexer; - + @Inject @Named(LexerBindings.RUNTIME) private Lexer combinedLexer; @@ -62,16 +62,14 @@ public boolean existsConflict(String lastCompleteText, String proposal, ContentA if (!equalTokenSequence(getProposalLexer(), getCombinedLexer())) return true; Token lastToken = getProposalLexer().nextToken(); - if (!lastToken.equals(Token.EOF_TOKEN)) - return true; - return false; + return lastToken.getType() != Token.EOF; } protected boolean equalTokenSequence(TokenSource first, TokenSource second) { Token token = null; - while(!(token = first.nextToken()).equals(Token.EOF_TOKEN)) { + while ((token = first.nextToken()).getType() != Token.EOF) { Token otherToken = second.nextToken(); - if (otherToken.equals(Token.EOF_TOKEN)) { + if (otherToken.getType() == Token.EOF) { return false; } if (!token.getText().equals(otherToken.getText())) { @@ -80,7 +78,7 @@ protected boolean equalTokenSequence(TokenSource first, TokenSource second) { } return true; } - + protected void initTokenSources(String lastCompleteText, String proposal, ContentAssistContext context) { String combinedText = lastCompleteText.concat(proposal); initTokenSource(combinedText, getCombinedLexer(), context); diff --git a/org.eclipse.xtext.ui/deprecated/org/eclipse/xtext/ui/editor/contentassist/antlr/ParserBasedContentAssistContextFactory.java b/org.eclipse.xtext.ui/deprecated/org/eclipse/xtext/ui/editor/contentassist/antlr/ParserBasedContentAssistContextFactory.java index f8b6ebdd597..d80fe485610 100644 --- a/org.eclipse.xtext.ui/deprecated/org/eclipse/xtext/ui/editor/contentassist/antlr/ParserBasedContentAssistContextFactory.java +++ b/org.eclipse.xtext.ui/deprecated/org/eclipse/xtext/ui/editor/contentassist/antlr/ParserBasedContentAssistContextFactory.java @@ -401,10 +401,10 @@ protected void createContextsForLastCompleteNode(EObject previousModel, boolean if (!Strings.isEmpty(currentNodePrefix) && !currentNode.getText().equals(currentNodePrefix)) { lexer.setCharStream(new ANTLRStringStream(currentNodePrefix)); Token token = lexer.nextToken(); - if (token == Token.EOF_TOKEN) { // error case - nothing could be parsed + if (token.getType() == Token.EOF) { // error case - nothing could be parsed return; } - while(token != Token.EOF_TOKEN) { + while(token.getType() != Token.EOF) { if (isErrorToken(token)) return; token = lexer.nextToken(); @@ -773,10 +773,10 @@ protected void createContextsForLastCompleteNode(EObject previousModel, boolean if (!Strings.isEmpty(currentNodePrefix) && !currentNode.getText().equals(currentNodePrefix)) { lexer.setCharStream(new ANTLRStringStream(currentNodePrefix)); Token token = lexer.nextToken(); - if (token == Token.EOF_TOKEN) { + if (token.getType() == Token.EOF) { return; } - while (token != Token.EOF_TOKEN) { + while (token.getType() != Token.EOF) { if (isErrorToken(token)) { return; } diff --git a/org.eclipse.xtext.ui/deprecated/org/eclipse/xtext/ui/editor/contentassist/antlr/internal/Lexer.java b/org.eclipse.xtext.ui/deprecated/org/eclipse/xtext/ui/editor/contentassist/antlr/internal/Lexer.java index 4d5ce3fbd2b..3df1e915e77 100644 --- a/org.eclipse.xtext.ui/deprecated/org/eclipse/xtext/ui/editor/contentassist/antlr/internal/Lexer.java +++ b/org.eclipse.xtext.ui/deprecated/org/eclipse/xtext/ui/editor/contentassist/antlr/internal/Lexer.java @@ -9,6 +9,7 @@ package org.eclipse.xtext.ui.editor.contentassist.antlr.internal; import org.antlr.runtime.CharStream; +import org.antlr.runtime.CommonToken; import org.antlr.runtime.FailedPredicateException; import org.antlr.runtime.NoViableAltException; import org.antlr.runtime.RecognitionException; @@ -45,7 +46,7 @@ public Token nextToken() { state.tokenStartLine = input.getLine(); state.text = null; if ( input.LA(1)==CharStream.EOF ) { - return Token.EOF_TOKEN; + return new CommonToken(Token.EOF); } try { mTokens(); diff --git a/org.eclipse.xtext.ui/src/org/eclipse/xtext/ui/editor/model/BacktrackingLexerDocumentTokenSource.java b/org.eclipse.xtext.ui/src/org/eclipse/xtext/ui/editor/model/BacktrackingLexerDocumentTokenSource.java index 051eb5bc29b..fffdb87716e 100644 --- a/org.eclipse.xtext.ui/src/org/eclipse/xtext/ui/editor/model/BacktrackingLexerDocumentTokenSource.java +++ b/org.eclipse.xtext.ui/src/org/eclipse/xtext/ui/editor/model/BacktrackingLexerDocumentTokenSource.java @@ -12,11 +12,12 @@ import org.antlr.runtime.Token; import org.antlr.runtime.TokenSource; import org.eclipse.jface.text.DocumentEvent; +import org.eclipse.xtext.parser.antlr.Lexer; /** * The old version of the {@link DocumentTokenSource}. It assumes that all tokens before the damaged region can change - * with a {@link DocumentEvent} therefore the entire document is being lexed. - * + * with a {@link DocumentEvent} therefore the entire document is being lexed. + * * @author koehnlein - Initial contribution and API * @since 2.4 */ @@ -33,7 +34,7 @@ protected RepairEntryData getRepairEntryData(DocumentEvent e) throws Exception { CommonToken token = (CommonToken) source.nextToken(); // find start idx while (true) { - if (token == Token.EOF_TOKEN) { + if (token.getType() == Token.EOF) { break; } if (tokenInfoIdx >= getInternalModifyableTokenInfos().size()) diff --git a/org.eclipse.xtext.ui/src/org/eclipse/xtext/ui/editor/model/DocumentTokenSource.java b/org.eclipse.xtext.ui/src/org/eclipse/xtext/ui/editor/model/DocumentTokenSource.java index 29ed315cd4f..60175771c96 100644 --- a/org.eclipse.xtext.ui/src/org/eclipse/xtext/ui/editor/model/DocumentTokenSource.java +++ b/org.eclipse.xtext.ui/src/org/eclipse/xtext/ui/editor/model/DocumentTokenSource.java @@ -229,7 +229,7 @@ protected List createTokenInfos(String string) { List result = Lists.newArrayListWithExpectedSize(string.length() / 3); TokenSource source = createTokenSource(string); CommonToken token = (CommonToken) source.nextToken(); - while (token != Token.EOF_TOKEN) { + while (token.getType() != Token.EOF) { TokenInfo info = createTokenInfo(token); result.add(info); token = (CommonToken) source.nextToken(); @@ -281,7 +281,7 @@ protected IRegion computeDamageRegion(final DocumentEvent e) { int tokenStartsAt = repairEntryData.offset; int tokenInfoIdx = repairEntryData.index; CommonToken token = repairEntryData.newToken; - if (token == Token.EOF_TOKEN) + if (token.getType() == Token.EOF) internalModifyableTokenInfos.subList(tokenInfoIdx, internalModifyableTokenInfos.size()).clear(); int regionOffset = tokenStartsAt; int regionLength = e.fDocument.getLength()- tokenStartsAt; @@ -289,7 +289,7 @@ protected IRegion computeDamageRegion(final DocumentEvent e) { int lengthDiff = e.fText.length() - e.fLength; // compute region length while (true) { - if (token == Token.EOF_TOKEN || tokenInfoIdx >= internalModifyableTokenInfos.size()) + if (token.getType() == Token.EOF || tokenInfoIdx >= internalModifyableTokenInfos.size()) break; while (true) { if (tokenInfoIdx >= internalModifyableTokenInfos.size()) @@ -314,7 +314,7 @@ protected IRegion computeDamageRegion(final DocumentEvent e) { internalModifyableTokenInfos.subList(tokenInfoIdx, internalModifyableTokenInfos.size()).clear(); // add subsequent tokens if (tokenInfoIdx >= internalModifyableTokenInfos.size()) { - while (token != Token.EOF_TOKEN) { + while (token.getType() != Token.EOF) { internalModifyableTokenInfos.add(createTokenInfo(token)); token = (CommonToken) repairEntryData.tokenSource.nextToken(); } diff --git a/org.eclipse.xtext/src/org/eclipse/xtext/parser/antlr/Lexer.java b/org.eclipse.xtext/src/org/eclipse/xtext/parser/antlr/Lexer.java index f235d497b0b..511c921e361 100644 --- a/org.eclipse.xtext/src/org/eclipse/xtext/parser/antlr/Lexer.java +++ b/org.eclipse.xtext/src/org/eclipse/xtext/parser/antlr/Lexer.java @@ -57,7 +57,7 @@ public Token nextToken() { this.state.tokenStartLine = input.getLine(); this.state.text = null; if (input.LA(1) == CharStream.EOF) { - return Token.EOF_TOKEN; + return new CommonToken(Token.EOF); } try { mTokens(); diff --git a/org.eclipse.xtext/src/org/eclipse/xtext/parser/antlr/XtextTokenStream.java b/org.eclipse.xtext/src/org/eclipse/xtext/parser/antlr/XtextTokenStream.java index 5cfe6841af4..9dd49fe9486 100644 --- a/org.eclipse.xtext/src/org/eclipse/xtext/parser/antlr/XtextTokenStream.java +++ b/org.eclipse.xtext/src/org/eclipse/xtext/parser/antlr/XtextTokenStream.java @@ -87,20 +87,18 @@ public String toString(int start, int stop) { if ( stop>=tokens.size() ) { stop = tokens.size()-1; } - if (tokenSource instanceof Lexer) { + if (tokenSource instanceof Lexer lexer) { Token startToken = (Token) tokens.get(start); Token stopToken = (Token) tokens.get(stop); - if (startToken instanceof CommonToken && stopToken instanceof CommonToken) { - CommonToken commonStart = (CommonToken) startToken; - CommonToken commonStop = (CommonToken) stopToken; - CharStream charStream = ((Lexer) tokenSource).getCharStream(); + if (startToken instanceof CommonToken commonStart && stopToken instanceof CommonToken commonStop) { + CharStream charStream = lexer.getCharStream(); String result = charStream.substring(commonStart.getStartIndex(), commonStop.getStopIndex()); return result; } } // fall back to super implementation but use StringBuilder instead of StringBuffer // and use reasonable initialization size - StringBuilder result = new StringBuilder(Math.max(1024, tokens.size() * 6)); + StringBuilder result = new StringBuilder(Math.max(1024, tokens.size() * 6)); for (int i = start; i <= stop; i++) { Token t = (Token)tokens.get(i); result.append(t.getText()); @@ -109,13 +107,13 @@ public String toString(int start, int stop) { } @SuppressWarnings({ "serial" }) - private final class TokenList extends ArrayList { + private final class TokenList extends ArrayList { private TokenList(int initialCapacity) { super(initialCapacity); } @Override - public Object get(int index) { + public Token get(int index) { Token tok = (Token) super.get(index); // adjust only tokens in the 'future', as we wont change the channel of previously parsed // tokens @@ -208,7 +206,7 @@ public int LA(int i) { * @since 2.22 */ protected int getTokenIndex(Token tok) { - if (tok == Token.EOF_TOKEN) { + if (tok.getType() == Token.EOF) { return size(); } return tok.getTokenIndex(); @@ -291,7 +289,7 @@ public Token LT(int k) { // copied from super.LT(k) except from the last assignment to p int k_ = k + 1; if ( (p+k_-1) >= tokens.size() ) { - return Token.EOF_TOKEN; + return new CommonToken(Token.EOF); } int i = p; int n = 1; @@ -305,7 +303,7 @@ public Token LT(int k) { n++; } if ( i>=tokens.size() ) { - return Token.EOF_TOKEN; + return new CommonToken(Token.EOF); } p = i; // adjust p to the valid pointer result = (Token)tokens.get(i); @@ -314,7 +312,7 @@ public Token LT(int k) { } public int getCurrentLookAhead() { - return currentLookAhead; + return currentLookAhead; } public void initCurrentLookAhead(int currentLookAhead) { diff --git a/org.eclipse.xtext/src/org/eclipse/xtext/parser/impl/TokenRegionProvider.java b/org.eclipse.xtext/src/org/eclipse/xtext/parser/impl/TokenRegionProvider.java index 648350d2863..b41711a540f 100644 --- a/org.eclipse.xtext/src/org/eclipse/xtext/parser/impl/TokenRegionProvider.java +++ b/org.eclipse.xtext/src/org/eclipse/xtext/parser/impl/TokenRegionProvider.java @@ -41,13 +41,13 @@ public ITextRegion getTokenRegion(String text, ITextRegion region) { CommonToken nextToken = (CommonToken) lexer.nextToken(); int regionStart = region.getOffset(); int regionEnd = region.getEndOffset(); - while (nextToken != Token.EOF_TOKEN && currentEnd <= regionStart) { + while (nextToken.getType() != Token.EOF && currentEnd <= regionStart) { currentStart = nextToken.getStartIndex(); currentEnd = nextToken.getStopIndex() + 1; nextToken = (CommonToken) lexer.nextToken(); } // nextToken is either EOF or the first token that follows the start of the given region - while (nextToken != Token.EOF_TOKEN && currentEnd < regionEnd) { + while (nextToken.getType() != Token.EOF && currentEnd < regionEnd) { currentEnd = nextToken.getStopIndex() + 1; nextToken = (CommonToken) lexer.nextToken(); }