shithub: rgbds

Info  •  Files  •  Log  •  Branches

Restore the "EOF-newline" lexer hack

This was removed in b3c0db218d564bbc9a801d8635591fd969c36e82
(along with two unrelated changes).

Removing this hack introduced issue #742, whereby INCLUDing
a file without a trailing newline can cause a syntax error.

A more proper fix would involve Bison's tracking locations,
but for now the EOF-newline hack fixes the issue while only
affecting some reported errors (expecting "newline"
instead of "end of file").

Fixes #742

--- a/src/asm/lexer.c

+++ b/src/asm/lexer.c

@@ -349,6 +349,7 @@

 	bool atLineStart;

 	uint32_t lineNo;

 	uint32_t colNo;

+	int lastToken;

 	struct IfStack *ifStack;

@@ -372,6 +373,7 @@

 {

 	state->mode = LEXER_NORMAL;

 	state->atLineStart = true; /* yylex() will init colNo due to this */

+	state->lastToken = T_EOF;

 	state->ifStack = NULL;

@@ -2235,17 +2237,15 @@

 		return T_STRING;

 	lexer_SetMode(LEXER_NORMAL);

-	// If a macro is invoked on the last line of a file, with no blank

-	// line afterwards, returning EOF afterwards will cause Bison to

-	// stop parsing, despite the lexer being ready to output more.

-	// To avoid this, return T_NEWLINE for EOF as well.

 	if (c == '\r' || c == '\n') {

 		shiftChars(1);

 		/* Handle CRLF */

 		if (c == '\r' && peek(0) == '\n')

 			shiftChars(1);

+		return T_NEWLINE;

 	}

-	return T_NEWLINE;

+

+	return T_EOF;

 }

 #undef append_yylval_tzString

@@ -2456,16 +2456,22 @@

 	int token = lexerModeFuncs[lexerState->mode]();

 	if (token == T_EOF) {

-		/* Try to switch to new buffer; if it succeeds, scan again */

-		dbgPrint("Reached EOB!\n");

-		/* Captures end at their buffer's boundary no matter what */

-		if (!lexerState->capturing) {

-			if (!yywrap())

-				goto restart;

-			dbgPrint("Reached end of input.\n");

-			return T_EOF;

+		if (lexerState->lastToken != T_NEWLINE) {

+			dbgPrint("Forcing EOL at EOF\n");

+			token = T_NEWLINE;

+		} else {

+			/* Try to switch to new buffer; if it succeeds, scan again */

+			dbgPrint("Reached EOF!\n");

+			/* Captures end at their buffer's boundary no matter what */

+			if (!lexerState->capturing) {

+				if (!yywrap())

+					goto restart;

+				dbgPrint("Reached end of input.\n");

+				return T_EOF;

+			}

 		}

 	}

+	lexerState->lastToken = token;

 	lexerState->atLineStart = token == T_NEWLINE;

@@ -2524,6 +2530,7 @@

 					 * We know we have read exactly "ENDR", not e.g. an EQUS

 					 */

 					lexerState->captureSize -= strlen("ENDR");

+					lexerState->lastToken = T_POP_ENDR; // Force EOL at EOF

 					goto finish;

 				}

 				level--;

@@ -2586,6 +2593,7 @@

 				 * We know we have read exactly "ENDM", not e.g. an EQUS

 				 */

 				lexerState->captureSize -= strlen("ENDM");

+				lexerState->lastToken = T_POP_ENDM; // Force EOL at EOF

 				goto finish;

 			}

 		}

--- a/src/asm/parser.y

+++ b/src/asm/parser.y

@@ -603,20 +603,19 @@

 %%

-asmfile		: lines last_line

+asmfile		: lines

 ;

+/* Note: The lexer adds T_NEWLINE at the end of the input */

 lines		: %empty

 		| lines line

 ;

-last_line	: label

-		| label cpu_command

-		| label macro

-		| label directive

-		| assignment_directive

-;

-line		: last_line T_NEWLINE

+line		: label T_NEWLINE

+		| label cpu_command T_NEWLINE

+		| label macro T_NEWLINE

+		| label directive T_NEWLINE

+		| assignment_directive T_NEWLINE

 		| line_directive /* Directives that manage newlines themselves */

 		| error T_NEWLINE { /* Continue parsing the next line on a syntax error */

 			fstk_StopRept();

--- a/test/asm/block-comment-termination-error.err

+++ b/test/asm/block-comment-termination-error.err

@@ -1,5 +1,5 @@

 ERROR: block-comment-termination-error.asm(1):

     Unterminated block comment

 ERROR: block-comment-termination-error.asm(1):

-    syntax error, unexpected end of file

+    syntax error, unexpected newline

 error: Assembly aborted (2 errors)!

--- a/test/asm/code-after-endm-endr-endc.err

+++ b/test/asm/code-after-endm-endr-endc.err

@@ -7,9 +7,9 @@

 ERROR: code-after-endm-endr-endc.asm(17):

     syntax error, unexpected PRINTLN, expecting newline

 ERROR: code-after-endm-endr-endc.asm(19):

-    syntax error, unexpected PRINTLN, expecting end of file or newline

+    syntax error, unexpected PRINTLN, expecting newline

 ERROR: code-after-endm-endr-endc.asm(23):

     syntax error, unexpected PRINTLN, expecting newline

 ERROR: code-after-endm-endr-endc.asm(25):

-    syntax error, unexpected PRINTLN, expecting end of file or newline

+    syntax error, unexpected PRINTLN, expecting newline

 error: Assembly aborted (7 errors)!