From e68234c4f991071c0c7ff05852975a74d6a21771 Mon Sep 17 00:00:00 2001
From: Robert Hensing <robert@roberthensing.nl>
Date: Tue, 9 Jul 2024 15:39:24 +0200
Subject: [PATCH] libexpr: Rearrange lexer files so that yylex_init_extra can
 be found

---
 src/libexpr/lexer-helpers.cc | 31 ++++++++++++++++++++++++++
 src/libexpr/lexer-helpers.hh |  9 ++++++++
 src/libexpr/lexer.l          | 43 ++++++++----------------------------
 src/libexpr/meson.build      |  2 ++
 4 files changed, 51 insertions(+), 34 deletions(-)
 create mode 100644 src/libexpr/lexer-helpers.cc
 create mode 100644 src/libexpr/lexer-helpers.hh

diff --git a/src/libexpr/lexer-helpers.cc b/src/libexpr/lexer-helpers.cc
new file mode 100644
index 000000000..87c514c71
--- /dev/null
+++ b/src/libexpr/lexer-helpers.cc
@@ -0,0 +1,31 @@
+#include "lexer-tab.hh"
+#include "lexer-helpers.hh"
+#include "parser-tab.hh"
+
+void nix::lexer::internal::initLoc(YYLTYPE * loc)
+{
+    loc->first_line = loc->last_line = 0;
+    loc->first_column = loc->last_column = 0;
+}
+
+void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len)
+{
+    loc->stash();
+
+    LexerState & lexerState = *yyget_extra(yyscanner);
+
+    if (lexerState.docCommentDistance == 1) {
+        // Preceding token was a doc comment.
+        ParserLocation doc;
+        doc.first_column = lexerState.lastDocCommentLoc.first_column;
+        ParserLocation docEnd;
+        docEnd.first_column = lexerState.lastDocCommentLoc.last_column;
+        DocComment docComment{lexerState.at(doc), lexerState.at(docEnd)};
+        PosIdx locPos = lexerState.at(*loc);
+        lexerState.positionToDocComment.emplace(locPos, docComment);
+    }
+    lexerState.docCommentDistance++;
+
+    loc->first_column = loc->last_column;
+    loc->last_column += len;
+}
diff --git a/src/libexpr/lexer-helpers.hh b/src/libexpr/lexer-helpers.hh
new file mode 100644
index 000000000..caba6e18f
--- /dev/null
+++ b/src/libexpr/lexer-helpers.hh
@@ -0,0 +1,9 @@
+#pragma once
+
+namespace nix::lexer::internal {
+
+void initLoc(YYLTYPE * loc);
+
+void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len);
+
+} // namespace nix::lexer
diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index 459f1d6f3..7d4e6963f 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -14,6 +14,10 @@
 %x INPATH_SLASH
 %x PATH_START
 
+%top {
+#include "parser-tab.hh" // YYSTYPE
+#include "parser-state.hh"
+}
 
 %{
 #ifdef __clang__
@@ -22,48 +26,19 @@
 
 #include "nixexpr.hh"
 #include "parser-tab.hh"
+#include "lexer-helpers.hh"
 
-// !!! FIXME !!!
-#define YY_EXTRA_TYPE ::nix::LexerState *
-int yylex_init_extra ( YY_EXTRA_TYPE user_defined, yyscan_t* scanner);
-YY_EXTRA_TYPE yyget_extra ( yyscan_t yyscanner );
-#undef YY_EXTRA_TYPE
+namespace nix {
+  struct LexerState;
+}
 
 using namespace nix;
+using namespace nix::lexer::internal;
 
 namespace nix {
 
 #define CUR_POS state->at(*yylloc)
 
-static void initLoc(YYLTYPE * loc)
-{
-    loc->first_line = loc->last_line = 0;
-    loc->first_column = loc->last_column = 0;
-}
-
-static void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len)
-{
-    loc->stash();
-
-    LexerState & lexerState = *yyget_extra(yyscanner);
-
-    if (lexerState.docCommentDistance == 1) {
-        // Preceding token was a doc comment.
-        ParserLocation doc;
-        doc.first_column = lexerState.lastDocCommentLoc.first_column;
-        ParserLocation docEnd;
-        docEnd.first_column = lexerState.lastDocCommentLoc.last_column;
-        DocComment docComment{lexerState.at(doc), lexerState.at(docEnd)};
-        PosIdx locPos = lexerState.at(*loc);
-        lexerState.positionToDocComment.emplace(locPos, docComment);
-    }
-    lexerState.docCommentDistance++;
-
-    loc->first_column = loc->last_column;
-    loc->last_column += len;
-}
-
-
 // we make use of the fact that the parser receives a private copy of the input
 // string and can munge around in it.
 static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length)
diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build
index e65cf545f..fa90e7b41 100644
--- a/src/libexpr/meson.build
+++ b/src/libexpr/meson.build
@@ -139,6 +139,7 @@ sources = files(
   'function-trace.cc',
   'get-drvs.cc',
   'json-to-value.cc',
+  'lexer-helpers.cc',
   'nixexpr.cc',
   'paths.cc',
   'primops.cc',
@@ -165,6 +166,7 @@ headers = [config_h] + files(
   'gc-small-vector.hh',
   'get-drvs.hh',
   'json-to-value.hh',
+  # internal: 'lexer-helpers.hh',
   'nixexpr.hh',
   'parser-state.hh',
   'pos-idx.hh',