compiler: Add Preprocessor Support

TODO: Test that file names are properly tracked in included
files.

Signed-off-by: Will Hawkins <hawkinsw@obs.cr>

Sources/Common/SourceCode.swift

@@ -0,0 +1,216 @@
+// p4rse, Copyright 2026, Will Hawkins
+//
+// This file is part of p4rse.
+//
+// This file is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+import Foundation
+import System
+
+/// Represent a location in a post-preprocessed piece of P4 source code.
+public struct SourceLocation: Equatable, CustomStringConvertible {
+
+  public let range: Range<Int>
+
+  public init(_ start: Int, _ extent: Int) {
+    self.range = start..<(start + extent)
+  }
+
+  public init(_ range: Range<Int>) {
+    self.range = range
+  }
+
+  public func contains(_ other: SourceLocation) -> Bool {
+    return self.range.contains(other.range)
+  }
+
+  public var description: String {
+    return "{\(self.range.lowerBound), \(self.self.range.upperBound - self.range.lowerBound)}"
+  }
+}
+
+/// Represent a set of directories containing P4 code that can be accessed with relative paths.
+public struct SourceManager {
+  let paths: [FilePath]
+  public init(_ paths: [FilePath]) {
+    self.paths = paths
+  }
+
+  public func firstExisting(_ file: FilePath) -> FilePath? {
+    let fm = FileManager()
+    for path in self.paths {
+      let combined = path.pushing(file)
+      if fm.fileExists(atPath: combined.string) {
+        return combined
+      }
+    }
+    return .none
+  }
+}
+
+/// Represent preprocessed P4 code and retain information about source filenames.
+public struct FileSourceLocation: Equatable, CustomStringConvertible {
+  let location: SourceLocation
+  let path: FilePath
+  let nested: [FileSourceLocation]
+
+  public init(
+    _ location: SourceLocation, _ path: FilePath, _ nested: [FileSourceLocation] = Array()
+  ) {
+    self.location = location
+    self.path = path
+    self.nested = nested
+  }
+
+  public func getLocation() -> SourceLocation {
+    return self.location
+  }
+
+  public func getPath() -> FilePath {
+    return self.path
+  }
+
+  public func getNestedLocations() -> [FileSourceLocation] {
+    return self.nested
+  }
+
+  public var description: String {
+    return "\(self.path): \(self.location) (Nested: "
+      + self.nested.map({ location in
+        return "\(location)"
+      }).joined(separator: ",") + ")"
+  }
+}
+
+/// Represent preprocessed P4 code.
+public struct SourceCode {
+  let code: String
+  let manager: SourceManager
+  let locations: FileSourceLocation
+
+  public init(_ contents: String, _ manager: SourceManager, _ locations: FileSourceLocation) {
+    self.code = contents
+    self.manager = manager
+    self.locations = locations
+  }
+
+  public func getSource() -> String {
+    return self.code
+  }
+
+  public func getManager() -> SourceManager {
+    return self.manager
+  }
+
+  public func getLocations() -> FileSourceLocation {
+    return self.locations
+  }
+}
+
+func do_preprocess(
+  _ file: URL, _ manager: SourceManager, _ starting: Int = 0
+) -> Result<(FileSourceLocation, String)> {
+  // First (1) match group has the name of the include file.
+  let re = /\#include[\s]*<([^\s>]*)>/
+
+  do {
+    var locations: [FileSourceLocation] = Array()
+    var contents = try String.init(contentsOf: file, encoding: String.defaultCStringEncoding)
+    var changed = true
+
+    while changed {
+      changed = false
+      for match in contents.matches(of: re) {
+
+        let before = contents[..<match.range.lowerBound]
+        let after = contents[match.range.upperBound...]
+
+        // Determine whether there is a file with that name in the include path.
+        guard let included_path = manager.firstExisting(FilePath.init("\(match.1)")) else {
+          return .Error(
+            Error(
+              withMessage:
+                "Could not open \(match.1) for inclusion"))
+        }
+
+        // Try to make a url from the configured file.
+        guard let included_url = URL.init(filePath: included_path) else {
+          return .Error(Error(withMessage: "Could not convert \(included_path) into a URL"))
+        }
+
+        // By calling ourselves recursively, the include being processed will
+        // be _completely_ expanded (including any nested includes).
+        switch do_preprocess(included_url, manager, starting + before.count) {
+        case .Ok((let location, let expanded)):
+          // Recombine what was before and after the include being processed
+          // with the expanded text.
+          contents = before + expanded + after
+          // Remember the location (and those it has nested) that were found in
+          // the expanded text.
+          locations.append(location)
+        case .Error(let e):
+          return .Error(e)
+        }
+
+        // Only process one at a time.
+        changed = true
+        break
+      }
+    }
+
+    return .Ok(
+      (
+        FileSourceLocation(
+          SourceLocation(starting..<(starting + contents.count)), FilePath(file.path()), locations),
+        contents
+      ))
+
+  } catch (let e) {
+    return .Error(Error(withMessage: "\(e)"))
+  }
+}
+
+/// Preprocess P4 code.
+public struct SourceCodePreprocessor {
+  let manager: SourceManager
+
+  public init(_ manager: SourceManager) {
+    self.manager = manager
+  }
+
+  public func preprocess(_ file: FilePath) -> Result<SourceCode> {
+    // First, decide whether the given file exists at the path the user gave.
+    let fm = FileManager()
+    let file_to_open =
+      if !fm.fileExists(atPath: file.string) {
+        self.manager.firstExisting(file)
+      } else {
+        file
+      }
+
+    guard let file_to_open else {
+      return .Error(Error(withMessage: "Could not open \(file) for preprocessing"))
+    }
+
+    guard let url = URL.init(filePath: file_to_open) else {
+      return .Error(Error(withMessage: "Could not convert \(file_to_open) into a URL"))
+    }
+
+    switch do_preprocess(url, self.manager) {
+    case .Ok((let location, let contents)):
+      return .Ok(SourceCode(contents, self.manager, location))
+    case .Error(let e): return .Error(e)
+    }
+  }
+}

Sources/Common/Support.swift

@@ -15,20 +15,6 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-public struct SourceLocation: Equatable, CustomStringConvertible {
-  public let start: Int
-  public let extent: Int
-
-  public init(_ start: Int, _ extent: Int) {
-    self.start = start
-    self.extent = extent
-  }
-
-  public var description: String {
-    return "{\(self.start), \(self.extent)}"
-  }
-}
-
 public enum DebugLevel {
   case Trace
   case Verbose

TestData/Sources/nested-split-2.p4

@@ -0,0 +1,2 @@
+#include <nested-split-a.p4>
+#include <nested-split-b.p4>

TestData/Sources/nested-split-a.p4

@@ -0,0 +1 @@
+       state start {

TestData/Sources/nested-split-b.p4

@@ -0,0 +1,5 @@
+           transition select (false) {
+              true: reject;
+              false: reject;
+           };
+       }

TestData/Sources/nested-split.p4

@@ -0,0 +1,3 @@
+    parser main_parser() {
+#include <nested-split-2.p4>
+    };

TestData/Sources/simple-split-oneline.p4

@@ -0,0 +1 @@
+#include <testing-split-oneline-a.p4>#include <testing-split-oneline-b.p4>

TestData/Sources/simple-split.p4

@@ -0,0 +1 @@
+#include <testing-split-a.p4>

TestData/Sources/simple-unfound.p4

@@ -0,0 +1 @@
+#include <unfound.p4>

TestData/Sources/simple.p4

@@ -0,0 +1 @@
+#include <testing.p4>

TestData/Sources/testing-split-a.p4

@@ -0,0 +1,3 @@
+    parser main_parser() {
+       state start {
+#include <testing-split-b.p4>

TestData/Sources/testing-split-b.p4

@@ -0,0 +1,6 @@
+           transition select (false) {
+              true: reject;
+              false: reject;
+           };
+       }
+    };

TestData/Sources/testing-split-oneline-a.p4

@@ -0,0 +1,3 @@
+    parser main_parser() {
+       state start {
+           transition 

TestData/Sources/testing-split-oneline-b.p4

@@ -0,0 +1,6 @@
+select (false) {
+              true: reject;
+              false: reject;
+           };
+       }
+    };

TestData/Sources/testing.p4

@@ -0,0 +1,8 @@
+    parser main_parser() {
+       state start {
+           transition select (false) {
+              true: reject;
+              false: reject;
+           };
+       }
+    };

Tests/p4rseTests/SupportTests/SourceCode.swift

@@ -0,0 +1,214 @@
+// p4rse, Copyright 2026, Will Hawkins
+//
+// This file is part of p4rse.
+//
+// This file is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+import Foundation
+import Macros
+import P4Compiler
+import P4Lang
+import P4Runtime
+import SwiftTreeSitter
+import System
+import Testing
+import TreeSitter
+import TreeSitterP4
+
+@testable import Common
+
+@Test func test_preprocessor() async throws {
+  let sm = SourceManager(["./TestData/Sources/"])
+  let prep = SourceCodePreprocessor(sm)
+  let file = FilePath.init(stringLiteral: "./TestData/Sources/simple.p4")
+  let expected_file = FilePath.init(FileManager().currentDirectoryPath + "/" + file.string).lexicallyNormalized()
+
+  let source = try! (#UseOkResult(prep.preprocess(file)))
+  let program = try! #UseOkResult(Program.Compile(source.getSource()))
+  #expect(#RequireOkResult((program.find_parser(withName: Identifier(name: "main_parser")))))
+  #expect(source.getLocations().getPath() == expected_file)
+}
+
+@Test func test_preprocessor_search_for_file() async throws {
+  let sm = SourceManager(["./TestData/Sources/"])
+  let prep = SourceCodePreprocessor(sm)
+  let file = FilePath.init(stringLiteral: "simple.p4")
+  let expected_file = FilePath.init(FileManager().currentDirectoryPath + "/TestData/Sources/simple.p4")
+
+  let source = try! (#UseOkResult(prep.preprocess(file)))
+  let program = try! #UseOkResult(Program.Compile(source.getSource()))
+  #expect(#RequireOkResult((program.find_parser(withName: Identifier(name: "main_parser")))))
+  #expect(source.getLocations().getPath() == expected_file)
+}
+
+@Test func test_preprocessor_nested_includes() async throws {
+  let sm = SourceManager(["./TestData/Sources/"])
+  let prep = SourceCodePreprocessor(sm)
+  let file = FilePath.init(stringLiteral: "./TestData/Sources/simple-split.p4")
+  let expected_file = FilePath.init(FileManager().currentDirectoryPath + "/" + file.string).lexicallyNormalized()
+
+  #expect(#RequireOkResult(prep.preprocess(file)))
+
+  let source = try! (#UseOkResult(prep.preprocess(file)))
+  let program = try! #UseOkResult(Program.Compile(source.getSource()))
+  #expect(#RequireOkResult((program.find_parser(withName: Identifier(name: "main_parser")))))
+  #expect(source.getLocations().getPath() == expected_file)
+}
+
+@Test func test_preprocessor_oneline_includes() async throws {
+  let sm = SourceManager(["./TestData/Sources/"])
+  let prep = SourceCodePreprocessor(sm)
+  let file = FilePath.init(stringLiteral: "./TestData/Sources/simple-split-oneline.p4")
+  let expected_file = FilePath.init(FileManager().currentDirectoryPath + "/" + file.string).lexicallyNormalized()
+
+  #expect(#RequireOkResult(prep.preprocess(file)))
+
+  let source = try! (#UseOkResult(prep.preprocess(file)))
+  let program = try! #UseOkResult(Program.Compile(source.getSource()))
+  #expect(#RequireOkResult((program.find_parser(withName: Identifier(name: "main_parser")))))
+  #expect(source.getLocations().getPath() == expected_file)
+}
+
+@Test func test_preprocessor_missing_file() async throws {
+  let sm = SourceManager(["./TestData/Sources/"])
+  let prep = SourceCodePreprocessor(sm)
+  let file = FilePath.init(stringLiteral: "./TestData/Sources/unfound.p4")
+
+  #expect(
+    #RequireErrorResult(
+      Error(withMessage: "Could not open ./TestData/Sources/unfound.p4 for preprocessing"),
+      (prep.preprocess(file))))
+
+}
+
+@Test func test_preprocessor_missing_included_file() async throws {
+  let sm = SourceManager(["./TestData/Sources/"])
+  let prep = SourceCodePreprocessor(sm)
+  let file = FilePath.init(stringLiteral: "./TestData/Sources/simple-unfound.p4")
+
+  #expect(
+    #RequireErrorResult(
+      Error(withMessage: "Could not open unfound.p4 for inclusion"), (prep.preprocess(file))))
+
+}
+
+@Test func test_preprocessor_no_change_locations() async throws {
+  let sm = SourceManager(["./TestData/Sources/"])
+  let prep = SourceCodePreprocessor(sm)
+  let file = FilePath.init(stringLiteral: "testing.p4")
+
+  let source = try! (#UseOkResult(prep.preprocess(file)))
+
+  #expect(source.getLocations().getLocation() == SourceLocation(0..<173))
+}
+
+@Test func test_preprocessor_change_locations() async throws {
+  let sm = SourceManager(["./TestData/Sources/"])
+  let prep = SourceCodePreprocessor(sm)
+  let file = FilePath.init(stringLiteral: "testing-split-a.p4")
+
+  let source = try! (#UseOkResult(prep.preprocess(file)))
+
+  #expect(source.getLocations().getLocation() == SourceLocation(0..<173))
+  #expect(source.getLocations().getNestedLocations()[0].getLocation() == SourceLocation(48..<173))
+}
+
+@Test func test_preprocessor_oneline_includes_locations() async throws {
+  let sm = SourceManager(["./TestData/Sources/"])
+  let prep = SourceCodePreprocessor(sm)
+  let file = FilePath.init(stringLiteral: "./TestData/Sources/simple-split-oneline.p4")
+
+  #expect(#RequireOkResult(prep.preprocess(file)))
+
+  let source = try! (#UseOkResult(prep.preprocess(file)))
+
+  #expect(source.getLocations().getLocation() == SourceLocation(0..<173))
+  #expect(source.getLocations().getNestedLocations()[0].getLocation() == SourceLocation(0..<70))
+  #expect(source.getLocations().getNestedLocations()[1].getLocation() == SourceLocation(70..<173))
+}
+
+@Test func test_preprocessor_nested_includes_locations() async throws {
+  let sm = SourceManager(["./TestData/Sources/"])
+  let prep = SourceCodePreprocessor(sm)
+  let file = FilePath.init(stringLiteral: "./TestData/Sources/nested-split.p4")
+
+  #expect(#RequireOkResult(prep.preprocess(file)))
+
+  let source = try! (#UseOkResult(prep.preprocess(file)))
+
+  #expect(source.getLocations().getLocation() == SourceLocation(0..<173))
+  #expect(
+    source.getLocations().getNestedLocations()[0].getNestedLocations()[0].getLocation()
+      == SourceLocation(27..<47))
+  #expect(
+    source.getLocations().getNestedLocations()[0].getNestedLocations()[1].getLocation()
+      == SourceLocation(48..<166))
+}
+
+
+@Test func test_source_location_contains() async throws {
+  let outer = SourceLocation(0..<500)
+  let inner = SourceLocation(0..<499)
+
+  #expect(outer.contains(inner))
+}
+
+@Test func test_source_location_contains_a() async throws {
+  let outer = SourceLocation(0, 500)
+  let inner = SourceLocation(0, 499)
+
+  #expect(outer.contains(inner))
+}
+
+@Test func test_source_location_contains2() async throws {
+  let outer = SourceLocation(0..<500)
+  let not_inner = SourceLocation(0..<501)
+
+  #expect(!outer.contains(not_inner))
+}
+
+@Test func test_source_location_contains2_a() async throws {
+  let outer = SourceLocation(0,500)
+  let not_inner = SourceLocation(0,501)
+
+  #expect(!outer.contains(not_inner))
+}
+
+@Test func test_source_location_contains3() async throws {
+  let outer = SourceLocation(200..<500)
+  let inner = SourceLocation(200..<499)
+
+  #expect(outer.contains(inner))
+}
+
+@Test func test_source_location_contains3_a() async throws {
+  let outer = SourceLocation(200,300)
+  let inner = SourceLocation(200,299)
+
+  #expect(outer.contains(inner))
+}
+
+@Test func test_source_location_contains4() async throws {
+  let outer = SourceLocation(200..<500)
+  let inner = SourceLocation(200, 299)
+
+  #expect(outer.contains(inner))
+}
+
+@Test func test_source_location_contains5() async throws {
+  let outer = SourceLocation(200..<300)
+  let not_inner = SourceLocation(200,101)
+
+  #expect(!outer.contains(not_inner))
+}