From 754102d601d1fb7e4af378a44dc3f41aae79e06a Mon Sep 17 00:00:00 2001 From: Will Hawkins Date: Thu, 7 May 2026 23:05:47 -0400 Subject: [PATCH] compiler: Add Preprocessor Support TODO: Test that file names are properly tracked in included files. Signed-off-by: Will Hawkins --- Sources/Common/SourceCode.swift | 216 ++++++++++++++++++ Sources/Common/Support.swift | 14 -- TestData/Sources/nested-split-2.p4 | 2 + TestData/Sources/nested-split-a.p4 | 1 + TestData/Sources/nested-split-b.p4 | 5 + TestData/Sources/nested-split.p4 | 3 + TestData/Sources/simple-split-oneline.p4 | 1 + TestData/Sources/simple-split.p4 | 1 + TestData/Sources/simple-unfound.p4 | 1 + TestData/Sources/simple.p4 | 1 + TestData/Sources/testing-split-a.p4 | 3 + TestData/Sources/testing-split-b.p4 | 6 + TestData/Sources/testing-split-oneline-a.p4 | 3 + TestData/Sources/testing-split-oneline-b.p4 | 6 + TestData/Sources/testing.p4 | 8 + .../p4rseTests/SupportTests/SourceCode.swift | 214 +++++++++++++++++ 16 files changed, 471 insertions(+), 14 deletions(-) create mode 100644 Sources/Common/SourceCode.swift create mode 100644 TestData/Sources/nested-split-2.p4 create mode 100644 TestData/Sources/nested-split-a.p4 create mode 100644 TestData/Sources/nested-split-b.p4 create mode 100644 TestData/Sources/nested-split.p4 create mode 100644 TestData/Sources/simple-split-oneline.p4 create mode 100644 TestData/Sources/simple-split.p4 create mode 100644 TestData/Sources/simple-unfound.p4 create mode 100644 TestData/Sources/simple.p4 create mode 100644 TestData/Sources/testing-split-a.p4 create mode 100644 TestData/Sources/testing-split-b.p4 create mode 100644 TestData/Sources/testing-split-oneline-a.p4 create mode 100644 TestData/Sources/testing-split-oneline-b.p4 create mode 100644 TestData/Sources/testing.p4 create mode 100644 Tests/p4rseTests/SupportTests/SourceCode.swift diff --git a/Sources/Common/SourceCode.swift b/Sources/Common/SourceCode.swift new file mode 100644 index 0000000..f839be4 --- /dev/null +++ b/Sources/Common/SourceCode.swift @@ -0,0 +1,216 @@ +// p4rse, Copyright 2026, Will Hawkins +// +// This file is part of p4rse. +// +// This file is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +import Foundation +import System + +/// Represent a location in a post-preprocessed piece of P4 source code. +public struct SourceLocation: Equatable, CustomStringConvertible { + + public let range: Range + + public init(_ start: Int, _ extent: Int) { + self.range = start..<(start + extent) + } + + public init(_ range: Range) { + self.range = range + } + + public func contains(_ other: SourceLocation) -> Bool { + return self.range.contains(other.range) + } + + public var description: String { + return "{\(self.range.lowerBound), \(self.self.range.upperBound - self.range.lowerBound)}" + } +} + +/// Represent a set of directories containing P4 code that can be accessed with relative paths. +public struct SourceManager { + let paths: [FilePath] + public init(_ paths: [FilePath]) { + self.paths = paths + } + + public func firstExisting(_ file: FilePath) -> FilePath? { + let fm = FileManager() + for path in self.paths { + let combined = path.pushing(file) + if fm.fileExists(atPath: combined.string) { + return combined + } + } + return .none + } +} + +/// Represent preprocessed P4 code and retain information about source filenames. +public struct FileSourceLocation: Equatable, CustomStringConvertible { + let location: SourceLocation + let path: FilePath + let nested: [FileSourceLocation] + + public init( + _ location: SourceLocation, _ path: FilePath, _ nested: [FileSourceLocation] = Array() + ) { + self.location = location + self.path = path + self.nested = nested + } + + public func getLocation() -> SourceLocation { + return self.location + } + + public func getPath() -> FilePath { + return self.path + } + + public func getNestedLocations() -> [FileSourceLocation] { + return self.nested + } + + public var description: String { + return "\(self.path): \(self.location) (Nested: " + + self.nested.map({ location in + return "\(location)" + }).joined(separator: ",") + ")" + } +} + +/// Represent preprocessed P4 code. +public struct SourceCode { + let code: String + let manager: SourceManager + let locations: FileSourceLocation + + public init(_ contents: String, _ manager: SourceManager, _ locations: FileSourceLocation) { + self.code = contents + self.manager = manager + self.locations = locations + } + + public func getSource() -> String { + return self.code + } + + public func getManager() -> SourceManager { + return self.manager + } + + public func getLocations() -> FileSourceLocation { + return self.locations + } +} + +func do_preprocess( + _ file: URL, _ manager: SourceManager, _ starting: Int = 0 +) -> Result<(FileSourceLocation, String)> { + // First (1) match group has the name of the include file. + let re = /\#include[\s]*<([^\s>]*)>/ + + do { + var locations: [FileSourceLocation] = Array() + var contents = try String.init(contentsOf: file, encoding: String.defaultCStringEncoding) + var changed = true + + while changed { + changed = false + for match in contents.matches(of: re) { + + let before = contents[.. Result { + // First, decide whether the given file exists at the path the user gave. + let fm = FileManager() + let file_to_open = + if !fm.fileExists(atPath: file.string) { + self.manager.firstExisting(file) + } else { + file + } + + guard let file_to_open else { + return .Error(Error(withMessage: "Could not open \(file) for preprocessing")) + } + + guard let url = URL.init(filePath: file_to_open) else { + return .Error(Error(withMessage: "Could not convert \(file_to_open) into a URL")) + } + + switch do_preprocess(url, self.manager) { + case .Ok((let location, let contents)): + return .Ok(SourceCode(contents, self.manager, location)) + case .Error(let e): return .Error(e) + } + } +} diff --git a/Sources/Common/Support.swift b/Sources/Common/Support.swift index c5d928f..84d5846 100644 --- a/Sources/Common/Support.swift +++ b/Sources/Common/Support.swift @@ -15,20 +15,6 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -public struct SourceLocation: Equatable, CustomStringConvertible { - public let start: Int - public let extent: Int - - public init(_ start: Int, _ extent: Int) { - self.start = start - self.extent = extent - } - - public var description: String { - return "{\(self.start), \(self.extent)}" - } -} - public enum DebugLevel { case Trace case Verbose diff --git a/TestData/Sources/nested-split-2.p4 b/TestData/Sources/nested-split-2.p4 new file mode 100644 index 0000000..ba6e593 --- /dev/null +++ b/TestData/Sources/nested-split-2.p4 @@ -0,0 +1,2 @@ +#include +#include \ No newline at end of file diff --git a/TestData/Sources/nested-split-a.p4 b/TestData/Sources/nested-split-a.p4 new file mode 100644 index 0000000..afdb571 --- /dev/null +++ b/TestData/Sources/nested-split-a.p4 @@ -0,0 +1 @@ + state start { \ No newline at end of file diff --git a/TestData/Sources/nested-split-b.p4 b/TestData/Sources/nested-split-b.p4 new file mode 100644 index 0000000..71aebc2 --- /dev/null +++ b/TestData/Sources/nested-split-b.p4 @@ -0,0 +1,5 @@ + transition select (false) { + true: reject; + false: reject; + }; + } \ No newline at end of file diff --git a/TestData/Sources/nested-split.p4 b/TestData/Sources/nested-split.p4 new file mode 100644 index 0000000..f04b671 --- /dev/null +++ b/TestData/Sources/nested-split.p4 @@ -0,0 +1,3 @@ + parser main_parser() { +#include + }; \ No newline at end of file diff --git a/TestData/Sources/simple-split-oneline.p4 b/TestData/Sources/simple-split-oneline.p4 new file mode 100644 index 0000000..1c52c75 --- /dev/null +++ b/TestData/Sources/simple-split-oneline.p4 @@ -0,0 +1 @@ +#include #include \ No newline at end of file diff --git a/TestData/Sources/simple-split.p4 b/TestData/Sources/simple-split.p4 new file mode 100644 index 0000000..519e8e1 --- /dev/null +++ b/TestData/Sources/simple-split.p4 @@ -0,0 +1 @@ +#include \ No newline at end of file diff --git a/TestData/Sources/simple-unfound.p4 b/TestData/Sources/simple-unfound.p4 new file mode 100644 index 0000000..93e1eb6 --- /dev/null +++ b/TestData/Sources/simple-unfound.p4 @@ -0,0 +1 @@ +#include \ No newline at end of file diff --git a/TestData/Sources/simple.p4 b/TestData/Sources/simple.p4 new file mode 100644 index 0000000..7708077 --- /dev/null +++ b/TestData/Sources/simple.p4 @@ -0,0 +1 @@ +#include \ No newline at end of file diff --git a/TestData/Sources/testing-split-a.p4 b/TestData/Sources/testing-split-a.p4 new file mode 100644 index 0000000..9b5e5c2 --- /dev/null +++ b/TestData/Sources/testing-split-a.p4 @@ -0,0 +1,3 @@ + parser main_parser() { + state start { +#include \ No newline at end of file diff --git a/TestData/Sources/testing-split-b.p4 b/TestData/Sources/testing-split-b.p4 new file mode 100644 index 0000000..8977da4 --- /dev/null +++ b/TestData/Sources/testing-split-b.p4 @@ -0,0 +1,6 @@ + transition select (false) { + true: reject; + false: reject; + }; + } + }; \ No newline at end of file diff --git a/TestData/Sources/testing-split-oneline-a.p4 b/TestData/Sources/testing-split-oneline-a.p4 new file mode 100644 index 0000000..f38646d --- /dev/null +++ b/TestData/Sources/testing-split-oneline-a.p4 @@ -0,0 +1,3 @@ + parser main_parser() { + state start { + transition \ No newline at end of file diff --git a/TestData/Sources/testing-split-oneline-b.p4 b/TestData/Sources/testing-split-oneline-b.p4 new file mode 100644 index 0000000..38d9068 --- /dev/null +++ b/TestData/Sources/testing-split-oneline-b.p4 @@ -0,0 +1,6 @@ +select (false) { + true: reject; + false: reject; + }; + } + }; \ No newline at end of file diff --git a/TestData/Sources/testing.p4 b/TestData/Sources/testing.p4 new file mode 100644 index 0000000..df9527a --- /dev/null +++ b/TestData/Sources/testing.p4 @@ -0,0 +1,8 @@ + parser main_parser() { + state start { + transition select (false) { + true: reject; + false: reject; + }; + } + }; \ No newline at end of file diff --git a/Tests/p4rseTests/SupportTests/SourceCode.swift b/Tests/p4rseTests/SupportTests/SourceCode.swift new file mode 100644 index 0000000..b32ba72 --- /dev/null +++ b/Tests/p4rseTests/SupportTests/SourceCode.swift @@ -0,0 +1,214 @@ +// p4rse, Copyright 2026, Will Hawkins +// +// This file is part of p4rse. +// +// This file is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +import Foundation +import Macros +import P4Compiler +import P4Lang +import P4Runtime +import SwiftTreeSitter +import System +import Testing +import TreeSitter +import TreeSitterP4 + +@testable import Common + +@Test func test_preprocessor() async throws { + let sm = SourceManager(["./TestData/Sources/"]) + let prep = SourceCodePreprocessor(sm) + let file = FilePath.init(stringLiteral: "./TestData/Sources/simple.p4") + let expected_file = FilePath.init(FileManager().currentDirectoryPath + "/" + file.string).lexicallyNormalized() + + let source = try! (#UseOkResult(prep.preprocess(file))) + let program = try! #UseOkResult(Program.Compile(source.getSource())) + #expect(#RequireOkResult((program.find_parser(withName: Identifier(name: "main_parser"))))) + #expect(source.getLocations().getPath() == expected_file) +} + +@Test func test_preprocessor_search_for_file() async throws { + let sm = SourceManager(["./TestData/Sources/"]) + let prep = SourceCodePreprocessor(sm) + let file = FilePath.init(stringLiteral: "simple.p4") + let expected_file = FilePath.init(FileManager().currentDirectoryPath + "/TestData/Sources/simple.p4") + + let source = try! (#UseOkResult(prep.preprocess(file))) + let program = try! #UseOkResult(Program.Compile(source.getSource())) + #expect(#RequireOkResult((program.find_parser(withName: Identifier(name: "main_parser"))))) + #expect(source.getLocations().getPath() == expected_file) +} + +@Test func test_preprocessor_nested_includes() async throws { + let sm = SourceManager(["./TestData/Sources/"]) + let prep = SourceCodePreprocessor(sm) + let file = FilePath.init(stringLiteral: "./TestData/Sources/simple-split.p4") + let expected_file = FilePath.init(FileManager().currentDirectoryPath + "/" + file.string).lexicallyNormalized() + + #expect(#RequireOkResult(prep.preprocess(file))) + + let source = try! (#UseOkResult(prep.preprocess(file))) + let program = try! #UseOkResult(Program.Compile(source.getSource())) + #expect(#RequireOkResult((program.find_parser(withName: Identifier(name: "main_parser"))))) + #expect(source.getLocations().getPath() == expected_file) +} + +@Test func test_preprocessor_oneline_includes() async throws { + let sm = SourceManager(["./TestData/Sources/"]) + let prep = SourceCodePreprocessor(sm) + let file = FilePath.init(stringLiteral: "./TestData/Sources/simple-split-oneline.p4") + let expected_file = FilePath.init(FileManager().currentDirectoryPath + "/" + file.string).lexicallyNormalized() + + #expect(#RequireOkResult(prep.preprocess(file))) + + let source = try! (#UseOkResult(prep.preprocess(file))) + let program = try! #UseOkResult(Program.Compile(source.getSource())) + #expect(#RequireOkResult((program.find_parser(withName: Identifier(name: "main_parser"))))) + #expect(source.getLocations().getPath() == expected_file) +} + +@Test func test_preprocessor_missing_file() async throws { + let sm = SourceManager(["./TestData/Sources/"]) + let prep = SourceCodePreprocessor(sm) + let file = FilePath.init(stringLiteral: "./TestData/Sources/unfound.p4") + + #expect( + #RequireErrorResult( + Error(withMessage: "Could not open ./TestData/Sources/unfound.p4 for preprocessing"), + (prep.preprocess(file)))) + +} + +@Test func test_preprocessor_missing_included_file() async throws { + let sm = SourceManager(["./TestData/Sources/"]) + let prep = SourceCodePreprocessor(sm) + let file = FilePath.init(stringLiteral: "./TestData/Sources/simple-unfound.p4") + + #expect( + #RequireErrorResult( + Error(withMessage: "Could not open unfound.p4 for inclusion"), (prep.preprocess(file)))) + +} + +@Test func test_preprocessor_no_change_locations() async throws { + let sm = SourceManager(["./TestData/Sources/"]) + let prep = SourceCodePreprocessor(sm) + let file = FilePath.init(stringLiteral: "testing.p4") + + let source = try! (#UseOkResult(prep.preprocess(file))) + + #expect(source.getLocations().getLocation() == SourceLocation(0..<173)) +} + +@Test func test_preprocessor_change_locations() async throws { + let sm = SourceManager(["./TestData/Sources/"]) + let prep = SourceCodePreprocessor(sm) + let file = FilePath.init(stringLiteral: "testing-split-a.p4") + + let source = try! (#UseOkResult(prep.preprocess(file))) + + #expect(source.getLocations().getLocation() == SourceLocation(0..<173)) + #expect(source.getLocations().getNestedLocations()[0].getLocation() == SourceLocation(48..<173)) +} + +@Test func test_preprocessor_oneline_includes_locations() async throws { + let sm = SourceManager(["./TestData/Sources/"]) + let prep = SourceCodePreprocessor(sm) + let file = FilePath.init(stringLiteral: "./TestData/Sources/simple-split-oneline.p4") + + #expect(#RequireOkResult(prep.preprocess(file))) + + let source = try! (#UseOkResult(prep.preprocess(file))) + + #expect(source.getLocations().getLocation() == SourceLocation(0..<173)) + #expect(source.getLocations().getNestedLocations()[0].getLocation() == SourceLocation(0..<70)) + #expect(source.getLocations().getNestedLocations()[1].getLocation() == SourceLocation(70..<173)) +} + +@Test func test_preprocessor_nested_includes_locations() async throws { + let sm = SourceManager(["./TestData/Sources/"]) + let prep = SourceCodePreprocessor(sm) + let file = FilePath.init(stringLiteral: "./TestData/Sources/nested-split.p4") + + #expect(#RequireOkResult(prep.preprocess(file))) + + let source = try! (#UseOkResult(prep.preprocess(file))) + + #expect(source.getLocations().getLocation() == SourceLocation(0..<173)) + #expect( + source.getLocations().getNestedLocations()[0].getNestedLocations()[0].getLocation() + == SourceLocation(27..<47)) + #expect( + source.getLocations().getNestedLocations()[0].getNestedLocations()[1].getLocation() + == SourceLocation(48..<166)) +} + + +@Test func test_source_location_contains() async throws { + let outer = SourceLocation(0..<500) + let inner = SourceLocation(0..<499) + + #expect(outer.contains(inner)) +} + +@Test func test_source_location_contains_a() async throws { + let outer = SourceLocation(0, 500) + let inner = SourceLocation(0, 499) + + #expect(outer.contains(inner)) +} + +@Test func test_source_location_contains2() async throws { + let outer = SourceLocation(0..<500) + let not_inner = SourceLocation(0..<501) + + #expect(!outer.contains(not_inner)) +} + +@Test func test_source_location_contains2_a() async throws { + let outer = SourceLocation(0,500) + let not_inner = SourceLocation(0,501) + + #expect(!outer.contains(not_inner)) +} + +@Test func test_source_location_contains3() async throws { + let outer = SourceLocation(200..<500) + let inner = SourceLocation(200..<499) + + #expect(outer.contains(inner)) +} + +@Test func test_source_location_contains3_a() async throws { + let outer = SourceLocation(200,300) + let inner = SourceLocation(200,299) + + #expect(outer.contains(inner)) +} + +@Test func test_source_location_contains4() async throws { + let outer = SourceLocation(200..<500) + let inner = SourceLocation(200, 299) + + #expect(outer.contains(inner)) +} + +@Test func test_source_location_contains5() async throws { + let outer = SourceLocation(200..<300) + let not_inner = SourceLocation(200,101) + + #expect(!outer.contains(not_inner)) +} \ No newline at end of file