diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
index 4e83a7b..358de88 100644
--- a/.github/workflows/main.yaml
+++ b/.github/workflows/main.yaml
@@ -57,4 +57,25 @@ jobs:
       - run: tree-sitter generate
         working-directory: ./tree-sitter-p4
       - run: ./ci/format.sh
+  cli-tests:
+    name: Cli Tests
+    runs-on: ubuntu-build
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v4
+      - uses: tree-sitter/setup-action@v2
+        with:
+          tree-sitter-ref: "master"
+      - uses: swift-actions/setup-swift@v3
+        with:
+          swift-version: "main-snapshot"
+          skip-verify-signature: true
+      # Because our tree-sitter code is in a subdirectory,
+      # and working-directory does not apply to uses, we
+      # are forced to specify calls to the CI ourselves.
+      # See https://github.com/orgs/community/discussions/25742
+      - run: tree-sitter generate
+        working-directory: ./tree-sitter-p4
+      - run: swift build
+      - run: ./ci/tests/test.sh
 
diff --git a/TestData/Cli/simple.golden b/TestData/Cli/simple.golden
new file mode 100644
index 0000000..3582111
--- /dev/null
+++ b/TestData/Cli/simple.golden
@@ -0,0 +1 @@
+Success
diff --git a/ci/tests/test.sh b/ci/tests/test.sh
new file mode 100755
index 0000000..0ffa2ee
--- /dev/null
+++ b/ci/tests/test.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+actual_output=`swift run p4ce --plain compile simple.p4 -I TestData/Sources/`
+expected_output=`cat TestData/Cli/simple.golden`
+if [ "${expected_output}" != "${actual_output}" ]; then
+  echo "Expected: ${expected_output}"
+  echo "Actual: ${actual_output}"
+  echo "Cli Tests: Error"
+  exit 1
+fi
+
+echo "Cli Tests: Success"
+exit 0
+