ci: split PR and release certification gates (#161)

.aiwg/bt6-maintainer.yaml

@@ -19,9 +19,15 @@ delivery:
   allowedMergeMethods: ["squash"]
 validation:
   quick:
-    - "npm run typecheck"
+    - "npm run test:pr"
+    - "COVERAGE_BASE=upstream/main PR_CHANGED_LINE_COVERAGE=50 npm run test:pr-coverage"
   full:
-    - "npm test"
+    - "npm run test:release"
+    - "npm audit --audit-level=high"
+  qualityPolicy:
+    pullRequestChangedLineCoverageFloor: 50
+    requireBehaviorTests: true
+    fullSuiteTrigger: "release-tag-certification"
   documentation:
     - "npm run docs:check"
   researchIntegrity:
@@ -107,6 +113,13 @@ research:
     - "notes/**"
   provenanceRequired: true
   citationVerificationRequired: true
+releaseEvidence:
+  artifactType: "source-zip"
+  hashAlgorithm: "sha256"
+  provenanceFormat: "slsa-v1"
+  attestationFormat: "sigstore-bundle"
+  snapshotOnce: true
+  verifyBeforePromotion: true
 security:
   disclosureUrl: "https://forms.gle/QvKoijJMtEhLG7nf8"
   sensitiveDataPaths:

.aiwg/deployment/ci-cd-scaffold.md

@@ -66,14 +66,20 @@ N/A for a transactional application database. Local configuration migrations are
 
 ## 10. Verification and Validation
 
-The current pipeline runs dependency installation, lint, typecheck, tests, coverage, doctor, claim verification, anti-fitting, provenance, prompt audit, and smoke checks. Release verification additionally confirms the actual packaged artifact starts, the local health route responds, interface documentation matches behavior, and sensitive/ignored corpora are absent.
+The pipeline intentionally has two gates. Pull requests run dependency
+installation, lint, typecheck, the deterministic test suite, doctor, and a 50%
+changed-executable-line coverage floor. Release tags rerun the complete
+per-file coverage contract plus claim verification, anti-fitting, provenance,
+prompt, smoke, build, dependency-audit, and package-manifest checks. The tested
+tag is archived once as a ZIP, checksum-verified, and bound to a retained
+Sigstore provenance bundle.
 
 Proposed additions:
 
 1. SDLC artifact/citation/traceability conformance.
 2. Cross-document maturity consistency.
 3. Network-adapter scope/approval/credential/timeout inventory completeness.
-4. Release artifact checksums/signatures and SBOM according to an approved supply-chain threat model.
+4. CycloneDX SBOM generation bound to the source ZIP.
 
 ## 11. Rollback and Contingency
 

.github/pull_request_template.md

@@ -33,4 +33,5 @@ Exact commands and results:
 - [ ] Diff is scoped against current `upstream/main`
 - [ ] No unrelated protected evidence, safety tests, or provider/config files were removed
 - [ ] Published review history was not force-pushed
-- [ ] Exact-head CI must be green before merge
+- [ ] Exact-head PR CI is green (including the 50% changed-line coverage floor)
+- [ ] I understand PR acceptance is not release certification

.github/workflows/ci.yml

@@ -18,40 +18,47 @@ concurrency:
 
 jobs:
   test:
+    name: Pull request gate
+    if: github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
       - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          fetch-depth: 0
       - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
         with:
           node-version: 22
           cache: npm
       - run: npm ci
-      - run: npm run lint
+      - name: Run PR correctness gate
-      - run: npm run typecheck
+        run: npm run test:pr
-      - run: npm test
+      - name: Resolve comparison base
-      # Enforce the 100% coverage gate on the new multi-language ingest files
+        env:
-      # (vitest exits non-zero if a per-file threshold is missed).
+          EVENT_BASE: ${{ github.event.pull_request.base.sha || github.event.before }}
-      - run: npm run test:coverage
+        run: |
-      - run: npm run doctor
+          coverage_base="$EVENT_BASE"
-      # Honesty gates: prove the headline numbers re-derive and that no
+          if ! git rev-parse --verify "${coverage_base}^{commit}" >/dev/null 2>&1; then
-      # challenge-fitting has crept in. Public contributors / CI can verify.
+            coverage_base="$(git rev-parse HEAD^)"
-      - run: npm run verify-claims
+          fi
-      - run: npm run test:no-fitting
+          echo "COVERAGE_BASE=$coverage_base" >> "$GITHUB_ENV"
-      - run: npm run test:no-self-fitting
+      - name: Enforce 50% changed-line coverage floor
-      # The provenance gate itself, in the compiled engine path (builds first).
+        env:
-      - run: npm run test:gate
+          PR_CHANGED_LINE_COVERAGE: "50"
-      # The operator doctrine / prompt-pack contract must hold.
+        run: npm run test:pr-coverage
-      - run: npm run prompt:audit
+      - name: Verify changed-line coverage policy tool
-      # Non-network smoke: core traps run deterministically; server/live tiers
+        run: npm run test:changed-coverage-script
-      # auto-skip when no server/LLM is present (they are absent in CI).
-      - run: npm run smoke
 
   release:
+    name: Release certification and ZIP archive
     if: startsWith(github.ref, 'refs/tags/v')
-    needs: test
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    permissions:
+      artifact-metadata: write
+      attestations: write
+      contents: read
+      id-token: write
     steps:
       - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
         with:
@@ -61,20 +68,40 @@ jobs:
           node-version: 22
           cache: npm
       - run: npm ci
-      - run: npm run test:release
+      - name: Run release-grade test and evidence gates
-      - run: npm audit --audit-level=high
+        run: npm run test:release
-      - run: npm run pack:dry-run
+      - name: Reject high-severity dependency vulnerabilities
-      - run: npm pack --json > pack-result.json
+        run: npm audit --audit-level=high
-      - name: Record release checksums
+      - name: Build deterministic source ZIP from tested tag
         run: |
-          package_file="$(node -e "const fs=require('fs'); const p=JSON.parse(fs.readFileSync('pack-result.json')); process.stdout.write(p[0].filename)")"
+          mkdir -p release release-evidence
-          sha256sum "$package_file" > SHA256SUMS
+          archive="release/T3MP3ST-${GITHUB_SHA}.zip"
+          git archive --format=zip --prefix="T3MP3ST-${GITHUB_SHA}/" --output="$archive" "$GITHUB_SHA"
+          unzip -t "$archive" | tee release-evidence/source-zip-check.txt
+          sha256sum "$archive" | tee release-evidence/SHA256SUMS
+      - name: Verify a clean user build from the ZIP
+        run: |
+          install_root="$RUNNER_TEMP/t3mp3st-release-smoke"
+          mkdir -p "$install_root"
+          unzip -q "release/T3MP3ST-${GITHUB_SHA}.zip" -d "$install_root"
+          cd "$install_root/T3MP3ST-${GITHUB_SHA}"
+          npm ci
+          npm run build
+          node dist/cli.js --help > /dev/null
+      - name: Attest tested ZIP provenance
+        id: provenance
+        uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2
+        with:
+          subject-checksums: release-evidence/SHA256SUMS
+      - name: Retain provenance bundle and verify archive checksum
+        run: |
+          cp "${{ steps.provenance.outputs.bundle-path }}" release-evidence/slsa-provenance.sigstore.json
+          sha256sum -c release-evidence/SHA256SUMS
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: tested-package-${{ github.sha }}
+          name: tested-source-zip-${{ github.sha }}
           path: |
-            *.tgz
+            release/
-            SHA256SUMS
+            release-evidence/
-            pack-result.json
           if-no-files-found: error
           retention-days: 14

CONTRIBUTING.md

@@ -53,12 +53,10 @@ least 50% coverage; documentation-only and metadata-only changes may mark this
 not applicable with a reason. Trust-boundary, scope, evidence, provider,
 installation, and release changes also need the focused checks for that risk.
 
-Before opening a PR:
+Before opening a PR, run the same correctness gate used by CI:
 
 ```bash
-npm run typecheck
+npm run test:pr
-npm test
-npm run doctor
 ```
 
 For changes that affect claims, run modes, agent/tool execution, target scope,
@@ -84,9 +82,11 @@ unrelated provider/config churn, benchmark fixture removals, provenance doc
 removals, or safety-test removals. If the branch has drifted, recreate it from
 current `main` and reapply only the intended change.
 
-Maintainers run `npm run test:release`, `npm audit --audit-level=high`, and the
+Maintainers run `npm run test:release` and `npm audit --audit-level=high` against
-package dry run against the exact release commit before publishing. A green PR
+the exact release tag before publishing. The tag workflow creates one
-gate is necessary for review and merge, but it is not release certification.
+deterministic source ZIP, verifies it, records SHA-256 checksums, and retains a
+Sigstore provenance bundle. A green PR gate is necessary for review and merge,
+but it is not release certification.
 
 Do not include secrets, private tracker content, unlicensed corpora, or
 uncoordinated vulnerability details. Use the disclosure channel in

docs/PULL_REQUEST_DELIVERY.md

@@ -39,14 +39,17 @@ both human contributors and coding agents.
 
 4. Run the smallest meaningful verification set.
 
-   For most code changes:
+   For most code changes, run the same correctness gate used by CI:
 
    ```bash
-   npm run typecheck
+   npm run test:pr
-   npm test
-   npm run doctor
    ```
 
+   CI additionally measures changed executable lines against the PR base and
+   requires at least 50% coverage. This permits incremental review without
+   claiming release certification. Release tags rerun the complete coverage,
+   evidence, anti-fitting, smoke, build, audit, and package gates.
+
    For docs-only changes, note why code tests were skipped. For UI-only
    `docs/index.html` changes, extract or otherwise parse-check the page scripts.
    For benchmark or headline-claim changes, also run:

docs/RELEASE_CHECKLIST.md

@@ -67,12 +67,14 @@ degrade gracefully — they never fail the core run.
   ```
 - Tag the release only after Sections 1–2 are green.
 - Push the `v*` tag and wait for the tag workflow. It reruns
-  `npm run test:release`, the high-severity dependency audit, and the package
+  `npm run test:release`, the high-severity dependency audit, and package dry
-  dry run against the exact tag, then retains the tested `.tgz`, its manifest,
+  run against the exact tag. It then creates one deterministic
-  and `SHA256SUMS` as workflow artifacts.
+  `T3MP3ST-<sha>.zip` directly from that tested Git object.
-- Inspect `pack-result.json`: only the allowlisted runtime, scripts, tools,
+- The workflow extracts that ZIP into a clean temporary directory, performs a
-  documentation, examples, and package metadata may ship. Workspace notes,
+  locked install and build, and invokes the packaged CLI help path before the
-  tests, AIWG/provider deployment internals, secrets, and local artifacts are
+  archive can be retained.
-  release blockers.
+- Verify `release-evidence/source-zip-check.txt`, `SHA256SUMS`, and the retained
-- Publish the retained, checksum-matched package artifact. Do not rebuild a
+  Sigstore provenance bundle. Workspace notes, secrets, ignored files, and
-  different archive from another checkout after certification.
+  local artifacts cannot enter a `git archive` snapshot.
+- Publish the retained, checksum-matched ZIP. Do not rebuild a different
+  archive from another checkout after certification.

docsite/t3mp3st-docs/content/PULL_REQUEST_DELIVERY.md

@@ -50,14 +50,17 @@ both human contributors and coding agents.
 
 4. Run the smallest meaningful verification set.
 
-   For most code changes:
+   For most code changes, run the same correctness gate used by CI:
 
    ```bash
-   npm run typecheck
+   npm run test:pr
-   npm test
-   npm run doctor
    ```
 
+   CI additionally measures changed executable lines against the PR base and
+   requires at least 50% coverage. This permits incremental review without
+   claiming release certification. Release tags rerun the complete coverage,
+   evidence, anti-fitting, smoke, build, audit, and package gates.
+
    For docs-only changes, note why code tests were skipped. For UI-only
    `docs/index.html` changes, extract or otherwise parse-check the page scripts.
    For benchmark or headline-claim changes, also run:

docsite/t3mp3st-docs/content/RELEASE_CHECKLIST.md

@@ -78,12 +78,14 @@ degrade gracefully — they never fail the core run.
   ```
 - Tag the release only after Sections 1–2 are green.
 - Push the `v*` tag and wait for the tag workflow. It reruns
-  `npm run test:release`, the high-severity dependency audit, and the package
+  `npm run test:release`, the high-severity dependency audit, and package dry
-  dry run against the exact tag, then retains the tested `.tgz`, its manifest,
+  run against the exact tag. It then creates one deterministic
-  and `SHA256SUMS` as workflow artifacts.
+  `T3MP3ST-<sha>.zip` directly from that tested Git object.
-- Inspect `pack-result.json`: only the allowlisted runtime, scripts, tools,
+- The workflow extracts that ZIP into a clean temporary directory, performs a
-  documentation, examples, and package metadata may ship. Workspace notes,
+  locked install and build, and invokes the packaged CLI help path before the
-  tests, AIWG/provider deployment internals, secrets, and local artifacts are
+  archive can be retained.
-  release blockers.
+- Verify `release-evidence/source-zip-check.txt`, `SHA256SUMS`, and the retained
-- Publish the retained, checksum-matched package artifact. Do not rebuild a
+  Sigstore provenance bundle. Workspace notes, secrets, ignored files, and
-  different archive from another checkout after certification.
+  local artifacts cannot enter a `git archive` snapshot.
+- Publish the retained, checksum-matched ZIP. Do not rebuild a different
+  archive from another checkout after certification.

package.json

@@ -75,7 +75,10 @@
     "mcp": "tsx src/mcp-server.ts",
     "mcp:prod": "node dist/mcp-server.js",
     "test": "vitest run src && node scripts/test-ops-preflight.mjs && node scripts/test-model-matrix.mjs && node scripts/refusal-frontier.mjs --self-test",
-    "test:release": "npm run lint && npm run typecheck && npm test && npm run test:coverage && npm run doctor && npm run verify-claims && npm run test:no-fitting && npm run test:no-self-fitting && npm run test:gate && npm run prompt:audit && npm run smoke && npm run build",
+    "test:pr": "npm run lint && npm run typecheck && npm test && npm run doctor",
+    "test:pr-coverage": "vitest run --coverage --config vitest.pr.config.ts && node scripts/check-changed-line-coverage.mjs",
+    "test:changed-coverage-script": "node scripts/check-changed-line-coverage.mjs --self-test",
+    "test:release": "npm run lint && npm run typecheck && npm test && npm run test:coverage && npm run doctor && npm run verify-claims && npm run test:no-fitting && npm run test:no-self-fitting && npm run test:no-phantom-tools && npm run test:gate && npm run prompt:audit && npm run smoke && npm run build && npm run pack:dry-run",
     "pack:dry-run": "npm pack --dry-run",
     "test:coverage": "vitest run --coverage",
     "test:watch": "vitest src",

scripts/check-changed-line-coverage.mjs

@@ -0,0 +1,64 @@
+#!/usr/bin/env node
+import { readFileSync } from 'node:fs';
+import { execFileSync } from 'node:child_process';
+
+export function changedLines(diff) {
+  const files = new Map();
+  let current;
+  for (const line of String(diff).split('\n')) {
+    if (line.startsWith('+++ b/')) {
+      current = line.slice(6);
+      if (!files.has(current)) files.set(current, new Set());
+      continue;
+    }
+    const match = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/);
+    if (!match || !current) continue;
+    const start = Number(match[1]);
+    const count = match[2] === undefined ? 1 : Number(match[2]);
+    for (let n = start; n < start + count; n += 1) files.get(current).add(n);
+  }
+  return files;
+}
+
+export function assessChangedCoverage(coverage, changes) {
+  let coverable = 0;
+  let covered = 0;
+  const uncovered = [];
+  for (const [file, lines] of changes) {
+    if (!file.startsWith('src/') || file.endsWith('.test.ts') || file.includes('/__tests__/')) continue;
+    const entry = Object.values(coverage).find((item) => String(item.path || '').replaceAll('\\', '/').endsWith(`/${file}`));
+    if (!entry) continue;
+    for (const line of lines) {
+      const statementIds = Object.entries(entry.statementMap || {})
+        .filter(([, span]) => line >= span.start.line && line <= span.end.line)
+        .map(([id]) => id);
+      if (!statementIds.length) continue;
+      coverable += 1;
+      if (statementIds.some((id) => Number(entry.s?.[id] || 0) > 0)) covered += 1;
+      else uncovered.push(`${file}:${line}`);
+    }
+  }
+  return { coverable, covered, percent: coverable ? (covered / coverable) * 100 : 100, uncovered };
+}
+
+function selfTest() {
+  const changes = changedLines('+++ b/src/a.ts\n@@ -1 +1,3 @@\n+x\n+y\n+z');
+  const coverage = { '/repo/src/a.ts': { path: '/repo/src/a.ts', statementMap: { 0: { start: { line: 1 }, end: { line: 1 } }, 1: { start: { line: 2 }, end: { line: 3 } } }, s: { 0: 1, 1: 0 } } };
+  const result = assessChangedCoverage(coverage, changes);
+  if (result.coverable !== 3 || result.covered !== 1 || Math.round(result.percent) !== 33) throw new Error('changed-line coverage self-test failed');
+  console.log('changed-line coverage self-test passed');
+}
+
+if (process.argv.includes('--self-test')) {
+  selfTest();
+} else {
+  const base = process.argv[2] || process.env.COVERAGE_BASE;
+  const minimum = Number(process.argv[3] || process.env.PR_CHANGED_LINE_COVERAGE || 50);
+  if (!base) throw new Error('base revision required as argv[2] or COVERAGE_BASE');
+  const diff = execFileSync('git', ['diff', '--unified=0', '--no-ext-diff', `${base}...HEAD`, '--', 'src'], { encoding: 'utf8' });
+  const coverage = JSON.parse(readFileSync('coverage-pr/coverage-final.json', 'utf8'));
+  const result = assessChangedCoverage(coverage, changedLines(diff));
+  console.log(`Changed executable lines: ${result.covered}/${result.coverable} (${result.percent.toFixed(1)}%; required ${minimum}%)`);
+  if (result.uncovered.length) console.log(`Uncovered changed lines:\n${result.uncovered.slice(0, 100).join('\n')}`);
+  if (result.percent < minimum) process.exitCode = 1;
+}

vitest.pr.config.ts

@@ -0,0 +1,16 @@
+import { defineConfig } from 'vitest/config';
+
+// Pull requests use changed-line coverage as a review signal. Release tags use
+// vitest.config.ts, whose narrow 100% per-file contract remains unchanged.
+export default defineConfig({
+  test: {
+    include: ['src/**/*.test.ts'],
+    coverage: {
+      provider: 'v8',
+      include: ['src/**/*.ts'],
+      exclude: ['src/**/*.test.ts', 'src/**/__tests__/**', 'src/types/**'],
+      reporter: ['json'],
+      reportsDirectory: 'coverage-pr',
+    },
+  },
+});