member-console/scripts/mutation-check/main.go

// SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Commercial
// SPDX-FileCopyrightText: 2025-2026 Christian Galo

// Command mutation-check applies deliberate breaks to the tree one at a time
// and reports which of them a test or a command catches.
//
// A list is a JSON file with an optional "prepare" command and a list of
// breaks. A test record names a file, the exact text to replace and its
// replacement, a package, and for each side the -run pattern and the message
// the failing test must print:
//
//	{"name": "...", "file": "...", "old": "...", "new": "...", "pkg": "./internal/db",
//	 "run_old": "^TestOld$", "run_new": "^TestNew$", "want_old": "...", "want_new": "..."}
//
// A command record replaces pkg and the run patterns with a "cmd" that bash
// runs from the repository root, the exit status "exit" it must end with and,
// optionally, "inspect", a file the command writes that is removed before each
// run of that record. It carries the breaks that only a command shows: the
// project lint, golangci-lint, or a screens run, which exits 0 whatever it
// finds. A record with a "side" runs only on that side; a break whose edit
// differs between the two commits is two records with one name.
//
// -side old|new picks run_old and want_old, or run_new and want_new. A list
// without "prepare" is stateless: every break's test or command runs once
// unbroken first, and the list stops unless each passes. A list with "prepare"
// is stateful: before each break, prepare runs and then that break's unbroken
// run, and a break whose control run fails is reported CONTROL-FAILED, not
// classified. After the last break of a stateful list, prepare runs once more
// on the restored tree.
//
// Each break is CAUGHT, MISSED (the run passed), WRONG-MESSAGE (it failed
// without the wanted text), NOT-FOUND (the old text does not occur exactly
// once), SKIPPED (the named test skipped, so it did not run) or
// CONTROL-FAILED. The exit status is 1 unless every break is CAUGHT, and 2 on
// a usage or list error. Run it from the repository root:
//
//	go run ./scripts/mutation-check -list breaks.json -side new
package main

import (
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"
)

const (
	caught        = "CAUGHT"
	missed        = "MISSED"
	wrongMessage  = "WRONG-MESSAGE"
	notFound      = "NOT-FOUND"
	skipped       = "SKIPPED"
	controlFailed = "CONTROL-FAILED"
)

type list struct {
	Prepare string   `json:"prepare"`
	Breaks  []record `json:"breaks"`
}

// record is one break: a test record when Cmd is empty, a command record
// otherwise.
type record struct {
	Name    string `json:"name"`
	Side    string `json:"side"`
	File    string `json:"file"`
	Old     string `json:"old"`
	New     string `json:"new"`
	Pkg     string `json:"pkg"`
	RunOld  string `json:"run_old"`
	RunNew  string `json:"run_new"`
	WantOld string `json:"want_old"`
	WantNew string `json:"want_new"`
	Cmd     string `json:"cmd"`
	Exit    int    `json:"exit"`
	Inspect string `json:"inspect"`
}

func (r record) isCommand() bool { return r.Cmd != "" }

func (r record) run(side string) string {
	if side == "old" {
		return r.RunOld
	}
	return r.RunNew
}

func (r record) want(side string) string {
	if side == "old" {
		return r.WantOld
	}
	return r.WantNew
}

// result is what one test or command run printed and how it ended.
type result struct {
	out  string
	exit int
}

// executor runs the things a list names. The real one starts processes; the
// tests fake it.
type executor interface {
	Prepare(ctx context.Context, cmd string) error
	Test(ctx context.Context, pkg, run string) (result, error)
	Command(ctx context.Context, cmd string) (result, error)
}

func main() {
	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()
	os.Exit(run(ctx, os.Args[1:], os.Stdout, os.Stderr))
}

func run(ctx context.Context, args []string, stdout, stderr io.Writer) int {
	fs := flag.NewFlagSet("mutation-check", flag.ContinueOnError)
	fs.SetOutput(stderr)
	path := fs.String("list", "", "JSON list of breaks")
	side := fs.String("side", "", "old or new")
	if err := fs.Parse(args); err != nil {
		return 2
	}
	if *path == "" || (*side != "old" && *side != "new") {
		sayln(stderr, "mutation-check: -list <file.json> and -side old|new are required")
		return 2
	}
	l, err := readList(*path)
	if err != nil {
		sayln(stderr, "mutation-check:", err)
		return 2
	}
	root, err := os.Getwd()
	if err != nil {
		sayln(stderr, "mutation-check:", err)
		return 2
	}
	r := &runner{ex: shell{dir: root}, root: root, side: *side, out: stdout}
	counts, err := r.runList(ctx, l)
	if err != nil {
		sayln(stderr, "mutation-check:", err)
		return 1
	}
	return summarize(stdout, counts)
}

func readList(path string) (list, error) {
	var l list
	data, err := os.ReadFile(path)
	if err != nil {
		return l, err
	}
	if err := json.Unmarshal(data, &l); err != nil {
		return l, fmt.Errorf("%s: %w", path, err)
	}
	return l, nil
}

// summarize prints the total line and returns the exit status.
func summarize(w io.Writer, counts map[string]int) int {
	total := 0
	for _, n := range counts {
		total += n
	}
	say(w, "%d breaks: %d %s", total, counts[caught], caught)
	for _, k := range []string{missed, wrongMessage, notFound, skipped, controlFailed} {
		if counts[k] > 0 {
			say(w, ", %d %s", counts[k], k)
		}
	}
	sayln(w)
	if counts[caught] != total {
		return 1
	}
	return 0
}

type runner struct {
	ex   executor
	root string
	side string
	out  io.Writer
}

// runList runs the list's breaks for the runner's side and returns how many
// ended in each outcome. An error means the list could not be run at all: a
// stateless control run failed, or prepare did.
func (r *runner) runList(ctx context.Context, l list) (map[string]int, error) {
	breaks := r.forSide(l.Breaks)
	if l.Prepare == "" {
		if err := r.controlAll(ctx, breaks); err != nil {
			return nil, err
		}
	}
	counts := map[string]int{}
	for _, b := range breaks {
		if err := ctx.Err(); err != nil {
			return counts, err
		}
		outcome, detail, err := r.one(ctx, l.Prepare, b)
		if err != nil {
			return counts, err
		}
		counts[outcome]++
		r.report(outcome, b.Name, detail)
	}
	if l.Prepare != "" {
		if err := r.ex.Prepare(ctx, l.Prepare); err != nil {
			return counts, fmt.Errorf("prepare after the last break: %w", err)
		}
	}
	return counts, nil
}

// forSide keeps the records that run on the runner's side.
func (r *runner) forSide(all []record) []record {
	var out []record
	for _, b := range all {
		if b.Side == "" || b.Side == r.side {
			out = append(out, b)
		}
	}
	return out
}

func (r *runner) report(outcome, name, detail string) {
	say(r.out, "%-14s %s\n", outcome, name)
	if detail != "" {
		sayln(r.out, indent(detail))
	}
}

func indent(s string) string {
	return "    " + strings.ReplaceAll(strings.TrimRight(s, "\n"), "\n", "\n    ")
}

// controlAll runs every distinct unbroken test or command once and fails
// unless each passes.
func (r *runner) controlAll(ctx context.Context, breaks []record) error {
	done := map[string]bool{}
	for _, b := range breaks {
		key := r.controlKey(b)
		if done[key] {
			continue
		}
		done[key] = true
		why, err := r.control(ctx, b)
		if err != nil {
			return err
		}
		if why != "" {
			return fmt.Errorf("control run of %q failed: %s", b.Name, why)
		}
	}
	return nil
}

func (r *runner) controlKey(b record) string {
	if b.isCommand() {
		return "cmd\x00" + b.Cmd + "\x00" + b.Inspect + "\x00" + r.side + b.want(r.side)
	}
	return "test\x00" + b.Pkg + "\x00" + b.run(r.side) + "\x00" + b.want(r.side)
}

// control runs the unbroken test or command and returns why it did not pass,
// or "" when it did.
func (r *runner) control(ctx context.Context, b record) (string, error) {
	res, seen, err := r.execute(ctx, b)
	if err != nil {
		return "", err
	}
	want := b.want(r.side)
	switch {
	case !b.isCommand() && didNotRun(res):
		return "the named test did not run or skipped\n" + tail(res.out), nil
	case res.exit != 0:
		return fmt.Sprintf("exit %d\n%s", res.exit, tail(res.out)), nil
	case want != "" && strings.Contains(seen, want):
		return fmt.Sprintf("%q is already in the unbroken output", want), nil
	}
	return "", nil
}

// one applies a break, runs it and classifies the run. For a stateful list it
// first prepares and runs the control.
func (r *runner) one(ctx context.Context, prepare string, b record) (string, string, error) {
	if prepare != "" {
		if err := r.ex.Prepare(ctx, prepare); err != nil {
			return "", "", fmt.Errorf("prepare before %q: %w", b.Name, err)
		}
		why, err := r.control(ctx, b)
		if err != nil {
			return "", "", err
		}
		if why != "" {
			return controlFailed, why, nil
		}
	}
	restore, outcome, err := r.apply(b)
	if err != nil {
		return "", "", err
	}
	if outcome != "" {
		return outcome, "old text not found exactly once in " + b.File, nil
	}
	defer restore()
	if prepare != "" {
		if err := r.ex.Prepare(ctx, prepare); err != nil {
			return "", "", fmt.Errorf("prepare after applying %q: %w", b.Name, err)
		}
	}
	res, seen, err := r.execute(ctx, b)
	if err != nil {
		return "", "", err
	}
	outcome, detail := r.classify(b, res, seen)
	return outcome, detail, nil
}

// apply replaces the record's old text in its file and returns a function
// that writes the original bytes back. The outcome is NOT-FOUND when the old
// text does not occur exactly once, and the file is then untouched.
func (r *runner) apply(b record) (func(), string, error) {
	path := filepath.Join(r.root, b.File)
	orig, err := os.ReadFile(path)
	if err != nil {
		return nil, "", err
	}
	if strings.Count(string(orig), b.Old) != 1 {
		return nil, notFound, nil
	}
	info, err := os.Stat(path)
	if err != nil {
		return nil, "", err
	}
	broken := strings.Replace(string(orig), b.Old, b.New, 1)
	if err := os.WriteFile(path, []byte(broken), info.Mode().Perm()); err != nil {
		return nil, "", err
	}
	restore := func() {
		if err := os.WriteFile(path, orig, info.Mode().Perm()); err != nil {
			say(os.Stderr, "mutation-check: restoring %s: %v\n", path, err)
		}
	}
	return restore, "", nil
}

// execute runs the record's test or command. seen is the text the want is
// looked for in: the output, plus the inspected file for a command record.
func (r *runner) execute(ctx context.Context, b record) (result, string, error) {
	if !b.isCommand() {
		res, err := r.ex.Test(ctx, b.Pkg, b.run(r.side))
		return res, res.out, err
	}
	inspect := ""
	if b.Inspect != "" {
		inspect = filepath.Join(r.root, b.Inspect)
		if err := os.Remove(inspect); err != nil && !errors.Is(err, os.ErrNotExist) {
			return result{}, "", err
		}
	}
	res, err := r.ex.Command(ctx, b.Cmd)
	if err != nil {
		return res, "", err
	}
	seen := res.out
	if inspect != "" {
		if data, err := os.ReadFile(inspect); err == nil {
			seen += "\n" + string(data)
		}
	}
	return res, seen, nil
}

// classify decides what a broken run means.
func (r *runner) classify(b record, res result, seen string) (string, string) {
	want := b.want(r.side)
	if !b.isCommand() {
		return classifyTest(res, want)
	}
	found := want != "" && strings.Contains(seen, want)
	switch {
	case res.exit == b.Exit && found:
		return caught, ""
	case res.exit != b.Exit && res.exit == 0:
		return missed, fmt.Sprintf("exited 0, want %d", b.Exit)
	case res.exit != b.Exit:
		return wrongMessage, fmt.Sprintf("exit %d, want %d\n%s", res.exit, b.Exit, tail(seen))
	case b.Exit == 0:
		return missed, fmt.Sprintf("%q is in neither the output nor the inspected file", want)
	}
	return wrongMessage, fmt.Sprintf("%q is in neither the output nor the inspected file\n%s", want, tail(seen))
}

func classifyTest(res result, want string) (string, string) {
	switch {
	case didNotRun(res):
		return skipped, "the named test did not run or skipped"
	case res.exit == 0:
		return missed, "the test passed"
	case want != "" && strings.Contains(res.out, want):
		return caught, ""
	}
	return wrongMessage, fmt.Sprintf("%q is not in the output\n%s", want, tail(res.out))
}

// didNotRun reports a run whose named test never passed or failed: go test
// printed no test result, or only skips.
func didNotRun(res result) bool {
	if strings.Contains(res.out, "[no tests to run]") {
		return true
	}
	if strings.Contains(res.out, "--- PASS") || strings.Contains(res.out, "--- FAIL") {
		return false
	}
	return strings.Contains(res.out, "--- SKIP") || res.exit == 0
}

// tail keeps the last lines of an output for a report.
func tail(s string) string {
	lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
	if len(lines) > 12 {
		lines = lines[len(lines)-12:]
	}
	return strings.Join(lines, "\n")
}

// shell is the real executor: it starts go test and bash from the repository
// root.
type shell struct{ dir string }

func (s shell) Prepare(ctx context.Context, cmd string) error {
	res, err := s.bash(ctx, cmd)
	if err != nil {
		return err
	}
	if res.exit != 0 {
		return fmt.Errorf("exit %d\n%s", res.exit, tail(res.out))
	}
	return nil
}

func (s shell) Test(ctx context.Context, pkg, run string) (result, error) {
	return s.start(exec.CommandContext(ctx, "go", "test", "-count=1", "-v", "-run", run, pkg))
}

func (s shell) Command(ctx context.Context, cmd string) (result, error) {
	return s.bash(ctx, cmd)
}

func (s shell) bash(ctx context.Context, cmd string) (result, error) {
	return s.start(exec.CommandContext(ctx, "bash", "-c", cmd))
}

func (s shell) start(c *exec.Cmd) (result, error) {
	c.Dir = s.dir
	out, err := c.CombinedOutput()
	var exit *exec.ExitError
	switch {
	case err == nil:
		return result{out: string(out)}, nil
	case errors.As(err, &exit) && c.ProcessState != nil && c.ProcessState.Exited():
		return result{out: string(out), exit: exit.ExitCode()}, nil
	}
	return result{out: string(out)}, err
}

// say and sayln write a report line; a failed write to a terminal or a test
// buffer has nowhere to be reported.
func say(w io.Writer, format string, args ...any) { _, _ = fmt.Fprintf(w, format, args...) }

func sayln(w io.Writer, args ...any) { _, _ = fmt.Fprintln(w, args...) }