test spring-cleaning

This makes a quick pass through our tests;

Discard output/err
----------------------------------------------

Many tests were testing for error-conditions, but didn't discard output.
This produced a lot of noise when running the tests, and made it hard
to discover if there were actual failures, or if the output was expected.
For example:

    === RUN   TestConfigCreateErrors
    Error: "create" requires exactly 2 arguments.
    See 'create --help'.

    Usage:  create [OPTIONS] CONFIG file|- [flags]

    Create a config from a file or STDIN
    Error: "create" requires exactly 2 arguments.
    See 'create --help'.

    Usage:  create [OPTIONS] CONFIG file|- [flags]

    Create a config from a file or STDIN
    Error: error creating config
    --- PASS: TestConfigCreateErrors (0.00s)

And after discarding output:

    === RUN   TestConfigCreateErrors
    --- PASS: TestConfigCreateErrors (0.00s)

Use sub-tests where possible
----------------------------------------------

Some tests were already set-up to use test-tables, and even had a usable
name (or in some cases "error" to check for). Change them to actual sub-
tests. Same test as above, but now with sub-tests and output discarded:

    === RUN   TestConfigCreateErrors
    === RUN   TestConfigCreateErrors/requires_exactly_2_arguments
    === RUN   TestConfigCreateErrors/requires_exactly_2_arguments#01
    === RUN   TestConfigCreateErrors/error_creating_config
    --- PASS: TestConfigCreateErrors (0.00s)
        --- PASS: TestConfigCreateErrors/requires_exactly_2_arguments (0.00s)
        --- PASS: TestConfigCreateErrors/requires_exactly_2_arguments#01 (0.00s)
        --- PASS: TestConfigCreateErrors/error_creating_config (0.00s)
    PASS

It's not perfect in all cases (in the above, there's duplicate "expected"
errors, but Go conveniently adds "#01" for the duplicate). There's probably
also various tests I missed that could still use the same changes applied;
we can improve these in follow-ups.

Set cmd.Args to prevent test-failures
----------------------------------------------

When running tests from my IDE, it compiles the tests before running,
then executes the compiled binary to run the tests. Cobra doesn't like
that, because in that situation `os.Args` is taken as argument for the
command that's executed. The command that's tested now sees the test-
flags as arguments (`-test.v -test.run ..`), which causes various tests
to fail ("Command XYZ does not accept arguments").

    # compile the tests:
    go test -c -o foo.test

    # execute the test:
    ./foo.test -test.v -test.run TestFoo
    === RUN   TestFoo
    Error: "foo" accepts no arguments.

The Cobra maintainers ran into the same situation, and for their own
use have added a special case to ignore `os.Args` in these cases;
https://github.com/spf13/cobra/blob/v1.8.1/command.go#L1078-L1083

    args := c.args

    // Workaround FAIL with "go test -v" or "cobra.test -test.v", see #155
    if c.args == nil && filepath.Base(os.Args[0]) != "cobra.test" {
        args = os.Args[1:]
    }

Unfortunately, that exception is too specific (only checks for `cobra.test`),
so doesn't automatically fix the issue for other test-binaries. They did
provide a `cmd.SetArgs()` utility for this purpose
https://github.com/spf13/cobra/blob/v1.8.1/command.go#L276-L280

    // SetArgs sets arguments for the command. It is set to os.Args[1:] by default, if desired, can be overridden
    // particularly useful when testing.
    func (c *Command) SetArgs(a []string) {
        c.args = a
    }

And the fix is to explicitly set the command's args to an empty slice to
prevent Cobra from falling back to using `os.Args[1:]` as arguments.

    cmd := newSomeThingCommand()
    cmd.SetArgs([]string{})

Some tests already take this issue into account, and I updated some tests
for this, but there's likely many other ones that can use the same treatment.

Perhaps the Cobra maintainers would accept a contribution to make their
condition less specific and to look for binaries ending with a `.test`
suffix (which is what compiled binaries usually are named as).

Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

cli/command/plugin/create_test.go

@@ -41,6 +41,7 @@ func TestCreateErrors(t *testing.T) {
 		cmd := newCreateCommand(cli)
 		cmd.SetArgs(tc.args)
 		cmd.SetOut(io.Discard)
+		cmd.SetErr(io.Discard)
 		assert.ErrorContains(t, cmd.Execute(), tc.expectedError)
 	}
 }
@@ -53,6 +54,7 @@ func TestCreateErrorOnFileAsContextDir(t *testing.T) {
 	cmd := newCreateCommand(cli)
 	cmd.SetArgs([]string{"plugin-foo", tmpFile.Path()})
 	cmd.SetOut(io.Discard)
+	cmd.SetErr(io.Discard)
 	assert.ErrorContains(t, cmd.Execute(), "context must be a directory")
 }
 
@@ -64,6 +66,7 @@ func TestCreateErrorOnContextDirWithoutConfig(t *testing.T) {
 	cmd := newCreateCommand(cli)
 	cmd.SetArgs([]string{"plugin-foo", tmpDir.Path()})
 	cmd.SetOut(io.Discard)
+	cmd.SetErr(io.Discard)
 
 	expectedErr := "config.json: no such file or directory"
 	if runtime.GOOS == "windows" {
@@ -82,6 +85,7 @@ func TestCreateErrorOnInvalidConfig(t *testing.T) {
 	cmd := newCreateCommand(cli)
 	cmd.SetArgs([]string{"plugin-foo", tmpDir.Path()})
 	cmd.SetOut(io.Discard)
+	cmd.SetErr(io.Discard)
 	assert.ErrorContains(t, cmd.Execute(), "invalid")
 }
 
@@ -98,6 +102,7 @@ func TestCreateErrorFromDaemon(t *testing.T) {
 	}))
 	cmd.SetArgs([]string{"plugin-foo", tmpDir.Path()})
 	cmd.SetOut(io.Discard)
+	cmd.SetErr(io.Discard)
 	assert.ErrorContains(t, cmd.Execute(), "error creating plugin")
 }
 

cli/command/plugin/disable_test.go

@@ -41,6 +41,7 @@ func TestPluginDisableErrors(t *testing.T) {
 			}))
 		cmd.SetArgs(tc.args)
 		cmd.SetOut(io.Discard)
+		cmd.SetErr(io.Discard)
 		assert.ErrorContains(t, cmd.Execute(), tc.expectedError)
 	}
 }

cli/command/plugin/enable_test.go

@@ -51,6 +51,7 @@ func TestPluginEnableErrors(t *testing.T) {
 			cmd.Flags().Set(key, value)
 		}
 		cmd.SetOut(io.Discard)
+		cmd.SetErr(io.Discard)
 		assert.ErrorContains(t, cmd.Execute(), tc.expectedError)
 	}
 }

cli/command/plugin/inspect_test.go

@@ -66,6 +66,7 @@ func TestInspectErrors(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
+		tc := tc
 		t.Run(tc.description, func(t *testing.T) {
 			cli := test.NewFakeCli(&fakeClient{pluginInspectFunc: tc.inspectFunc})
 			cmd := newInspectCommand(cli)
@@ -74,6 +75,7 @@ func TestInspectErrors(t *testing.T) {
 				cmd.Flags().Set(key, value)
 			}
 			cmd.SetOut(io.Discard)
+			cmd.SetErr(io.Discard)
 			assert.ErrorContains(t, cmd.Execute(), tc.expectedError)
 		})
 	}
@@ -136,6 +138,7 @@ func TestInspect(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
+		tc := tc
 		t.Run(tc.description, func(t *testing.T) {
 			cli := test.NewFakeCli(&fakeClient{pluginInspectFunc: tc.inspectFunc})
 			cmd := newInspectCommand(cli)

cli/command/plugin/install_test.go

@@ -54,11 +54,15 @@ func TestInstallErrors(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		cli := test.NewFakeCli(&fakeClient{pluginInstallFunc: tc.installFunc})
-		cmd := newInstallCommand(cli)
-		cmd.SetArgs(tc.args)
-		cmd.SetOut(io.Discard)
-		assert.ErrorContains(t, cmd.Execute(), tc.expectedError)
+		tc := tc
+		t.Run(tc.description, func(t *testing.T) {
+			cli := test.NewFakeCli(&fakeClient{pluginInstallFunc: tc.installFunc})
+			cmd := newInstallCommand(cli)
+			cmd.SetArgs(tc.args)
+			cmd.SetOut(io.Discard)
+			cmd.SetErr(io.Discard)
+			assert.ErrorContains(t, cmd.Execute(), tc.expectedError)
+		})
 	}
 }
 
@@ -90,16 +94,20 @@ func TestInstallContentTrustErrors(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		cli := test.NewFakeCli(&fakeClient{
-			pluginInstallFunc: func(name string, options types.PluginInstallOptions) (io.ReadCloser, error) {
-				return nil, errors.New("should not try to install plugin")
-			},
-		}, test.EnableContentTrust)
-		cli.SetNotaryClient(tc.notaryFunc)
-		cmd := newInstallCommand(cli)
-		cmd.SetArgs(tc.args)
-		cmd.SetOut(io.Discard)
-		assert.ErrorContains(t, cmd.Execute(), tc.expectedError)
+		tc := tc
+		t.Run(tc.description, func(t *testing.T) {
+			cli := test.NewFakeCli(&fakeClient{
+				pluginInstallFunc: func(name string, options types.PluginInstallOptions) (io.ReadCloser, error) {
+					return nil, errors.New("should not try to install plugin")
+				},
+			}, test.EnableContentTrust)
+			cli.SetNotaryClient(tc.notaryFunc)
+			cmd := newInstallCommand(cli)
+			cmd.SetArgs(tc.args)
+			cmd.SetOut(io.Discard)
+			cmd.SetErr(io.Discard)
+			assert.ErrorContains(t, cmd.Execute(), tc.expectedError)
+		})
 	}
 }
 
@@ -130,10 +138,13 @@ func TestInstall(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		cli := test.NewFakeCli(&fakeClient{pluginInstallFunc: tc.installFunc})
-		cmd := newInstallCommand(cli)
-		cmd.SetArgs(tc.args)
-		assert.NilError(t, cmd.Execute())
-		assert.Check(t, strings.Contains(cli.OutBuffer().String(), tc.expectedOutput))
+		tc := tc
+		t.Run(tc.description, func(t *testing.T) {
+			cli := test.NewFakeCli(&fakeClient{pluginInstallFunc: tc.installFunc})
+			cmd := newInstallCommand(cli)
+			cmd.SetArgs(tc.args)
+			assert.NilError(t, cmd.Execute())
+			assert.Check(t, strings.Contains(cli.OutBuffer().String(), tc.expectedOutput))
+		})
 	}
 }

cli/command/plugin/list_test.go

@@ -46,14 +46,18 @@ func TestListErrors(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		cli := test.NewFakeCli(&fakeClient{pluginListFunc: tc.listFunc})
-		cmd := newListCommand(cli)
-		cmd.SetArgs(tc.args)
-		for key, value := range tc.flags {
-			cmd.Flags().Set(key, value)
-		}
-		cmd.SetOut(io.Discard)
-		assert.ErrorContains(t, cmd.Execute(), tc.expectedError)
+		tc := tc
+		t.Run(tc.description, func(t *testing.T) {
+			cli := test.NewFakeCli(&fakeClient{pluginListFunc: tc.listFunc})
+			cmd := newListCommand(cli)
+			cmd.SetArgs(tc.args)
+			for key, value := range tc.flags {
+				cmd.Flags().Set(key, value)
+			}
+			cmd.SetOut(io.Discard)
+			cmd.SetErr(io.Discard)
+			assert.ErrorContains(t, cmd.Execute(), tc.expectedError)
+		})
 	}
 }
 
@@ -162,13 +166,16 @@ func TestList(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		cli := test.NewFakeCli(&fakeClient{pluginListFunc: tc.listFunc})
-		cmd := newListCommand(cli)
-		cmd.SetArgs(tc.args)
-		for key, value := range tc.flags {
-			cmd.Flags().Set(key, value)
-		}
-		assert.NilError(t, cmd.Execute())
-		golden.Assert(t, cli.OutBuffer().String(), tc.golden)
+		tc := tc
+		t.Run(tc.description, func(t *testing.T) {
+			cli := test.NewFakeCli(&fakeClient{pluginListFunc: tc.listFunc})
+			cmd := newListCommand(cli)
+			cmd.SetArgs(tc.args)
+			for key, value := range tc.flags {
+				cmd.Flags().Set(key, value)
+			}
+			assert.NilError(t, cmd.Execute())
+			golden.Assert(t, cli.OutBuffer().String(), tc.golden)
+		})
 	}
 }

cli/command/plugin/remove_test.go

@@ -37,6 +37,7 @@ func TestRemoveErrors(t *testing.T) {
 		cmd := newRemoveCommand(cli)
 		cmd.SetArgs(tc.args)
 		cmd.SetOut(io.Discard)
+		cmd.SetErr(io.Discard)
 		assert.ErrorContains(t, cmd.Execute(), tc.expectedError)
 	}
 }

cli/command/plugin/upgrade_test.go

@@ -32,6 +32,8 @@ func TestUpgradePromptTermination(t *testing.T) {
 	// need to set a remote address that does not match the plugin
 	// reference sent by the `pluginInspectFunc`
 	cmd.SetArgs([]string{"foo/bar", "localhost:5000/foo/bar:v1.0.0"})
+	cmd.SetOut(io.Discard)
+	cmd.SetErr(io.Discard)
 	test.TerminatePrompt(ctx, t, cmd, cli)
 	golden.Assert(t, cli.OutBuffer().String(), "plugin-upgrade-terminate.golden")
 }