feat(ansible): support repository credentials for ansible-galaxy roles

db_lib/AnsibleApp.go

@@ -81,6 +81,22 @@ func (t *AnsibleApp) InstallRequirements(args LocalAppInstallingArgs) error {
 		return nil
 	}
 
+	environmentVars := galaxyGitEnv(t.Repository)
+
+	// An SSH repository key reaches galaxy's git clones through an agent, the
+	// same way TerraformApp.init hands one to `terraform init`.
+	if args.Installer != nil {
+		keyInstallation, err := args.Installer.Install(t.Repository.SSHKey, db.AccessKeyRoleGit, t.Logger)
+		if err != nil {
+			return err
+		}
+		defer keyInstallation.Destroy() //nolint: errcheck
+		environmentVars = append(environmentVars, keyInstallation.GetGitEnv()...)
+	}
+
+	// Task variables come last so a manually configured GIT_* var still wins.
+	environmentVars = append(environmentVars, args.EnvironmentVars...)
+
 	collectionArgs, err := galaxyExtraArgs(args, GalaxyCollection)
 	if err != nil {
 		return err
@@ -90,11 +106,11 @@ func (t *AnsibleApp) InstallRequirements(args LocalAppInstallingArgs) error {
 		return err
 	}
 
-	err = t.installCollectionsRequirements(args.EnvironmentVars, collectionArgs)
+	err = t.installCollectionsRequirements(environmentVars, collectionArgs)
 	if err != nil {
 		return err
 	}
-	return t.installRolesRequirements(args.EnvironmentVars, roleArgs)
+	return t.installRolesRequirements(environmentVars, roleArgs)
 }
 
 // skipGalaxyInstall reports whether the Galaxy install step must be skipped.

db_lib/GalaxyGitEnv.go

@@ -0,0 +1,53 @@
+package db_lib
+
+import (
+	"net/url"
+	"strings"
+
+	"github.com/semaphoreui/semaphore/db"
+)
+
+// sqQuote quotes s for GIT_CONFIG_PARAMETERS: the value is wrapped in single
+// quotes, and any single quote inside it is escaped the way sh requires.
+func sqQuote(s string) string {
+	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
+}
+
+// galaxyGitEnv gives ansible-galaxy the credential already configured on the
+// repository, so roles and collections hosted on that same server install
+// without a .netrc or ssh config workaround (SEM-198, GitHub #3677, #3708).
+//
+// For `scm: git` requirements ansible-galaxy shells out to `git clone`, and
+// those clones inherit nothing from Semaphore. Credentials travel in
+// GIT_CONFIG_PARAMETERS rather than on the command line so they stay out of
+// `ps` output; git reports the pre-rewrite URL, so they stay out of the task
+// log too.
+func galaxyGitEnv(repo db.Repository) (env []string) {
+	// Without this git prompts on /dev/tty and the task hangs instead of failing.
+	env = append(env, "GIT_TERMINAL_PROMPT=0")
+
+	if repo.GetType() != db.RepositoryHTTP || repo.SSHKey.Type != db.AccessKeyLoginPassword {
+		return
+	}
+
+	plain, err := url.Parse(repo.GitURL)
+	if err != nil || plain.Host == "" {
+		return
+	}
+
+	// Scoped to this exact scheme://host[:port] so no other server named in
+	// requirements.yml is ever offered the credential.
+	plain.Path, plain.RawQuery, plain.Fragment, plain.User = "/", "", "", nil
+
+	withAuth := *plain
+	// An empty login means the password is the whole credential, matching
+	// Repository.GetGitURL.
+	if login := repo.SSHKey.LoginPassword.Login; login == "" {
+		withAuth.User = url.User(repo.SSHKey.LoginPassword.Password)
+	} else {
+		withAuth.User = url.UserPassword(login, repo.SSHKey.LoginPassword.Password)
+	}
+
+	return append(env, "GIT_CONFIG_PARAMETERS="+sqQuote(
+		"url."+withAuth.String()+".insteadOf="+plain.String()))
+}

db_lib/GalaxyGitEnv_test.go

@@ -0,0 +1,177 @@
+package db_lib
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/semaphoreui/semaphore/db"
+	"github.com/semaphoreui/semaphore/pkg/ssh"
+	"github.com/semaphoreui/semaphore/pkg/task_logger"
+	"github.com/semaphoreui/semaphore/util"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func httpRepo(gitURL, login, password string) db.Repository {
+	return db.Repository{
+		GitURL: gitURL,
+		SSHKey: db.AccessKey{
+			Type:          db.AccessKeyLoginPassword,
+			LoginPassword: db.LoginPassword{Login: login, Password: password},
+		},
+	}
+}
+
+func TestSqQuote(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{"plain", "abc", "'abc'"},
+		{"empty", "", "''"},
+		{"embedded quote", "a'b", `'a'\''b'`},
+		{"only quote", "'", `''\'''`},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, tt.expected, sqQuote(tt.input))
+		})
+	}
+}
+
+func TestGalaxyGitEnv_AlwaysDisablesTerminalPrompt(t *testing.T) {
+	for _, repo := range []db.Repository{
+		{GitURL: "git@github.com:acme/roles.git"},
+		{GitURL: "https://git.private.repo/acme/roles.git"},
+		httpRepo("https://git.private.repo/acme/roles.git", "u", "p"),
+	} {
+		assert.Contains(t, galaxyGitEnv(repo), "GIT_TERMINAL_PROMPT=0")
+	}
+}
+
+func TestGalaxyGitEnv_HTTPSWithLoginPassword(t *testing.T) {
+	env := galaxyGitEnv(httpRepo("https://git.private.repo/acme/roles.git", "semuser", "sempass"))
+
+	require.Len(t, env, 2)
+	assert.Equal(t,
+		`GIT_CONFIG_PARAMETERS='url.https://semuser:sempass@git.private.repo/.insteadOf=https://git.private.repo/'`,
+		env[1])
+}
+
+// The rewrite must match host and port, or git will not apply it.
+func TestGalaxyGitEnv_KeepsPort(t *testing.T) {
+	env := galaxyGitEnv(httpRepo("http://127.0.0.1:3300/semuser/main-repo.git", "semuser", "sempass"))
+
+	require.Len(t, env, 2)
+	assert.Contains(t, env[1], "url.http://semuser:sempass@127.0.0.1:3300/.insteadOf=http://127.0.0.1:3300/")
+}
+
+// Matches GetGitURL: an empty login means the password is the whole credential.
+func TestGalaxyGitEnv_TokenOnlyKeyUsesPasswordAsUser(t *testing.T) {
+	env := galaxyGitEnv(httpRepo("https://git.private.repo/acme/roles.git", "", "gho_token"))
+
+	require.Len(t, env, 2)
+	assert.Contains(t, env[1], "url.https://gho_token@git.private.repo/.insteadOf=")
+}
+
+// A password containing '@' or '/' would otherwise corrupt the URL.
+func TestGalaxyGitEnv_EncodesCredentials(t *testing.T) {
+	env := galaxyGitEnv(httpRepo("https://git.private.repo/acme/roles.git", "user@corp", "p@ss/w:rd"))
+
+	require.Len(t, env, 2)
+	assert.Contains(t, env[1], "user%40corp:p%40ss%2Fw%3Ard@git.private.repo")
+	assert.NotContains(t, env[1], "p@ss/w:rd")
+}
+
+func TestGalaxyGitEnv_NoCredentialsForOtherRepoTypes(t *testing.T) {
+	tests := []struct {
+		name string
+		repo db.Repository
+	}{
+		{"ssh url", db.Repository{
+			GitURL: "git@github.com:acme/roles.git",
+			SSHKey: db.AccessKey{Type: db.AccessKeyLoginPassword,
+				LoginPassword: db.LoginPassword{Login: "u", Password: "p"}},
+		}},
+		{"https url but ssh key", db.Repository{
+			GitURL: "https://git.private.repo/acme/roles.git",
+			SSHKey: db.AccessKey{Type: db.AccessKeySSH},
+		}},
+		{"https url but no key", db.Repository{
+			GitURL: "https://git.private.repo/acme/roles.git",
+			SSHKey: db.AccessKey{Type: db.AccessKeyNone},
+		}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equal(t, []string{"GIT_TERMINAL_PROMPT=0"}, galaxyGitEnv(tt.repo))
+		})
+	}
+}
+
+// requirements.yml may name several servers; only the repository's own host
+// may ever be offered its credential.
+func TestGalaxyGitEnv_ScopesCredentialToOneHost(t *testing.T) {
+	env := galaxyGitEnv(httpRepo("https://git.private.repo/acme/roles.git", "semuser", "sempass"))
+
+	require.Len(t, env, 2)
+	assert.Contains(t, env[1], ".insteadOf=https://git.private.repo/")
+	assert.NotContains(t, env[1], "acme/roles.git")
+}
+
+type fakeInstaller struct {
+	key   db.AccessKey
+	usage db.AccessKeyRole
+	err   error
+}
+
+func (f *fakeInstaller) Install(key db.AccessKey, usage db.AccessKeyRole, _ task_logger.Logger) (ssh.AccessKeyInstallation, error) {
+	f.key, f.usage = key, usage
+	return ssh.AccessKeyInstallation{}, f.err
+}
+
+// setupGalaxyConfig gives the package-level util.Config a temp dir to resolve
+// repository paths against, and restores it afterwards.
+func setupGalaxyConfig(t *testing.T) {
+	original := util.Config
+	t.Cleanup(func() { util.Config = original })
+	util.Config = &util.ConfigType{TmpPath: t.TempDir(), Process: &util.ConfigProcess{}}
+}
+
+// The repository's own key must be the one galaxy gets, under the git role.
+func TestInstallRequirements_InstallsRepositoryKey(t *testing.T) {
+	setupGalaxyConfig(t)
+
+	inst := &fakeInstaller{}
+	app := &AnsibleApp{
+		Logger:     task_logger.NopLogger{},
+		Repository: db.Repository{SSHKey: db.AccessKey{ID: 42, Type: db.AccessKeySSH}},
+	}
+
+	_ = app.InstallRequirements(LocalAppInstallingArgs{Installer: inst})
+
+	assert.Equal(t, 42, inst.key.ID)
+	assert.Equal(t, db.AccessKeyRole(db.AccessKeyRoleGit), inst.usage)
+}
+
+func TestInstallRequirements_FailsWhenKeyInstallFails(t *testing.T) {
+	setupGalaxyConfig(t)
+
+	app := &AnsibleApp{Logger: task_logger.NopLogger{}}
+
+	err := app.InstallRequirements(LocalAppInstallingArgs{
+		Installer: &fakeInstaller{err: errors.New("agent unavailable")},
+	})
+
+	assert.ErrorContains(t, err, "agent unavailable")
+}
+
+// A nil installer is the remote-runner path; it must not panic.
+func TestInstallRequirements_NilInstaller(t *testing.T) {
+	setupGalaxyConfig(t)
+
+	app := &AnsibleApp{Logger: task_logger.NopLogger{}}
+
+	assert.NoError(t, app.InstallRequirements(LocalAppInstallingArgs{}))
+}

pkg/ssh/agent.go

@@ -174,7 +174,9 @@ func gitHostKeyCheckingOpts() string {
 	case util.SshStrictHostKeyCheckingYes:
 		return fmt.Sprintf("-o StrictHostKeyChecking=yes -o UserKnownHostsFile=%s", util.Config.Ssh.KnownHostsFile)
 	case util.SshStrictHostKeyCheckingNo:
-		return "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
+		// No leading "ssh": the caller prepends it, and a second one is taken by
+		// ssh as the host to connect to ("Could not resolve hostname ssh").
+		return "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
 	case util.SshStrictHostKeyCheckingAcceptNew:
 		return fmt.Sprintf("-o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=%s", util.Config.Ssh.KnownHostsFile)
 	default:

pkg/ssh/git_env_test.go

@@ -0,0 +1,98 @@
+package ssh
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/semaphoreui/semaphore/util"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func withSshConfig(t *testing.T, mode util.SshStrictHostKeyChecking, knownHosts, configPath string) {
+	t.Helper()
+
+	original := util.Config
+	t.Cleanup(func() { util.Config = original })
+
+	util.Config = &util.ConfigType{
+		Ssh: &util.SshConfig{
+			StrictHostKeyChecking: mode,
+			KnownHostsFile:        knownHosts,
+		},
+		SshConfigPath: configPath,
+	}
+}
+
+func gitSSHCommand(t *testing.T, env []string) string {
+	t.Helper()
+
+	for _, e := range env {
+		if strings.HasPrefix(e, "GIT_SSH_COMMAND=") {
+			return strings.TrimPrefix(e, "GIT_SSH_COMMAND=")
+		}
+	}
+	return ""
+}
+
+// GIT_SSH_COMMAND must name the ssh executable exactly once. A second "ssh" is
+// read by ssh as the host to connect to, which broke every SSH repository on
+// the default configuration.
+func TestGetGitEnv_SshCommandHasSingleExecutable(t *testing.T) {
+	modes := []util.SshStrictHostKeyChecking{
+		util.SshStrictHostKeyCheckingNo,
+		util.SshStrictHostKeyCheckingYes,
+		util.SshStrictHostKeyCheckingAcceptNew,
+	}
+
+	for _, mode := range modes {
+		t.Run(string(mode), func(t *testing.T) {
+			withSshConfig(t, mode, "/tmp/known_hosts", "")
+
+			installation := AccessKeyInstallation{SSHAgent: &Agent{SocketFile: "/tmp/agent.sock"}}
+			cmd := gitSSHCommand(t, installation.GetGitEnv())
+
+			require.NotEmpty(t, cmd, "GIT_SSH_COMMAND must be set when an agent is running")
+			assert.Equal(t, "ssh", strings.Fields(cmd)[0], "must start with the ssh executable")
+			assert.NotContains(t, cmd, "ssh ssh", "the executable must not be repeated")
+
+			// Every remaining field is a flag or a flag's value, never a bare word
+			// that ssh would treat as a hostname.
+			assert.True(t, strings.HasPrefix(strings.Fields(cmd)[1], "-"),
+				"expected an option after the executable, got %q", cmd)
+		})
+	}
+}
+
+func TestGetGitEnv_IncludesAgentSocketAndOptions(t *testing.T) {
+	withSshConfig(t, util.SshStrictHostKeyCheckingNo, "", "")
+
+	installation := AccessKeyInstallation{SSHAgent: &Agent{SocketFile: "/tmp/agent.sock"}}
+	env := installation.GetGitEnv()
+
+	assert.Contains(t, env, "GIT_TERMINAL_PROMPT=0")
+	assert.Contains(t, env, "SSH_AUTH_SOCK=/tmp/agent.sock")
+	assert.Equal(t,
+		"ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null",
+		gitSSHCommand(t, env))
+}
+
+func TestGetGitEnv_AppendsSshConfigPath(t *testing.T) {
+	withSshConfig(t, util.SshStrictHostKeyCheckingNo, "", "/etc/semaphore/ssh_config")
+
+	installation := AccessKeyInstallation{SSHAgent: &Agent{SocketFile: "/tmp/agent.sock"}}
+
+	assert.Equal(t,
+		"ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -F /etc/semaphore/ssh_config",
+		gitSSHCommand(t, installation.GetGitEnv()))
+}
+
+// Without an agent there is no key to offer, so no GIT_SSH_COMMAND is set.
+func TestGetGitEnv_NoAgent(t *testing.T) {
+	withSshConfig(t, util.SshStrictHostKeyCheckingNo, "", "")
+
+	installation := AccessKeyInstallation{}
+	env := installation.GetGitEnv()
+
+	assert.Equal(t, []string{"GIT_TERMINAL_PROMPT=0"}, env)
+}