fix(git)!: use internal proxy for all git operations (#39426)

Introduces gitproxy module which spawns a small forward proxy as scanner
for git calls
Replaces hostmatcher with matchlist which supports port rules
Deprecates ALLOWED_DOMAINS/BLOCKED_DOMAINS and ALLOW_LOCALNETWORKS
settings in migration in favor of full names we have in security
configs.
Removes `external` preset in favor of lax/strict modes, strict mode
requiring explicit ports if they aren't standard http/s ones.

Breaking changes:
- `external` preset no longer works as deny rule. To enforce that, use
`strict` mode and allow ranges to connect to
- Wildcards are no longer accepted in IP addresses
- `*` is no longer allowed as entry in lists
- domain rules now use curl like syntax `*.example.com` matching
subdomains but not `example.com`, `example.com` matching itself and all
subdomains. `example.*` is not a valid rule
- In the default `lax` mode, `[security] ALLOWED_HOST_LIST` no longer
restricts public hosts, set `EGRESS_MODE = strict` to keep an exclusive
list. A startup warning flags this
- Invalid list entries are logged at startup, invalid
`BLOCKED_HOST_LIST`/`BLOCKED_DOMAINS` entries stop it

Docs: https://gitea.com/gitea/docs/pulls/557
Signed-off-by: wxiaoguang <wxiaoguang@gmail.com>
Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: bircni <bircni@icloud.com>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

services/mirror/mirror_pull.go

@@ -18,7 +18,6 @@ import (
 	"gitea.dev/modules/lfs"
 	"gitea.dev/modules/log"
 	"gitea.dev/modules/process"
-	"gitea.dev/modules/proxy"
 	repo_module "gitea.dev/modules/repository"
 	"gitea.dev/modules/setting"
 	"gitea.dev/modules/timeutil"
@@ -69,9 +68,7 @@ func UpdateAddress(ctx context.Context, m *repo_model.Mirror, addr string) error
 }
 
 func pruneBrokenReferences(ctx context.Context, m *repo_model.Mirror, repoLogName string, gitRepo git.RepositoryFacade, timeout time.Duration) error {
-	cmd := gitcmd.NewCommand("remote", "prune").AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout)
-	git.HandleGitCmdHTTPRedirection(cmd, m.GetRemoteName())
-	stdout, _, pruneErr := cmd.WithRepo(gitRepo).RunStdString(ctx)
+	stdout, _, pruneErr := gitcmd.NewCommand("remote", "prune").AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout).WithRepo(gitRepo).RunStdString(ctx)
 	if pruneErr != nil {
 		// sanitize the output, since it may contain the remote address, which may contain a password
 		stderrMessage := util.SanitizeCredentialURLs(pruneErr.Stderr())
@@ -122,17 +119,15 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*repo_module.SyncResu
 			return nil, false
 		}
 	}
-	envs := proxy.EnvWithProxy(remoteURL.URL)
 	timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second
 
 	// use fetch but not remote update because git fetch support --tags but remote update doesn't
 	cmdFetch := func() *gitcmd.Command {
 		cmd := gitcmd.NewCommand("fetch", "--tags")
-		git.HandleGitCmdHTTPRedirection(cmd, m.GetRemoteName())
 		if m.EnablePrune {
 			cmd.AddArguments("--prune")
 		}
-		return cmd.AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout).WithEnv(envs)
+		return cmd.AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout)
 	}
 
 	var err error
@@ -209,9 +204,7 @@ func runSync(ctx context.Context, m *repo_model.Mirror) ([]*repo_module.SyncResu
 	}
 
 	cmdRemoteUpdatePrune := func() *gitcmd.Command {
-		cmd := gitcmd.NewCommand("remote", "update", "--prune").AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout).WithEnv(envs)
-		git.HandleGitCmdHTTPRedirection(cmd, m.GetRemoteName())
-		return cmd
+		return gitcmd.NewCommand("remote", "update", "--prune").AddDynamicArguments(m.GetRemoteName()).WithTimeout(timeout)
 	}
 
 	if repo_service.HasWiki(ctx, m.Repo) {

services/mirror/mirror_push.go

@@ -18,7 +18,6 @@ import (
 	"gitea.dev/modules/lfs"
 	"gitea.dev/modules/log"
 	"gitea.dev/modules/process"
-	"gitea.dev/modules/proxy"
 	"gitea.dev/modules/repository"
 	"gitea.dev/modules/setting"
 	"gitea.dev/modules/timeutil"
@@ -156,13 +155,11 @@ func runPushSync(ctx context.Context, m *repo_model.PushMirror) error {
 
 		log.Trace("Pushing mirror %d repo %s to remote %s", m.ID, storageRepo.LogString(), m.RemoteName)
 
-		envs := proxy.EnvWithProxy(remoteURL.URL)
 		if err := git.PushToExternal(ctx, storageRepo, git.PushOptions{
 			Remote:  m.RemoteName,
 			Force:   true,
 			Mirror:  true,
 			Timeout: timeout,
-			Env:     envs,
 		}); err != nil {
 			return fmt.Errorf("PushToExternal failed: %w", err)
 		}