Merge remote-tracking branch 'upstream/main' into limit-repo-size

routers/api/packages/alpine/alpine.go

@@ -0,0 +1,253 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package alpine
+
+import (
+	"crypto/x509"
+	"encoding/hex"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	packages_model "code.gitea.io/gitea/models/packages"
+	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/json"
+	packages_module "code.gitea.io/gitea/modules/packages"
+	alpine_module "code.gitea.io/gitea/modules/packages/alpine"
+	"code.gitea.io/gitea/modules/util"
+	"code.gitea.io/gitea/routers/api/packages/helper"
+	packages_service "code.gitea.io/gitea/services/packages"
+	alpine_service "code.gitea.io/gitea/services/packages/alpine"
+)
+
+func apiError(ctx *context.Context, status int, obj interface{}) {
+	helper.LogAndProcessError(ctx, status, obj, func(message string) {
+		ctx.PlainText(status, message)
+	})
+}
+
+func GetRepositoryKey(ctx *context.Context) {
+	_, pub, err := alpine_service.GetOrCreateKeyPair(ctx.Package.Owner.ID)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	pubPem, _ := pem.Decode([]byte(pub))
+	if pubPem == nil {
+		apiError(ctx, http.StatusInternalServerError, "failed to decode private key pem")
+		return
+	}
+
+	pubKey, err := x509.ParsePKIXPublicKey(pubPem.Bytes)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	fingerprint, err := util.CreatePublicKeyFingerprint(pubKey)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	ctx.ServeContent(strings.NewReader(pub), &context.ServeHeaderOptions{
+		ContentType: "application/x-pem-file",
+		Filename:    fmt.Sprintf("%s@%s.rsa.pub", ctx.Package.Owner.LowerName, hex.EncodeToString(fingerprint)),
+	})
+}
+
+func GetRepositoryFile(ctx *context.Context) {
+	pv, err := alpine_service.GetOrCreateRepositoryVersion(ctx.Package.Owner.ID)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	s, pf, err := packages_service.GetFileStreamByPackageVersion(
+		ctx,
+		pv,
+		&packages_service.PackageFileInfo{
+			Filename:     alpine_service.IndexFilename,
+			CompositeKey: fmt.Sprintf("%s|%s|%s", ctx.Params("branch"), ctx.Params("repository"), ctx.Params("architecture")),
+		},
+	)
+	if err != nil {
+		if errors.Is(err, util.ErrNotExist) {
+			apiError(ctx, http.StatusNotFound, err)
+		} else {
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+	defer s.Close()
+
+	ctx.ServeContent(s, &context.ServeHeaderOptions{
+		Filename:     pf.Name,
+		LastModified: pf.CreatedUnix.AsLocalTime(),
+	})
+}
+
+func UploadPackageFile(ctx *context.Context) {
+	branch := strings.TrimSpace(ctx.Params("branch"))
+	repository := strings.TrimSpace(ctx.Params("repository"))
+	if branch == "" || repository == "" {
+		apiError(ctx, http.StatusBadRequest, "invalid branch or repository")
+		return
+	}
+
+	upload, close, err := ctx.UploadStream()
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	if close {
+		defer upload.Close()
+	}
+
+	buf, err := packages_module.CreateHashedBufferFromReader(upload)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	defer buf.Close()
+
+	pck, err := alpine_module.ParsePackage(buf)
+	if err != nil {
+		if errors.Is(err, util.ErrInvalidArgument) || err == io.EOF {
+			apiError(ctx, http.StatusBadRequest, err)
+		} else {
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+
+	if _, err := buf.Seek(0, io.SeekStart); err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	fileMetadataRaw, err := json.Marshal(pck.FileMetadata)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	_, _, err = packages_service.CreatePackageOrAddFileToExisting(
+		&packages_service.PackageCreationInfo{
+			PackageInfo: packages_service.PackageInfo{
+				Owner:       ctx.Package.Owner,
+				PackageType: packages_model.TypeAlpine,
+				Name:        pck.Name,
+				Version:     pck.Version,
+			},
+			Creator:  ctx.Doer,
+			Metadata: pck.VersionMetadata,
+		},
+		&packages_service.PackageFileCreationInfo{
+			PackageFileInfo: packages_service.PackageFileInfo{
+				Filename:     fmt.Sprintf("%s-%s.apk", pck.Name, pck.Version),
+				CompositeKey: fmt.Sprintf("%s|%s|%s", branch, repository, pck.FileMetadata.Architecture),
+			},
+			Creator: ctx.Doer,
+			Data:    buf,
+			IsLead:  true,
+			Properties: map[string]string{
+				alpine_module.PropertyBranch:       branch,
+				alpine_module.PropertyRepository:   repository,
+				alpine_module.PropertyArchitecture: pck.FileMetadata.Architecture,
+				alpine_module.PropertyMetadata:     string(fileMetadataRaw),
+			},
+		},
+	)
+	if err != nil {
+		switch err {
+		case packages_model.ErrDuplicatePackageVersion, packages_model.ErrDuplicatePackageFile:
+			apiError(ctx, http.StatusBadRequest, err)
+		case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
+			apiError(ctx, http.StatusForbidden, err)
+		default:
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+
+	if err := alpine_service.BuildSpecificRepositoryFiles(ctx, ctx.Package.Owner.ID, branch, repository, pck.FileMetadata.Architecture); err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	ctx.Status(http.StatusCreated)
+}
+
+func DownloadPackageFile(ctx *context.Context) {
+	pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{
+		OwnerID:      ctx.Package.Owner.ID,
+		PackageType:  packages_model.TypeAlpine,
+		Query:        ctx.Params("filename"),
+		CompositeKey: fmt.Sprintf("%s|%s|%s", ctx.Params("branch"), ctx.Params("repository"), ctx.Params("architecture")),
+	})
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	if len(pfs) != 1 {
+		apiError(ctx, http.StatusNotFound, nil)
+		return
+	}
+
+	s, pf, err := packages_service.GetPackageFileStream(ctx, pfs[0])
+	if err != nil {
+		if errors.Is(err, util.ErrNotExist) {
+			apiError(ctx, http.StatusNotFound, err)
+		} else {
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+	defer s.Close()
+
+	ctx.ServeContent(s, &context.ServeHeaderOptions{
+		Filename:     pf.Name,
+		LastModified: pf.CreatedUnix.AsLocalTime(),
+	})
+}
+
+func DeletePackageFile(ctx *context.Context) {
+	branch, repository, architecture := ctx.Params("branch"), ctx.Params("repository"), ctx.Params("architecture")
+
+	pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{
+		OwnerID:      ctx.Package.Owner.ID,
+		PackageType:  packages_model.TypeAlpine,
+		Query:        ctx.Params("filename"),
+		CompositeKey: fmt.Sprintf("%s|%s|%s", branch, repository, architecture),
+	})
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	if len(pfs) != 1 {
+		apiError(ctx, http.StatusNotFound, nil)
+		return
+	}
+
+	if err := packages_service.RemovePackageFileAndVersionIfUnreferenced(ctx.Doer, pfs[0]); err != nil {
+		if errors.Is(err, util.ErrNotExist) {
+			apiError(ctx, http.StatusNotFound, err)
+		} else {
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+
+	if err := alpine_service.BuildSpecificRepositoryFiles(ctx, ctx.Package.Owner.ID, branch, repository, architecture); err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	ctx.Status(http.StatusNoContent)
+}

routers/api/packages/api.go

@@ -15,6 +15,7 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/web"
+	"code.gitea.io/gitea/routers/api/packages/alpine"
 	"code.gitea.io/gitea/routers/api/packages/cargo"
 	"code.gitea.io/gitea/routers/api/packages/chef"
 	"code.gitea.io/gitea/routers/api/packages/composer"
@@ -23,6 +24,7 @@ import (
 	"code.gitea.io/gitea/routers/api/packages/container"
 	"code.gitea.io/gitea/routers/api/packages/debian"
 	"code.gitea.io/gitea/routers/api/packages/generic"
+	"code.gitea.io/gitea/routers/api/packages/goproxy"
 	"code.gitea.io/gitea/routers/api/packages/helm"
 	"code.gitea.io/gitea/routers/api/packages/maven"
 	"code.gitea.io/gitea/routers/api/packages/npm"
@@ -107,6 +109,19 @@ func CommonRoutes(ctx gocontext.Context) *web.Route {
 	})
 
 	r.Group("/{username}", func() {
+		r.Group("/alpine", func() {
+			r.Get("/key", alpine.GetRepositoryKey)
+			r.Group("/{branch}/{repository}", func() {
+				r.Put("", reqPackageAccess(perm.AccessModeWrite), alpine.UploadPackageFile)
+				r.Group("/{architecture}", func() {
+					r.Get("/APKINDEX.tar.gz", alpine.GetRepositoryFile)
+					r.Group("/{filename}", func() {
+						r.Get("", alpine.DownloadPackageFile)
+						r.Delete("", reqPackageAccess(perm.AccessModeWrite), alpine.DeletePackageFile)
+					})
+				})
+			})
+		}, reqPackageAccess(perm.AccessModeRead))
 		r.Group("/cargo", func() {
 			r.Group("/api/v1/crates", func() {
 				r.Get("", cargo.SearchPackages)
@@ -298,6 +313,64 @@ func CommonRoutes(ctx gocontext.Context) *web.Route {
 				}, reqPackageAccess(perm.AccessModeWrite))
 			})
 		}, reqPackageAccess(perm.AccessModeRead))
+		r.Group("/go", func() {
+			r.Put("/upload", reqPackageAccess(perm.AccessModeWrite), goproxy.UploadPackage)
+			r.Get("/sumdb/sum.golang.org/supported", func(ctx *context.Context) {
+				ctx.Status(http.StatusNotFound)
+			})
+
+			// Manual mapping of routes because the package name contains slashes which chi does not support
+			// https://go.dev/ref/mod#goproxy-protocol
+			r.Get("/*", func(ctx *context.Context) {
+				path := ctx.Params("*")
+
+				if strings.HasSuffix(path, "/@latest") {
+					ctx.SetParams("name", path[:len(path)-len("/@latest")])
+					ctx.SetParams("version", "latest")
+
+					goproxy.PackageVersionMetadata(ctx)
+					return
+				}
+
+				parts := strings.SplitN(path, "/@v/", 2)
+				if len(parts) != 2 {
+					ctx.Status(http.StatusNotFound)
+					return
+				}
+
+				ctx.SetParams("name", parts[0])
+
+				// <package/name>/@v/list
+				if parts[1] == "list" {
+					goproxy.EnumeratePackageVersions(ctx)
+					return
+				}
+
+				// <package/name>/@v/<version>.zip
+				if strings.HasSuffix(parts[1], ".zip") {
+					ctx.SetParams("version", parts[1][:len(parts[1])-len(".zip")])
+
+					goproxy.DownloadPackageFile(ctx)
+					return
+				}
+				// <package/name>/@v/<version>.info
+				if strings.HasSuffix(parts[1], ".info") {
+					ctx.SetParams("version", parts[1][:len(parts[1])-len(".info")])
+
+					goproxy.PackageVersionMetadata(ctx)
+					return
+				}
+				// <package/name>/@v/<version>.mod
+				if strings.HasSuffix(parts[1], ".mod") {
+					ctx.SetParams("version", parts[1][:len(parts[1])-len(".mod")])
+
+					goproxy.PackageVersionGoModContent(ctx)
+					return
+				}
+
+				ctx.Status(http.StatusNotFound)
+			})
+		}, reqPackageAccess(perm.AccessModeRead))
 		r.Group("/generic", func() {
 			r.Group("/{packagename}/{packageversion}", func() {
 				r.Delete("", reqPackageAccess(perm.AccessModeWrite), generic.DeletePackage)

routers/api/packages/goproxy/goproxy.go

@@ -0,0 +1,226 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package goproxy
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"sort"
+	"time"
+
+	packages_model "code.gitea.io/gitea/models/packages"
+	"code.gitea.io/gitea/modules/context"
+	packages_module "code.gitea.io/gitea/modules/packages"
+	goproxy_module "code.gitea.io/gitea/modules/packages/goproxy"
+	"code.gitea.io/gitea/modules/util"
+	"code.gitea.io/gitea/routers/api/packages/helper"
+	packages_service "code.gitea.io/gitea/services/packages"
+)
+
+func apiError(ctx *context.Context, status int, obj interface{}) {
+	helper.LogAndProcessError(ctx, status, obj, func(message string) {
+		ctx.PlainText(status, message)
+	})
+}
+
+func EnumeratePackageVersions(ctx *context.Context) {
+	pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypeGo, ctx.Params("name"))
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	if len(pvs) == 0 {
+		apiError(ctx, http.StatusNotFound, err)
+		return
+	}
+
+	sort.Slice(pvs, func(i, j int) bool {
+		return pvs[i].CreatedUnix < pvs[j].CreatedUnix
+	})
+
+	ctx.Resp.Header().Set("Content-Type", "text/plain;charset=utf-8")
+
+	for _, pv := range pvs {
+		fmt.Fprintln(ctx.Resp, pv.Version)
+	}
+}
+
+func PackageVersionMetadata(ctx *context.Context) {
+	pv, err := resolvePackage(ctx, ctx.Package.Owner.ID, ctx.Params("name"), ctx.Params("version"))
+	if err != nil {
+		if errors.Is(err, util.ErrNotExist) {
+			apiError(ctx, http.StatusNotFound, err)
+		} else {
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+
+	ctx.JSON(http.StatusOK, struct {
+		Version string    `json:"Version"`
+		Time    time.Time `json:"Time"`
+	}{
+		Version: pv.Version,
+		Time:    pv.CreatedUnix.AsLocalTime(),
+	})
+}
+
+func PackageVersionGoModContent(ctx *context.Context) {
+	pv, err := resolvePackage(ctx, ctx.Package.Owner.ID, ctx.Params("name"), ctx.Params("version"))
+	if err != nil {
+		if errors.Is(err, util.ErrNotExist) {
+			apiError(ctx, http.StatusNotFound, err)
+		} else {
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+
+	pps, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeVersion, pv.ID, goproxy_module.PropertyGoMod)
+	if err != nil || len(pps) != 1 {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	ctx.PlainText(http.StatusOK, pps[0].Value)
+}
+
+func DownloadPackageFile(ctx *context.Context) {
+	pv, err := resolvePackage(ctx, ctx.Package.Owner.ID, ctx.Params("name"), ctx.Params("version"))
+	if err != nil {
+		if errors.Is(err, util.ErrNotExist) {
+			apiError(ctx, http.StatusNotFound, err)
+		} else {
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+
+	pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID)
+	if err != nil || len(pfs) != 1 {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	s, _, err := packages_service.GetPackageFileStream(ctx, pfs[0])
+	if err != nil {
+		if errors.Is(err, util.ErrNotExist) {
+			apiError(ctx, http.StatusNotFound, err)
+		} else {
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+	defer s.Close()
+
+	ctx.ServeContent(s, &context.ServeHeaderOptions{
+		Filename:     pfs[0].Name,
+		LastModified: pfs[0].CreatedUnix.AsLocalTime(),
+	})
+}
+
+func resolvePackage(ctx *context.Context, ownerID int64, name, version string) (*packages_model.PackageVersion, error) {
+	var pv *packages_model.PackageVersion
+
+	if version == "latest" {
+		pvs, _, err := packages_model.SearchLatestVersions(ctx, &packages_model.PackageSearchOptions{
+			OwnerID: ownerID,
+			Type:    packages_model.TypeGo,
+			Name: packages_model.SearchValue{
+				Value:      name,
+				ExactMatch: true,
+			},
+			IsInternal: util.OptionalBoolFalse,
+			Sort:       packages_model.SortCreatedDesc,
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		if len(pvs) != 1 {
+			return nil, packages_model.ErrPackageNotExist
+		}
+
+		pv = pvs[0]
+	} else {
+		var err error
+		pv, err = packages_model.GetVersionByNameAndVersion(ctx, ownerID, packages_model.TypeGo, name, version)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return pv, nil
+}
+
+func UploadPackage(ctx *context.Context) {
+	upload, close, err := ctx.UploadStream()
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	if close {
+		defer upload.Close()
+	}
+
+	buf, err := packages_module.CreateHashedBufferFromReader(upload)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	defer buf.Close()
+
+	pck, err := goproxy_module.ParsePackage(buf, buf.Size())
+	if err != nil {
+		if errors.Is(err, util.ErrInvalidArgument) {
+			apiError(ctx, http.StatusBadRequest, err)
+		} else {
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+
+	if _, err := buf.Seek(0, io.SeekStart); err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	_, _, err = packages_service.CreatePackageAndAddFile(
+		&packages_service.PackageCreationInfo{
+			PackageInfo: packages_service.PackageInfo{
+				Owner:       ctx.Package.Owner,
+				PackageType: packages_model.TypeGo,
+				Name:        pck.Name,
+				Version:     pck.Version,
+			},
+			Creator: ctx.Doer,
+			VersionProperties: map[string]string{
+				goproxy_module.PropertyGoMod: pck.GoMod,
+			},
+		},
+		&packages_service.PackageFileCreationInfo{
+			PackageFileInfo: packages_service.PackageFileInfo{
+				Filename: fmt.Sprintf("%v.zip", pck.Version),
+			},
+			Creator: ctx.Doer,
+			Data:    buf,
+			IsLead:  true,
+		},
+	)
+	if err != nil {
+		switch err {
+		case packages_model.ErrDuplicatePackageVersion:
+			apiError(ctx, http.StatusConflict, err)
+		case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
+			apiError(ctx, http.StatusForbidden, err)
+		default:
+			apiError(ctx, http.StatusInternalServerError, err)
+		}
+		return
+	}
+
+	ctx.Status(http.StatusCreated)
+}

routers/api/v1/org/team.go

@@ -126,7 +126,7 @@ func GetTeam(ctx *context.APIContext) {
 	//   "200":
 	//     "$ref": "#/responses/Team"
 
-	apiTeam, err := convert.ToTeam(ctx, ctx.Org.Team)
+	apiTeam, err := convert.ToTeam(ctx, ctx.Org.Team, true)
 	if err != nil {
 		ctx.InternalServerError(err)
 		return

routers/api/v1/packages/package.go

@@ -40,7 +40,7 @@ func ListPackages(ctx *context.APIContext) {
 	//   in: query
 	//   description: package type filter
 	//   type: string
-	//   enum: [cargo, chef, composer, conan, conda, container, debian, generic, helm, maven, npm, nuget, pub, pypi, rpm, rubygems, swift, vagrant]
+	//   enum: [alpine, cargo, chef, composer, conan, conda, container, debian, generic, go, helm, maven, npm, nuget, pub, pypi, rpm, rubygems, swift, vagrant]
 	// - name: q
 	//   in: query
 	//   description: name filter