Merge remote-tracking branch 'upstream/main' into limit-repo-size

services/auth/source/oauth2/jwtsigningkey.go

@@ -23,7 +23,6 @@ import (
 	"code.gitea.io/gitea/modules/util"
 
 	"github.com/golang-jwt/jwt/v4"
-	"github.com/minio/sha256-simd"
 )
 
 // ErrInvalidAlgorithmType represents an invalid algorithm error.
@@ -82,7 +81,7 @@ type rsaSingingKey struct {
 }
 
 func newRSASingingKey(signingMethod jwt.SigningMethod, key *rsa.PrivateKey) (rsaSingingKey, error) {
-	kid, err := createPublicKeyFingerprint(key.Public().(*rsa.PublicKey))
+	kid, err := util.CreatePublicKeyFingerprint(key.Public().(*rsa.PublicKey))
 	if err != nil {
 		return rsaSingingKey{}, err
 	}
@@ -133,7 +132,7 @@ type eddsaSigningKey struct {
 }
 
 func newEdDSASingingKey(signingMethod jwt.SigningMethod, key ed25519.PrivateKey) (eddsaSigningKey, error) {
-	kid, err := createPublicKeyFingerprint(key.Public().(ed25519.PublicKey))
+	kid, err := util.CreatePublicKeyFingerprint(key.Public().(ed25519.PublicKey))
 	if err != nil {
 		return eddsaSigningKey{}, err
 	}
@@ -184,7 +183,7 @@ type ecdsaSingingKey struct {
 }
 
 func newECDSASingingKey(signingMethod jwt.SigningMethod, key *ecdsa.PrivateKey) (ecdsaSingingKey, error) {
-	kid, err := createPublicKeyFingerprint(key.Public().(*ecdsa.PublicKey))
+	kid, err := util.CreatePublicKeyFingerprint(key.Public().(*ecdsa.PublicKey))
 	if err != nil {
 		return ecdsaSingingKey{}, err
 	}
@@ -229,19 +228,6 @@ func (key ecdsaSingingKey) PreProcessToken(token *jwt.Token) {
 	token.Header["kid"] = key.id
 }
 
-// createPublicKeyFingerprint creates a fingerprint of the given key.
-// The fingerprint is the sha256 sum of the PKIX structure of the key.
-func createPublicKeyFingerprint(key interface{}) ([]byte, error) {
-	bytes, err := x509.MarshalPKIXPublicKey(key)
-	if err != nil {
-		return nil, err
-	}
-
-	checksum := sha256.Sum256(bytes)
-
-	return checksum[:], nil
-}
-
 // CreateJWTSigningKey creates a signing key from an algorithm / key pair.
 func CreateJWTSigningKey(algorithm string, key interface{}) (JWTSigningKey, error) {
 	var signingMethod jwt.SigningMethod

services/convert/repository.go

@@ -124,11 +124,10 @@ func innerToRepo(ctx context.Context, repo *repo_model.Repository, mode perm.Acc
 	mirrorInterval := ""
 	var mirrorUpdated time.Time
 	if repo.IsMirror {
-		var err error
-		repo.Mirror, err = repo_model.GetMirrorByRepoID(ctx, repo.ID)
+		pullMirror, err := repo_model.GetMirrorByRepoID(ctx, repo.ID)
 		if err == nil {
-			mirrorInterval = repo.Mirror.Interval.String()
-			mirrorUpdated = repo.Mirror.UpdatedUnix.AsTime()
+			mirrorInterval = pullMirror.Interval.String()
+			mirrorUpdated = pullMirror.UpdatedUnix.AsTime()
 		}
 	}
 

services/forms/package_form.go

@@ -15,7 +15,7 @@ import (
 type PackageCleanupRuleForm struct {
 	ID            int64
 	Enabled       bool
-	Type          string `binding:"Required;In(cargo,chef,composer,conan,conda,container,debian,generic,helm,maven,npm,nuget,pub,pypi,rpm,rubygems,swift,vagrant)"`
+	Type          string `binding:"Required;In(alpine,cargo,chef,composer,conan,conda,container,debian,generic,go,helm,maven,npm,nuget,pub,pypi,rpm,rubygems,swift,vagrant)"`
 	KeepCount     int    `binding:"In(0,1,5,10,25,50,100)"`
 	KeepPattern   string `binding:"RegexPattern"`
 	RemoveDays    int    `binding:"In(0,7,14,30,60,90,180)"`

services/packages/alpine/repository.go

@@ -0,0 +1,328 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package alpine
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/sha1"
+	"crypto/x509"
+	"encoding/hex"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+
+	packages_model "code.gitea.io/gitea/models/packages"
+	alpine_model "code.gitea.io/gitea/models/packages/alpine"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/json"
+	packages_module "code.gitea.io/gitea/modules/packages"
+	alpine_module "code.gitea.io/gitea/modules/packages/alpine"
+	"code.gitea.io/gitea/modules/util"
+	packages_service "code.gitea.io/gitea/services/packages"
+)
+
+const IndexFilename = "APKINDEX.tar.gz"
+
+// GetOrCreateRepositoryVersion gets or creates the internal repository package
+// The Alpine registry needs multiple index files which are stored in this package.
+func GetOrCreateRepositoryVersion(ownerID int64) (*packages_model.PackageVersion, error) {
+	return packages_service.GetOrCreateInternalPackageVersion(ownerID, packages_model.TypeAlpine, alpine_module.RepositoryPackage, alpine_module.RepositoryVersion)
+}
+
+// GetOrCreateKeyPair gets or creates the RSA keys used to sign repository files
+func GetOrCreateKeyPair(ownerID int64) (string, string, error) {
+	priv, err := user_model.GetSetting(ownerID, alpine_module.SettingKeyPrivate)
+	if err != nil && !errors.Is(err, util.ErrNotExist) {
+		return "", "", err
+	}
+
+	pub, err := user_model.GetSetting(ownerID, alpine_module.SettingKeyPublic)
+	if err != nil && !errors.Is(err, util.ErrNotExist) {
+		return "", "", err
+	}
+
+	if priv == "" || pub == "" {
+		priv, pub, err = util.GenerateKeyPair(4096)
+		if err != nil {
+			return "", "", err
+		}
+
+		if err := user_model.SetUserSetting(ownerID, alpine_module.SettingKeyPrivate, priv); err != nil {
+			return "", "", err
+		}
+
+		if err := user_model.SetUserSetting(ownerID, alpine_module.SettingKeyPublic, pub); err != nil {
+			return "", "", err
+		}
+	}
+
+	return priv, pub, nil
+}
+
+// BuildAllRepositoryFiles (re)builds all repository files for every available distributions, components and architectures
+func BuildAllRepositoryFiles(ctx context.Context, ownerID int64) error {
+	pv, err := GetOrCreateRepositoryVersion(ownerID)
+	if err != nil {
+		return err
+	}
+
+	// 1. Delete all existing repository files
+	pfs, err := packages_model.GetFilesByVersionID(ctx, pv.ID)
+	if err != nil {
+		return err
+	}
+
+	for _, pf := range pfs {
+		if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
+			return err
+		}
+		if err := packages_model.DeleteFileByID(ctx, pf.ID); err != nil {
+			return err
+		}
+	}
+
+	// 2. (Re)Build repository files for existing packages
+	branches, err := alpine_model.GetBranches(ctx, ownerID)
+	if err != nil {
+		return err
+	}
+	for _, branch := range branches {
+		repositories, err := alpine_model.GetRepositories(ctx, ownerID, branch)
+		if err != nil {
+			return err
+		}
+		for _, repository := range repositories {
+			architectures, err := alpine_model.GetArchitectures(ctx, ownerID, repository)
+			if err != nil {
+				return err
+			}
+			for _, architecture := range architectures {
+				if err := buildPackagesIndex(ctx, ownerID, pv, branch, repository, architecture); err != nil {
+					return fmt.Errorf("failed to build repository files [%s/%s/%s]: %w", branch, repository, architecture, err)
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+// BuildSpecificRepositoryFiles builds index files for the repository
+func BuildSpecificRepositoryFiles(ctx context.Context, ownerID int64, branch, repository, architecture string) error {
+	pv, err := GetOrCreateRepositoryVersion(ownerID)
+	if err != nil {
+		return err
+	}
+
+	return buildPackagesIndex(ctx, ownerID, pv, branch, repository, architecture)
+}
+
+type packageData struct {
+	Package         *packages_model.Package
+	Version         *packages_model.PackageVersion
+	Blob            *packages_model.PackageBlob
+	VersionMetadata *alpine_module.VersionMetadata
+	FileMetadata    *alpine_module.FileMetadata
+}
+
+type packageCache = map[*packages_model.PackageFile]*packageData
+
+// https://wiki.alpinelinux.org/wiki/Apk_spec#APKINDEX_Format
+func buildPackagesIndex(ctx context.Context, ownerID int64, repoVersion *packages_model.PackageVersion, branch, repository, architecture string) error {
+	pfs, _, err := packages_model.SearchFiles(ctx, &packages_model.PackageFileSearchOptions{
+		OwnerID:     ownerID,
+		PackageType: packages_model.TypeAlpine,
+		Query:       "%.apk",
+		Properties: map[string]string{
+			alpine_module.PropertyBranch:       branch,
+			alpine_module.PropertyRepository:   repository,
+			alpine_module.PropertyArchitecture: architecture,
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	// Delete the package indices if there are no packages
+	if len(pfs) == 0 {
+		pf, err := packages_model.GetFileForVersionByName(ctx, repoVersion.ID, IndexFilename, fmt.Sprintf("%s|%s|%s", branch, repository, architecture))
+		if err != nil && !errors.Is(err, util.ErrNotExist) {
+			return err
+		}
+
+		if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeFile, pf.ID); err != nil {
+			return err
+		}
+		return packages_model.DeleteFileByID(ctx, pf.ID)
+	}
+
+	// Cache data needed for all repository files
+	cache := make(packageCache)
+	for _, pf := range pfs {
+		pv, err := packages_model.GetVersionByID(ctx, pf.VersionID)
+		if err != nil {
+			return err
+		}
+		p, err := packages_model.GetPackageByID(ctx, pv.PackageID)
+		if err != nil {
+			return err
+		}
+		pb, err := packages_model.GetBlobByID(ctx, pf.BlobID)
+		if err != nil {
+			return err
+		}
+		pps, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeFile, pf.ID, alpine_module.PropertyMetadata)
+		if err != nil {
+			return err
+		}
+
+		pd := &packageData{
+			Package: p,
+			Version: pv,
+			Blob:    pb,
+		}
+
+		if err := json.Unmarshal([]byte(pv.MetadataJSON), &pd.VersionMetadata); err != nil {
+			return err
+		}
+		if len(pps) > 0 {
+			if err := json.Unmarshal([]byte(pps[0].Value), &pd.FileMetadata); err != nil {
+				return err
+			}
+		}
+
+		cache[pf] = pd
+	}
+
+	var buf bytes.Buffer
+	for _, pf := range pfs {
+		pd := cache[pf]
+
+		fmt.Fprintf(&buf, "C:%s\n", pd.FileMetadata.Checksum)
+		fmt.Fprintf(&buf, "P:%s\n", pd.Package.Name)
+		fmt.Fprintf(&buf, "V:%s\n", pd.Version.Version)
+		fmt.Fprintf(&buf, "A:%s\n", pd.FileMetadata.Architecture)
+		if pd.VersionMetadata.Description != "" {
+			fmt.Fprintf(&buf, "T:%s\n", pd.VersionMetadata.Description)
+		}
+		if pd.VersionMetadata.ProjectURL != "" {
+			fmt.Fprintf(&buf, "U:%s\n", pd.VersionMetadata.ProjectURL)
+		}
+		if pd.VersionMetadata.License != "" {
+			fmt.Fprintf(&buf, "L:%s\n", pd.VersionMetadata.License)
+		}
+		fmt.Fprintf(&buf, "S:%d\n", pd.Blob.Size)
+		fmt.Fprintf(&buf, "I:%d\n", pd.FileMetadata.Size)
+		fmt.Fprintf(&buf, "o:%s\n", pd.FileMetadata.Origin)
+		fmt.Fprintf(&buf, "m:%s\n", pd.VersionMetadata.Maintainer)
+		fmt.Fprintf(&buf, "t:%d\n", pd.FileMetadata.BuildDate)
+		if pd.FileMetadata.CommitHash != "" {
+			fmt.Fprintf(&buf, "c:%s\n", pd.FileMetadata.CommitHash)
+		}
+		if len(pd.FileMetadata.Dependencies) > 0 {
+			fmt.Fprintf(&buf, "D:%s\n", strings.Join(pd.FileMetadata.Dependencies, " "))
+		}
+		if len(pd.FileMetadata.Provides) > 0 {
+			fmt.Fprintf(&buf, "p:%s\n", strings.Join(pd.FileMetadata.Provides, " "))
+		}
+		fmt.Fprint(&buf, "\n")
+	}
+
+	unsignedIndexContent, _ := packages_module.NewHashedBuffer()
+	h := sha1.New()
+
+	if err := writeGzipStream(io.MultiWriter(unsignedIndexContent, h), "APKINDEX", buf.Bytes(), true); err != nil {
+		return err
+	}
+
+	priv, _, err := GetOrCreateKeyPair(ownerID)
+	if err != nil {
+		return err
+	}
+
+	privPem, _ := pem.Decode([]byte(priv))
+	if privPem == nil {
+		return fmt.Errorf("failed to decode private key pem")
+	}
+
+	privKey, err := x509.ParsePKCS1PrivateKey(privPem.Bytes)
+	if err != nil {
+		return err
+	}
+
+	sign, err := rsa.SignPKCS1v15(rand.Reader, privKey, crypto.SHA1, h.Sum(nil))
+	if err != nil {
+		return err
+	}
+
+	owner, err := user_model.GetUserByID(ctx, ownerID)
+	if err != nil {
+		return err
+	}
+
+	fingerprint, err := util.CreatePublicKeyFingerprint(&privKey.PublicKey)
+	if err != nil {
+		return err
+	}
+
+	signedIndexContent, _ := packages_module.NewHashedBuffer()
+
+	if err := writeGzipStream(
+		signedIndexContent,
+		fmt.Sprintf(".SIGN.RSA.%s@%s.rsa.pub", owner.LowerName, hex.EncodeToString(fingerprint)),
+		sign,
+		false,
+	); err != nil {
+		return err
+	}
+
+	if _, err := io.Copy(signedIndexContent, unsignedIndexContent); err != nil {
+		return err
+	}
+
+	_, err = packages_service.AddFileToPackageVersionInternal(
+		repoVersion,
+		&packages_service.PackageFileCreationInfo{
+			PackageFileInfo: packages_service.PackageFileInfo{
+				Filename:     IndexFilename,
+				CompositeKey: fmt.Sprintf("%s|%s|%s", branch, repository, architecture),
+			},
+			Creator:           user_model.NewGhostUser(),
+			Data:              signedIndexContent,
+			IsLead:            false,
+			OverwriteExisting: true,
+		},
+	)
+	return err
+}
+
+func writeGzipStream(w io.Writer, filename string, content []byte, addTarEnd bool) error {
+	zw := gzip.NewWriter(w)
+	defer zw.Close()
+
+	tw := tar.NewWriter(zw)
+	if addTarEnd {
+		defer tw.Close()
+	}
+	hdr := &tar.Header{
+		Name: filename,
+		Mode: 0o600,
+		Size: int64(len(content)),
+	}
+	if err := tw.WriteHeader(hdr); err != nil {
+		return err
+	}
+	if _, err := tw.Write(content); err != nil {
+		return err
+	}
+	return nil
+}

services/packages/packages.go

@@ -351,6 +351,8 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p
 
 	var typeSpecificSize int64
 	switch packageType {
+	case packages_model.TypeAlpine:
+		typeSpecificSize = setting.Packages.LimitSizeAlpine
 	case packages_model.TypeCargo:
 		typeSpecificSize = setting.Packages.LimitSizeCargo
 	case packages_model.TypeChef:
@@ -367,6 +369,8 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p
 		typeSpecificSize = setting.Packages.LimitSizeDebian
 	case packages_model.TypeGeneric:
 		typeSpecificSize = setting.Packages.LimitSizeGeneric
+	case packages_model.TypeGo:
+		typeSpecificSize = setting.Packages.LimitSizeGo
 	case packages_model.TypeHelm:
 		typeSpecificSize = setting.Packages.LimitSizeHelm
 	case packages_model.TypeMaven:
@@ -486,6 +490,47 @@ func RemovePackageVersion(doer *user_model.User, pv *packages_model.PackageVersi
 	return nil
 }
 
+// RemovePackageFileAndVersionIfUnreferenced deletes the package file and the version if there are no referenced files afterwards
+func RemovePackageFileAndVersionIfUnreferenced(doer *user_model.User, pf *packages_model.PackageFile) error {
+	var pd *packages_model.PackageDescriptor
+
+	if err := db.WithTx(db.DefaultContext, func(ctx context.Context) error {
+		if err := DeletePackageFile(ctx, pf); err != nil {
+			return err
+		}
+
+		has, err := packages_model.HasVersionFileReferences(ctx, pf.VersionID)
+		if err != nil {
+			return err
+		}
+		if !has {
+			pv, err := packages_model.GetVersionByID(ctx, pf.VersionID)
+			if err != nil {
+				return err
+			}
+
+			pd, err = packages_model.GetPackageDescriptor(ctx, pv)
+			if err != nil {
+				return err
+			}
+
+			if err := DeletePackageVersionAndReferences(ctx, pv); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	if pd != nil {
+		notification.NotifyPackageDelete(db.DefaultContext, doer, pd)
+	}
+
+	return nil
+}
+
 // DeletePackageVersionAndReferences deletes the package version and its properties and files
 func DeletePackageVersionAndReferences(ctx context.Context, pv *packages_model.PackageVersion) error {
 	if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypeVersion, pv.ID); err != nil {

services/repository/avatar.go

@@ -6,7 +6,6 @@ package repository
 import (
 	"context"
 	"fmt"
-	"image/png"
 	"io"
 	"strconv"
 	"strings"
@@ -21,7 +20,7 @@ import (
 // UploadAvatar saves custom avatar for repository.
 // FIXME: split uploads to different subdirs in case we have massive number of repos.
 func UploadAvatar(ctx context.Context, repo *repo_model.Repository, data []byte) error {
-	m, err := avatar.Prepare(data)
+	avatarData, err := avatar.ProcessAvatarImage(data)
 	if err != nil {
 		return err
 	}
@@ -47,9 +46,7 @@ func UploadAvatar(ctx context.Context, repo *repo_model.Repository, data []byte)
 	}
 
 	if err := storage.SaveFrom(storage.RepoAvatars, repo.CustomAvatarRelativePath(), func(w io.Writer) error {
-		if err := png.Encode(w, *m); err != nil {
-			log.Error("Encode: %v", err)
-		}
+		_, err := w.Write(avatarData)
 		return err
 	}); err != nil {
 		return fmt.Errorf("UploadAvatar %s failed: Failed to remove old repo avatar %s: %w", repo.RepoPath(), newAvatar, err)

services/repository/branch.go

@@ -53,6 +53,10 @@ func GetBranches(ctx context.Context, repo *repo_model.Repository, skip, limit i
 	return git.GetBranchesByPath(ctx, repo.RepoPath(), skip, limit)
 }
 
+func GetBranchCommitID(ctx context.Context, repo *repo_model.Repository, branch string) (string, error) {
+	return git.GetBranchCommitID(ctx, repo.RepoPath(), branch)
+}
+
 // checkBranchName validates branch name with existing repository branches
 func checkBranchName(ctx context.Context, repo *repo_model.Repository, name string) error {
 	_, err := git.WalkReferences(ctx, repo.RepoPath(), func(_, refName string) error {

services/repository/repository.go

@@ -9,6 +9,7 @@ import (
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/models/git"
 	issues_model "code.gitea.io/gitea/models/issues"
 	"code.gitea.io/gitea/models/organization"
 	packages_model "code.gitea.io/gitea/models/packages"
@@ -20,9 +21,22 @@ import (
 	"code.gitea.io/gitea/modules/notification"
 	repo_module "code.gitea.io/gitea/modules/repository"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/structs"
 	pull_service "code.gitea.io/gitea/services/pull"
 )
 
+// WebSearchRepository represents a repository returned by web search
+type WebSearchRepository struct {
+	Repository         *structs.Repository `json:"repository"`
+	LatestCommitStatus *git.CommitStatus   `json:"latest_commit_status"`
+}
+
+// WebSearchResults results of a successful web search
+type WebSearchResults struct {
+	OK   bool                   `json:"ok"`
+	Data []*WebSearchRepository `json:"data"`
+}
+
 // CreateRepository creates a repository for the user/organization.
 func CreateRepository(ctx context.Context, doer, owner *user_model.User, opts repo_module.CreateRepoOptions) (*repo_model.Repository, error) {
 	repo, err := repo_module.CreateRepository(doer, owner, opts)

services/user/user.go

@@ -6,7 +6,6 @@ package user
 import (
 	"context"
 	"fmt"
-	"image/png"
 	"io"
 	"time"
 
@@ -244,7 +243,7 @@ func DeleteInactiveUsers(ctx context.Context, olderThan time.Duration) error {
 
 // UploadAvatar saves custom avatar for user.
 func UploadAvatar(u *user_model.User, data []byte) error {
-	m, err := avatar.Prepare(data)
+	avatarData, err := avatar.ProcessAvatarImage(data)
 	if err != nil {
 		return err
 	}
@@ -262,9 +261,7 @@ func UploadAvatar(u *user_model.User, data []byte) error {
 	}
 
 	if err := storage.SaveFrom(storage.Avatars, u.CustomAvatarRelativePath(), func(w io.Writer) error {
-		if err := png.Encode(w, *m); err != nil {
-			log.Error("Encode: %v", err)
-		}
+		_, err := w.Write(avatarData)
 		return err
 	}); err != nil {
 		return fmt.Errorf("Failed to create dir %s: %w", u.CustomAvatarRelativePath(), err)