mirror of
https://github.com/go-gitea/gitea.git
synced 2026-01-21 04:45:23 +01:00
Merge a619bbae8c790067fab7b3ac3aa81aaa729a3f7d into 65422fde4dc53ff806cc47d0788491183d2e2dbc
This commit is contained in:
commit
eff7a712c9
@ -2676,7 +2676,12 @@ LEVEL = Info
|
||||
;LIMIT_SIZE_HELM = -1
|
||||
;; Maximum size of a Maven upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
|
||||
;LIMIT_SIZE_MAVEN = -1
|
||||
;; Specifies the number of most recent Maven snapshot builds to retain. `-1` retains all builds, while `1` retains only the latest build. Value should be -1 or positive.
|
||||
;; Cleanup expired packages/data then targets the files within all maven snapshots versions
|
||||
;RETAIN_MAVEN_SNAPSHOT_BUILDS = -1
|
||||
;; Maximum size of a npm upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
|
||||
; Enable debug logging for Maven cleanup. Enabling debug will stop snapshot version artifacts from being deleted but will log the files which were meant for deletion.
|
||||
; DEBUG_MAVEN_CLEANUP = true
|
||||
;LIMIT_SIZE_NPM = -1
|
||||
;; Maximum size of a NuGet upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
|
||||
;LIMIT_SIZE_NUGET = -1
|
||||
|
||||
6
models/fixtures/package.yml
Normal file
6
models/fixtures/package.yml
Normal file
@ -0,0 +1,6 @@
|
||||
-
|
||||
id: 1
|
||||
owner_id: 2
|
||||
type: maven
|
||||
name: com.gitea:test-project
|
||||
lower_name: com.gitea:test-project
|
||||
88
models/fixtures/package_blob.yml
Normal file
88
models/fixtures/package_blob.yml
Normal file
@ -0,0 +1,88 @@
|
||||
-
|
||||
id: 1
|
||||
size: 14
|
||||
hash_md5: 27224a672372115e5a1d125ed7b2a0b1
|
||||
hash_sha1: 9854582a2958b2d31541ce4a6a1a36201b80c01
|
||||
hash_sha256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a01
|
||||
hash_sha512: ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a801
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 2
|
||||
size: 14
|
||||
hash_md5: 27224a672372115e5a1d125ed7b2a0b2
|
||||
hash_sha1: 9854582a2958b2d31541ce4a6a1a36201b80c02
|
||||
hash_sha256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a02
|
||||
hash_sha512: ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a802
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 3
|
||||
size: 14
|
||||
hash_md5: 27224a672372115e5a1d125ed7b2a0b3
|
||||
hash_sha1: 9854582a2958b2d31541ce4a6a1a36201b80c03
|
||||
hash_sha256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a03
|
||||
hash_sha512: ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a803
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 4
|
||||
size: 14
|
||||
hash_md5: 27224a672372115e5a1d125ed7b2a0b4
|
||||
hash_sha1: 9854582a2958b2d31541ce4a6a1a36201b80c04
|
||||
hash_sha256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a04
|
||||
hash_sha512: ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a804
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 5
|
||||
size: 14
|
||||
hash_md5: 27224a672372115e5a1d125ed7b2a0b5
|
||||
hash_sha1: 9854582a2958b2d31541ce4a6a1a36201b80c05
|
||||
hash_sha256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a05
|
||||
hash_sha512: ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a805
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 6
|
||||
size: 14
|
||||
hash_md5: 27224a672372115e5a1d125ed7b2a0b6
|
||||
hash_sha1: 9854582a2958b2d31541ce4a6a1a36201b80c06
|
||||
hash_sha256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a06
|
||||
hash_sha512: ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a806
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 7
|
||||
size: 14
|
||||
hash_md5: 27224a672372115e5a1d125ed7b2a0b7
|
||||
hash_sha1: 9854582a2958b2d31541ce4a6a1a36201b80c07
|
||||
hash_sha256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a07
|
||||
hash_sha512: ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a807
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 8
|
||||
size: 14
|
||||
hash_md5: 27224a672372115e5a1d125ed7b2a0b8
|
||||
hash_sha1: 9854582a2958b2d31541ce4a6a1a36201b80c08
|
||||
hash_sha256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08
|
||||
hash_sha512: ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a808
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 9
|
||||
size: 14
|
||||
hash_md5: 27224a672372115e5a1d125ed7b2a0b9
|
||||
hash_sha1: 9854582a2958b2d31541ce4a6a1a36201b80c09
|
||||
hash_sha256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a09
|
||||
hash_sha512: ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a809
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 10
|
||||
size: 14
|
||||
hash_md5: 27224a672372115e5a1d125ed7b2a0ba
|
||||
hash_sha1: 9854582a2958b2d31541ce4a6a1a36201b80c10
|
||||
hash_sha256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a10
|
||||
hash_sha512: ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a810
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 11
|
||||
size: 14
|
||||
hash_md5: 27224a672372115e5a1d125ed7b2a0bb
|
||||
hash_sha1: 9854582a2958b2d31541ce4a6a1a36201b80c11
|
||||
hash_sha256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a11
|
||||
hash_sha512: ee26b0dd4af7e749aa1a8ee3c10ae9923f618980772e473f8819a5d4940e0db27ac185f8a0e1d5f84f88bc887fd67b143732c304cc5fa9ad8e6f57f50028a811
|
||||
created_unix: 1672531200
|
||||
99
models/fixtures/package_file.yml
Normal file
99
models/fixtures/package_file.yml
Normal file
@ -0,0 +1,99 @@
|
||||
-
|
||||
id: 1
|
||||
version_id: 1
|
||||
blob_id: 1
|
||||
name: gitea-test-1.0-20230101.000000-1.jar
|
||||
lower_name: gitea-test-1.0-20230101.000000-1.jar
|
||||
composite_key: ""
|
||||
is_lead: false
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 2
|
||||
version_id: 1
|
||||
blob_id: 2
|
||||
name: gitea-test-1.0-20230101.000000-2.jar
|
||||
lower_name: gitea-test-1.0-20230101.000000-2.jar
|
||||
composite_key: ""
|
||||
is_lead: false
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 3
|
||||
version_id: 1
|
||||
blob_id: 3
|
||||
name: gitea-test-1.0-20230101.000000-3.jar
|
||||
lower_name: gitea-test-1.0-20230101.000000-3.jar
|
||||
composite_key: ""
|
||||
is_lead: false
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 4
|
||||
version_id: 1
|
||||
blob_id: 4
|
||||
name: gitea-test-1.0-20230101.000000-4.jar
|
||||
lower_name: gitea-test-1.0-20230101.000000-4.jar
|
||||
composite_key: ""
|
||||
is_lead: false
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 5
|
||||
version_id: 1
|
||||
blob_id: 5
|
||||
name: gitea-test-1.0-20230101.000000-5.jar
|
||||
lower_name: gitea-test-1.0-20230101.000000-5.jar
|
||||
composite_key: ""
|
||||
is_lead: false
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 6
|
||||
version_id: 1
|
||||
blob_id: 6
|
||||
name: gitea-test-1.0-20230101.000000-3-sources.jar
|
||||
lower_name: gitea-test-1.0-20230101.000000-3-sources.jar
|
||||
composite_key: ""
|
||||
is_lead: false
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 7
|
||||
version_id: 1
|
||||
blob_id: 7
|
||||
name: gitea-test-1.0-20230101.000000-3-javadoc.jar
|
||||
lower_name: gitea-test-1.0-20230101.000000-3-javadoc.jar
|
||||
composite_key: ""
|
||||
is_lead: false
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 8
|
||||
version_id: 1
|
||||
blob_id: 8
|
||||
name: gitea-test-1.0-20230101.000000-4-sources.jar
|
||||
lower_name: gitea-test-1.0-20230101.000000-4-sources.jar
|
||||
composite_key: ""
|
||||
is_lead: false
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 9
|
||||
version_id: 1
|
||||
blob_id: 9
|
||||
name: gitea-test-1.0-20230101.000000-4-javadoc.jar
|
||||
lower_name: gitea-test-1.0-20230101.000000-4-javadoc.jar
|
||||
composite_key: ""
|
||||
is_lead: false
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 10
|
||||
version_id: 1
|
||||
blob_id: 10
|
||||
name: gitea-test-1.0-20230101.000000-5-sources.jar
|
||||
lower_name: gitea-test-1.0-20230101.000000-5-sources.jar
|
||||
composite_key: ""
|
||||
is_lead: false
|
||||
created_unix: 1672531200
|
||||
-
|
||||
id: 11
|
||||
version_id: 1
|
||||
blob_id: 11
|
||||
name: gitea-test-1.0-20230101.000000-5-javadoc.jar
|
||||
lower_name: gitea-test-1.0-20230101.000000-5-javadoc.jar
|
||||
composite_key: ""
|
||||
is_lead: false
|
||||
created_unix: 1672531200
|
||||
8
models/fixtures/package_version.yml
Normal file
8
models/fixtures/package_version.yml
Normal file
@ -0,0 +1,8 @@
|
||||
-
|
||||
id: 1
|
||||
package_id: 1
|
||||
creator_id: 2
|
||||
version: 1.0-SNAPSHOT
|
||||
lower_version: 1.0-snapshot
|
||||
metadata_json: '{"artifact_id":"test-project","group_id":"com.gitea"}'
|
||||
created_unix: 1672531200
|
||||
@ -5,6 +5,9 @@ package packages
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@ -21,6 +24,8 @@ func init() {
|
||||
}
|
||||
|
||||
var (
|
||||
// ErrMetadataFile indicated a metadata file
|
||||
ErrMetadataFile = errors.New("metadata file")
|
||||
// ErrDuplicatePackageFile indicates a duplicated package file error
|
||||
ErrDuplicatePackageFile = util.NewAlreadyExistErrorf("package file already exists")
|
||||
// ErrPackageFileNotExist indicates a package file not exist error
|
||||
@ -231,6 +236,80 @@ func HasFiles(ctx context.Context, opts *PackageFileSearchOptions) (bool, error)
|
||||
return db.Exist[PackageFile](ctx, opts.toConds())
|
||||
}
|
||||
|
||||
// GetFilesBelowBuildNumber retrieves all files for a Maven snapshot version where the build number is <= maxBuildNumber.
|
||||
// Returns two slices: one for filtered files and one for skipped files.
|
||||
func GetFilesBelowBuildNumber(ctx context.Context, versionID int64, maxBuildNumber int, classifiers ...string) ([]*PackageFile, []*PackageFile, error) {
|
||||
if maxBuildNumber <= 0 {
|
||||
return nil, nil, errors.New("maxBuildNumber must be a positive integer")
|
||||
}
|
||||
|
||||
files, err := GetFilesByVersionID(ctx, versionID)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to retrieve files: %w", err)
|
||||
}
|
||||
|
||||
// Sort classifiers by length (longest first) once per call
|
||||
sort.SliceStable(classifiers, func(i, j int) bool {
|
||||
return len(classifiers[i]) > len(classifiers[j])
|
||||
})
|
||||
|
||||
var filteredFiles, skippedFiles []*PackageFile
|
||||
for _, file := range files {
|
||||
buildNumber, err := extractBuildNumberFromFileName(file.Name, classifiers...)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrMetadataFile) {
|
||||
skippedFiles = append(skippedFiles, file)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if buildNumber <= maxBuildNumber {
|
||||
filteredFiles = append(filteredFiles, file)
|
||||
}
|
||||
}
|
||||
|
||||
return filteredFiles, skippedFiles, nil
|
||||
}
|
||||
|
||||
// extractBuildNumberFromFileName extracts the build number from a Maven snapshot file name.
|
||||
// Expected formats:
|
||||
//
|
||||
// "artifact-1.0.0-20250311.083409-9.tgz" returns 9
|
||||
// "artifact-to-test-2.0.0-20250311.083409-10-sources.tgz" returns 10
|
||||
func extractBuildNumberFromFileName(filename string, classifiers ...string) (int, error) {
|
||||
if strings.Contains(filename, "maven-metadata.xml") {
|
||||
return 0, ErrMetadataFile
|
||||
}
|
||||
|
||||
dotIdx := strings.LastIndex(filename, ".")
|
||||
if dotIdx == -1 {
|
||||
return 0, fmt.Errorf("extract build number from filename: no file extension found in '%s'", filename)
|
||||
}
|
||||
base := filename[:dotIdx]
|
||||
|
||||
// Remove classifier suffix if present.
|
||||
for _, classifier := range classifiers {
|
||||
suffix := "-" + classifier
|
||||
if strings.HasSuffix(base, suffix) {
|
||||
base = base[:len(base)-len(suffix)]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// The build number should be the token after the last dash.
|
||||
lastDash := strings.LastIndex(base, "-")
|
||||
if lastDash == -1 {
|
||||
return 0, fmt.Errorf("extract build number from filename: invalid file name format in '%s'", filename)
|
||||
}
|
||||
buildNumberStr := base[lastDash+1:]
|
||||
buildNumber, err := strconv.Atoi(buildNumberStr)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("extract build number from filename: failed to convert build number '%s' to integer in '%s': %v", buildNumberStr, filename, err)
|
||||
}
|
||||
|
||||
return buildNumber, nil
|
||||
}
|
||||
|
||||
// CalculateFileSize sums up all blob sizes matching the search options.
|
||||
// It does NOT respect the deduplication of blobs.
|
||||
func CalculateFileSize(ctx context.Context, opts *PackageFileSearchOptions) (int64, error) {
|
||||
|
||||
@ -129,11 +129,16 @@ func getVersionByNameAndVersion(ctx context.Context, ownerID int64, packageType
|
||||
|
||||
// GetVersionsByPackageType gets all versions of a specific type
|
||||
func GetVersionsByPackageType(ctx context.Context, ownerID int64, packageType Type) ([]*PackageVersion, error) {
|
||||
pvs, _, err := SearchVersions(ctx, &PackageSearchOptions{
|
||||
OwnerID: ownerID,
|
||||
opts := &PackageSearchOptions{
|
||||
Type: packageType,
|
||||
IsInternal: optional.Some(false),
|
||||
})
|
||||
}
|
||||
|
||||
if ownerID != 0 {
|
||||
opts.OwnerID = ownerID
|
||||
}
|
||||
|
||||
pvs, _, err := SearchVersions(ctx, opts)
|
||||
return pvs, err
|
||||
}
|
||||
|
||||
|
||||
@ -5,7 +5,9 @@ package maven
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"io"
|
||||
"strconv"
|
||||
|
||||
"code.gitea.io/gitea/modules/util"
|
||||
"code.gitea.io/gitea/modules/validation"
|
||||
@ -31,6 +33,12 @@ type Dependency struct {
|
||||
Version string `json:"version,omitempty"`
|
||||
}
|
||||
|
||||
// SnapshotMetadata struct holds the build number and the list of classifiers for a snapshot version
|
||||
type SnapshotMetadata struct {
|
||||
BuildNumber int `json:"build_number,omitempty"`
|
||||
Classifiers []string `json:"classifiers,omitempty"`
|
||||
}
|
||||
|
||||
type pomStruct struct {
|
||||
XMLName xml.Name `xml:"project"`
|
||||
|
||||
@ -61,6 +69,26 @@ type pomStruct struct {
|
||||
} `xml:"dependencies>dependency"`
|
||||
}
|
||||
|
||||
type snapshotMetadataStruct struct {
|
||||
XMLName xml.Name `xml:"metadata"`
|
||||
GroupID string `xml:"groupId"`
|
||||
ArtifactID string `xml:"artifactId"`
|
||||
Version string `xml:"version"`
|
||||
Versioning struct {
|
||||
LastUpdated string `xml:"lastUpdated"`
|
||||
Snapshot struct {
|
||||
Timestamp string `xml:"timestamp"`
|
||||
BuildNumber string `xml:"buildNumber"`
|
||||
} `xml:"snapshot"`
|
||||
SnapshotVersions []struct {
|
||||
Extension string `xml:"extension"`
|
||||
Classifier string `xml:"classifier"`
|
||||
Value string `xml:"value"`
|
||||
Updated string `xml:"updated"`
|
||||
} `xml:"snapshotVersions>snapshotVersion"`
|
||||
} `xml:"versioning"`
|
||||
}
|
||||
|
||||
// ParsePackageMetaData parses the metadata of a pom file
|
||||
func ParsePackageMetaData(r io.Reader) (*Metadata, error) {
|
||||
var pom pomStruct
|
||||
@ -109,3 +137,31 @@ func ParsePackageMetaData(r io.Reader) (*Metadata, error) {
|
||||
Dependencies: dependencies,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ParseSnapshotVersionMetadata parses the Maven Snapshot Version metadata to extract the build number and list of available classifiers.
|
||||
func ParseSnapshotVersionMetaData(r io.Reader) (*SnapshotMetadata, error) {
|
||||
var metadata snapshotMetadataStruct
|
||||
|
||||
dec := xml.NewDecoder(r)
|
||||
dec.CharsetReader = charset.NewReaderLabel
|
||||
if err := dec.Decode(&metadata); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
buildNumber, err := strconv.Atoi(metadata.Versioning.Snapshot.BuildNumber)
|
||||
if err != nil {
|
||||
return nil, errors.New("invalid or missing build number in snapshot metadata")
|
||||
}
|
||||
|
||||
var classifiers []string
|
||||
for _, snapshotVersion := range metadata.Versioning.SnapshotVersions {
|
||||
if snapshotVersion.Classifier != "" {
|
||||
classifiers = append(classifiers, snapshotVersion.Classifier)
|
||||
}
|
||||
}
|
||||
|
||||
return &SnapshotMetadata{
|
||||
BuildNumber: buildNumber,
|
||||
Classifiers: classifiers,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@ -41,10 +41,13 @@ var (
|
||||
LimitSizeSwift int64
|
||||
LimitSizeVagrant int64
|
||||
|
||||
DefaultRPMSignEnabled bool
|
||||
DefaultRPMSignEnabled bool
|
||||
RetainMavenSnapshotBuilds int
|
||||
DebugMavenCleanup bool
|
||||
}{
|
||||
Enabled: true,
|
||||
LimitTotalOwnerCount: -1,
|
||||
Enabled: true,
|
||||
LimitTotalOwnerCount: -1,
|
||||
RetainMavenSnapshotBuilds: -1,
|
||||
}
|
||||
)
|
||||
|
||||
@ -88,6 +91,8 @@ func loadPackagesFrom(rootCfg ConfigProvider) (err error) {
|
||||
Packages.LimitSizeSwift = mustBytes(sec, "LIMIT_SIZE_SWIFT")
|
||||
Packages.LimitSizeVagrant = mustBytes(sec, "LIMIT_SIZE_VAGRANT")
|
||||
Packages.DefaultRPMSignEnabled = sec.Key("DEFAULT_RPM_SIGN_ENABLED").MustBool(false)
|
||||
Packages.RetainMavenSnapshotBuilds = sec.Key("RETAIN_MAVEN_SNAPSHOT_BUILDS").MustInt(Packages.RetainMavenSnapshotBuilds)
|
||||
Packages.DebugMavenCleanup = sec.Key("DEBUG_MAVEN_CLEANUP").MustBool(true)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
// Copyright 2022 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
package cleanup
|
||||
|
||||
import (
|
||||
"context"
|
||||
@ -20,6 +20,7 @@ import (
|
||||
cargo_service "code.gitea.io/gitea/services/packages/cargo"
|
||||
container_service "code.gitea.io/gitea/services/packages/container"
|
||||
debian_service "code.gitea.io/gitea/services/packages/debian"
|
||||
maven_service "code.gitea.io/gitea/services/packages/maven"
|
||||
rpm_service "code.gitea.io/gitea/services/packages/rpm"
|
||||
)
|
||||
|
||||
@ -171,6 +172,10 @@ func CleanupExpiredData(ctx context.Context, olderThan time.Duration) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := maven_service.CleanupSnapshotVersions(ctx); err != nil {
|
||||
log.Error("Error during maven snapshot versions cleanup: %v", err)
|
||||
}
|
||||
|
||||
ps, err := packages_model.FindUnreferencedPackages(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
157
services/packages/maven/cleanup.go
Normal file
157
services/packages/maven/cleanup.go
Normal file
@ -0,0 +1,157 @@
|
||||
// Copyright 2025 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package maven
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"code.gitea.io/gitea/models/packages"
|
||||
"code.gitea.io/gitea/modules/json"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/packages/maven"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
packages_service "code.gitea.io/gitea/services/packages"
|
||||
)
|
||||
|
||||
// CleanupSnapshotVersions removes outdated files for SNAPHOT versions for all Maven packages.
|
||||
func CleanupSnapshotVersions(ctx context.Context) error {
|
||||
retainBuilds := setting.Packages.RetainMavenSnapshotBuilds
|
||||
debugSession := setting.Packages.DebugMavenCleanup
|
||||
log.Debug("Maven Cleanup: starting with retainBuilds: %d, debugSession: %t", retainBuilds, debugSession)
|
||||
|
||||
if retainBuilds < 1 {
|
||||
log.Warn("Maven Cleanup: skipped as value for retainBuilds less than 1: %d. Minimum 1 build should be retained", retainBuilds)
|
||||
return nil
|
||||
}
|
||||
|
||||
versions, err := packages.GetVersionsByPackageType(ctx, 0, packages.TypeMaven)
|
||||
if err != nil {
|
||||
return fmt.Errorf("maven Cleanup: failed to retrieve Maven package versions: %w", err)
|
||||
}
|
||||
|
||||
var errs []error
|
||||
var metadataErrors []error
|
||||
|
||||
for _, version := range versions {
|
||||
if !isSnapshotVersion(version.Version) {
|
||||
continue
|
||||
}
|
||||
|
||||
var artifactID, groupID string
|
||||
if version.MetadataJSON != "" {
|
||||
var metadata map[string]any
|
||||
if err := json.Unmarshal([]byte(version.MetadataJSON), &metadata); err != nil {
|
||||
log.Warn("Maven Cleanup: error during cleanup: failed to unmarshal metadataJSON for package version ID: %d: %w", version.ID, err)
|
||||
} else {
|
||||
artifactID, _ = metadata["artifact_id"].(string)
|
||||
groupID, _ = metadata["group_id"].(string)
|
||||
log.Debug("Maven Cleanup: processing package version with ID: %s, Group ID: %s, Artifact ID: %s, Version: %s", version.ID, groupID, artifactID, version.Version)
|
||||
}
|
||||
}
|
||||
|
||||
if err := cleanSnapshotFiles(ctx, version.ID, retainBuilds, debugSession); err != nil {
|
||||
formattedErr := fmt.Errorf("version '%s' (ID: %d, Group ID: %s, Artifact ID: %s): %w",
|
||||
version.Version, version.ID, groupID, artifactID, err)
|
||||
|
||||
if errors.Is(err, packages.ErrMetadataFile) {
|
||||
metadataErrors = append(metadataErrors, formattedErr)
|
||||
} else {
|
||||
errs = append(errs, formattedErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, err := range metadataErrors {
|
||||
log.Warn("Maven Cleanup: error during cleanup: %v", err)
|
||||
}
|
||||
|
||||
if len(errs) > 0 {
|
||||
for _, err := range errs {
|
||||
log.Error("Maven Cleanup: error during cleanup: %v", err)
|
||||
}
|
||||
return fmt.Errorf("maven Cleanup: completed with errors: %v", errs)
|
||||
}
|
||||
|
||||
log.Debug("Completed Maven Cleanup")
|
||||
return nil
|
||||
}
|
||||
|
||||
func isSnapshotVersion(version string) bool {
|
||||
return strings.HasSuffix(version, "-SNAPSHOT")
|
||||
}
|
||||
|
||||
func cleanSnapshotFiles(ctx context.Context, versionID int64, retainBuilds int, debugSession bool) error {
|
||||
log.Debug("Maven Cleanup: starting cleanSnapshotFiles for versionID: %d with retainBuilds: %d, debugSession: %t", versionID, retainBuilds, debugSession)
|
||||
|
||||
metadataFile, err := packages.GetFileForVersionByName(ctx, versionID, "maven-metadata.xml", packages.EmptyFileKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: failed to retrieve maven-metadata.xml: %w", packages.ErrMetadataFile, err)
|
||||
}
|
||||
|
||||
maxBuildNumber, classifiers, err := extractMaxBuildNumber(ctx, metadataFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: failed to extract max build number from maven-metadata.xml: %w", packages.ErrMetadataFile, err)
|
||||
}
|
||||
|
||||
thresholdBuildNumber := maxBuildNumber - retainBuilds
|
||||
if thresholdBuildNumber <= 0 {
|
||||
log.Debug("Maven Cleanup: no files to clean up, as the threshold build number is less than or equal to zero for versionID %d", versionID)
|
||||
return nil
|
||||
}
|
||||
|
||||
filesToRemove, skippedFiles, err := packages.GetFilesBelowBuildNumber(ctx, versionID, thresholdBuildNumber, classifiers...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cleanSnapshotFiles: failed to retrieve files for version: %w", err)
|
||||
}
|
||||
|
||||
if debugSession {
|
||||
var fileNamesToRemove, skippedFileNames []string
|
||||
|
||||
for _, file := range filesToRemove {
|
||||
fileNamesToRemove = append(fileNamesToRemove, file.Name)
|
||||
}
|
||||
|
||||
for _, file := range skippedFiles {
|
||||
skippedFileNames = append(skippedFileNames, file.Name)
|
||||
}
|
||||
|
||||
log.Debug("Maven Cleanup: debug session active. Files to remove: %v, Skipped files: %v", fileNamesToRemove, skippedFileNames)
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, file := range filesToRemove {
|
||||
log.Debug("Maven Cleanup: removing file '%s' below threshold %d", file.Name, thresholdBuildNumber)
|
||||
if err := packages_service.DeletePackageFile(ctx, file); err != nil {
|
||||
return fmt.Errorf("cleanSnapshotFiles: failed to delete file '%s': %w", file.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func extractMaxBuildNumber(ctx context.Context, metadataFile *packages.PackageFile) (int, []string, error) {
|
||||
pb, err := packages.GetBlobByID(ctx, metadataFile.BlobID)
|
||||
if err != nil {
|
||||
return 0, nil, fmt.Errorf("failed to get package blob: %w", err)
|
||||
}
|
||||
|
||||
content, _, _, err := packages_service.OpenBlobForDownload(ctx, metadataFile, pb, "", nil, true)
|
||||
if err != nil {
|
||||
return 0, nil, fmt.Errorf("failed to get package file stream: %w", err)
|
||||
}
|
||||
defer content.Close()
|
||||
|
||||
snapshotMetadata, err := maven.ParseSnapshotVersionMetaData(content)
|
||||
if err != nil {
|
||||
return 0, nil, fmt.Errorf("failed to parse maven-metadata.xml: %w", err)
|
||||
}
|
||||
|
||||
buildNumber := snapshotMetadata.BuildNumber
|
||||
classifiers := snapshotMetadata.Classifiers
|
||||
|
||||
return buildNumber, classifiers, nil
|
||||
}
|
||||
250
services/packages/maven/cleanup_test.go
Normal file
250
services/packages/maven/cleanup_test.go
Normal file
@ -0,0 +1,250 @@
|
||||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package maven
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"code.gitea.io/gitea/models/packages"
|
||||
"code.gitea.io/gitea/models/unittest"
|
||||
packages_module "code.gitea.io/gitea/modules/packages"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
packages_service "code.gitea.io/gitea/services/packages"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
unittest.MainTest(m)
|
||||
}
|
||||
|
||||
func addMavenMetadataToPackageVersion(t *testing.T, pv *packages.PackageVersion) {
|
||||
// Create maven-metadata.xml content with build number 5 (matching the fixtures)
|
||||
// Maven metadata structure explanation:
|
||||
// - <snapshot>: Contains the latest snapshot timestamp and build number
|
||||
// - <snapshotVersions>: Lists all available files for each build number
|
||||
// - <extension>: File extension (jar, pom, etc.)
|
||||
// - <classifier>: Optional classifier (sources, javadoc, tests, etc.)
|
||||
// - <value>: The actual version string with timestamp and build number
|
||||
// - <updated>: Timestamp when the artifact was deployed
|
||||
metadataXML := `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<metadata>
|
||||
<groupId>com.gitea</groupId>
|
||||
<artifactId>test-project</artifactId>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<versioning>
|
||||
<snapshot>
|
||||
<timestamp>20230101.000000</timestamp>
|
||||
<buildNumber>5</buildNumber>
|
||||
</snapshot>
|
||||
<lastUpdated>20230101000000</lastUpdated>
|
||||
<snapshotVersions>
|
||||
<snapshotVersion>
|
||||
<extension>jar</extension>
|
||||
<value>1.0-20230101.000000-1</value>
|
||||
<updated>20230101000000</updated>
|
||||
</snapshotVersion>
|
||||
<snapshotVersion>
|
||||
<extension>jar</extension>
|
||||
<value>1.0-20230101.000000-2</value>
|
||||
<updated>20230101000000</updated>
|
||||
</snapshotVersion>
|
||||
<snapshotVersion>
|
||||
<extension>jar</extension>
|
||||
<value>1.0-20230101.000000-3</value>
|
||||
<updated>20230101000000</updated>
|
||||
</snapshotVersion>
|
||||
<snapshotVersion>
|
||||
<classifier>sources</classifier>
|
||||
<extension>jar</extension>
|
||||
<value>1.0-20230101.000000-3</value>
|
||||
<updated>20230101000000</updated>
|
||||
</snapshotVersion>
|
||||
<snapshotVersion>
|
||||
<classifier>javadoc</classifier>
|
||||
<extension>jar</extension>
|
||||
<value>1.0-20230101.000000-3</value>
|
||||
<updated>20230101000000</updated>
|
||||
</snapshotVersion>
|
||||
<snapshotVersion>
|
||||
<extension>jar</extension>
|
||||
<value>1.0-20230101.000000-4</value>
|
||||
<updated>20230101000000</updated>
|
||||
</snapshotVersion>
|
||||
<snapshotVersion>
|
||||
<classifier>sources</classifier>
|
||||
<extension>jar</extension>
|
||||
<value>1.0-20230101.000000-4</value>
|
||||
<updated>20230101000000</updated>
|
||||
</snapshotVersion>
|
||||
<snapshotVersion>
|
||||
<classifier>javadoc</classifier>
|
||||
<extension>jar</extension>
|
||||
<value>1.0-20230101.000000-4</value>
|
||||
<updated>20230101000000</updated>
|
||||
</snapshotVersion>
|
||||
<snapshotVersion>
|
||||
<extension>jar</extension>
|
||||
<value>1.0-20230101.000000-5</value>
|
||||
<updated>20230101000000</updated>
|
||||
</snapshotVersion>
|
||||
<snapshotVersion>
|
||||
<classifier>sources</classifier>
|
||||
<extension>jar</extension>
|
||||
<value>1.0-20230101.000000-5</value>
|
||||
<updated>20230101000000</updated>
|
||||
</snapshotVersion>
|
||||
<snapshotVersion>
|
||||
<classifier>javadoc</classifier>
|
||||
<extension>jar</extension>
|
||||
<value>1.0-20230101.000000-5</value>
|
||||
<updated>20230101000000</updated>
|
||||
</snapshotVersion>
|
||||
</snapshotVersions>
|
||||
</versioning>
|
||||
</metadata>`
|
||||
|
||||
// Add metadata file to the existing package version using service method
|
||||
metadataReader := bytes.NewReader([]byte(metadataXML))
|
||||
hsr, err := packages_module.CreateHashedBufferFromReader(metadataReader)
|
||||
assert.NoError(t, err)
|
||||
|
||||
pfci := &packages_service.PackageFileCreationInfo{
|
||||
PackageFileInfo: packages_service.PackageFileInfo{
|
||||
Filename: "maven-metadata.xml",
|
||||
},
|
||||
Data: hsr,
|
||||
}
|
||||
|
||||
_, err = packages_service.AddFileToPackageVersionInternal(t.Context(), pv, pfci)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestCleanupSnapshotVersions(t *testing.T) {
|
||||
assert.NoError(t, unittest.PrepareTestDatabase())
|
||||
|
||||
t.Run("Should skip when retainBuilds is negative", func(t *testing.T) {
|
||||
setting.Packages.RetainMavenSnapshotBuilds = -1
|
||||
setting.Packages.DebugMavenCleanup = false
|
||||
t.Logf("Test settings: retainBuilds=%d, debug=%t", setting.Packages.RetainMavenSnapshotBuilds, setting.Packages.DebugMavenCleanup)
|
||||
err := CleanupSnapshotVersions(t.Context())
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("Should skip when retainBuilds is zero", func(t *testing.T) {
|
||||
setting.Packages.RetainMavenSnapshotBuilds = 0
|
||||
setting.Packages.DebugMavenCleanup = false
|
||||
t.Logf("Test settings: retainBuilds=%d, debug=%t", setting.Packages.RetainMavenSnapshotBuilds, setting.Packages.DebugMavenCleanup)
|
||||
err := CleanupSnapshotVersions(t.Context())
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("Should handle missing metadata file gracefully", func(t *testing.T) {
|
||||
assert.NoError(t, unittest.PrepareTestDatabase())
|
||||
|
||||
setting.Packages.RetainMavenSnapshotBuilds = 2
|
||||
setting.Packages.DebugMavenCleanup = false
|
||||
|
||||
// Get the existing package version from fixtures (ID 1)
|
||||
pv, err := packages.GetVersionByID(t.Context(), 1)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify all 11 files exist before cleanup (5 base jars + 6 classifier jars)
|
||||
filesBefore, err := packages.GetFilesByVersionID(t.Context(), pv.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, filesBefore, 11) // 5 base jars + 6 classifier jars (sources + javadoc for builds 3,4,5)
|
||||
|
||||
// No metadata file exists in fixtures - should handle gracefully
|
||||
err = CleanupSnapshotVersions(t.Context())
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify all 11 files still exist after cleanup (no cleanup should occur without metadata)
|
||||
filesAfter, err := packages.GetFilesByVersionID(t.Context(), pv.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, filesAfter, 11, "All files should remain when metadata is missing")
|
||||
})
|
||||
|
||||
t.Run("Should work with debug mode", func(t *testing.T) {
|
||||
assert.NoError(t, unittest.PrepareTestDatabase())
|
||||
|
||||
setting.Packages.RetainMavenSnapshotBuilds = 2
|
||||
setting.Packages.DebugMavenCleanup = true
|
||||
|
||||
pv, err := packages.GetVersionByID(t.Context(), 1)
|
||||
assert.NoError(t, err)
|
||||
|
||||
addMavenMetadataToPackageVersion(t, pv)
|
||||
|
||||
filesBefore, err := packages.GetFilesByVersionID(t.Context(), pv.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, filesBefore, 12) // 11 jar files + 1 metadata file
|
||||
|
||||
err = CleanupSnapshotVersions(t.Context())
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify all files still exist after cleanup (debug mode should not delete anything)
|
||||
filesAfter, err := packages.GetFilesByVersionID(t.Context(), pv.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, filesAfter, 12, "All files should remain in debug mode")
|
||||
})
|
||||
|
||||
t.Run("Should test actual cleanup with metadata", func(t *testing.T) {
|
||||
assert.NoError(t, unittest.PrepareTestDatabase())
|
||||
|
||||
setting.Packages.DebugMavenCleanup = false
|
||||
setting.Packages.RetainMavenSnapshotBuilds = 2
|
||||
t.Logf("Test settings: retainBuilds=%d, debug=%t", setting.Packages.RetainMavenSnapshotBuilds, setting.Packages.DebugMavenCleanup)
|
||||
|
||||
// Get the existing package version from fixtures (ID 1)
|
||||
pv, err := packages.GetVersionByID(t.Context(), 1)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "1.0-SNAPSHOT", pv.Version)
|
||||
|
||||
addMavenMetadataToPackageVersion(t, pv)
|
||||
|
||||
filesBefore, err := packages.GetFilesByVersionID(t.Context(), pv.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, filesBefore, 12) // 11 jar files + 1 metadata file
|
||||
|
||||
// Test cleanup with retainBuilds = 2 (should keep builds 4 and 5, remove 1, 2, 3)
|
||||
// Build 4: base jar + sources + javadoc = 3 files
|
||||
// Build 5: base jar + sources + javadoc = 3 files
|
||||
// Total retained: 6 files + 1 metadata = 7 files
|
||||
err = CleanupSnapshotVersions(t.Context())
|
||||
assert.NoError(t, err)
|
||||
|
||||
filesAfter, err := packages.GetFilesByVersionID(t.Context(), pv.ID)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Should have metadata file + 6 retained build artifacts (2 builds × 3 files each)
|
||||
assert.Len(t, filesAfter, 7)
|
||||
|
||||
// Check that metadata file is still there
|
||||
var hasMetadata bool
|
||||
var retainedBuilds []string
|
||||
for _, file := range filesAfter {
|
||||
if file.Name == "maven-metadata.xml" {
|
||||
hasMetadata = true
|
||||
} else {
|
||||
retainedBuilds = append(retainedBuilds, file.Name)
|
||||
}
|
||||
}
|
||||
|
||||
assert.True(t, hasMetadata, "maven-metadata.xml should be retained")
|
||||
assert.Len(t, retainedBuilds, 6, "Should retain exactly 6 files (2 builds with 3 artifacts each)")
|
||||
|
||||
t.Logf("Retained builds: %v", retainedBuilds)
|
||||
|
||||
// Verify build 4 artifacts are retained
|
||||
assert.Contains(t, retainedBuilds, "gitea-test-1.0-20230101.000000-4.jar")
|
||||
assert.Contains(t, retainedBuilds, "gitea-test-1.0-20230101.000000-4-sources.jar")
|
||||
assert.Contains(t, retainedBuilds, "gitea-test-1.0-20230101.000000-4-javadoc.jar")
|
||||
|
||||
// Verify build 5 artifacts are retained
|
||||
assert.Contains(t, retainedBuilds, "gitea-test-1.0-20230101.000000-5.jar")
|
||||
assert.Contains(t, retainedBuilds, "gitea-test-1.0-20230101.000000-5-sources.jar")
|
||||
assert.Contains(t, retainedBuilds, "gitea-test-1.0-20230101.000000-5-javadoc.jar")
|
||||
})
|
||||
}
|
||||
@ -598,8 +598,8 @@ func OpenBlobStream(pb *packages_model.PackageBlob) (io.ReadSeekCloser, error) {
|
||||
}
|
||||
|
||||
// OpenBlobForDownload returns the content of the specific package blob and increases the download counter.
|
||||
// If the storage supports direct serving and it's enabled, only the direct serving url is returned.
|
||||
func OpenBlobForDownload(ctx context.Context, pf *packages_model.PackageFile, pb *packages_model.PackageBlob, method string, serveDirectReqParams url.Values) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) {
|
||||
// If the storage supports direct serving and it's enabled, only the direct serving url is returned; otherwise, forceInternalServe should be set to true.
|
||||
func OpenBlobForDownload(ctx context.Context, pf *packages_model.PackageFile, pb *packages_model.PackageBlob, method string, serveDirectReqParams url.Values, forceInternalServe ...bool) (io.ReadSeekCloser, *url.URL, *packages_model.PackageFile, error) {
|
||||
key := packages_module.BlobHash256Key(pb.HashSHA256)
|
||||
|
||||
cs := packages_module.NewContentStore()
|
||||
@ -608,7 +608,9 @@ func OpenBlobForDownload(ctx context.Context, pf *packages_model.PackageFile, pb
|
||||
var u *url.URL
|
||||
var err error
|
||||
|
||||
if cs.ShouldServeDirect() {
|
||||
internalServe := len(forceInternalServe) > 0 && forceInternalServe[0]
|
||||
|
||||
if !internalServe && cs.ShouldServeDirect() {
|
||||
u, err = cs.GetServeDirectURL(key, pf.Name, method, serveDirectReqParams)
|
||||
if err != nil && !errors.Is(err, storage.ErrURLNotSupported) {
|
||||
log.Error("Error getting serve direct url (fallback to local reader): %v", err)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user