0
0
mirror of https://github.com/go-gitea/gitea.git synced 2026-05-14 00:01:05 +02:00

Merge branch 'go-gitea:main' into main

This commit is contained in:
Karthik Bhandary 2026-03-07 16:35:03 +05:30 committed by GitHub
commit 988ec17d9d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 120 additions and 71 deletions

View File

@ -75,7 +75,7 @@ func (f *file) readAt(fileMeta *dbfsMeta, offset int64, p []byte) (n int, err er
}
func (f *file) Read(p []byte) (n int, err error) {
if f.metaID == 0 || !f.allowRead {
if !f.allowRead {
return 0, os.ErrInvalid
}
@ -89,7 +89,7 @@ func (f *file) Read(p []byte) (n int, err error) {
}
func (f *file) Write(p []byte) (n int, err error) {
if f.metaID == 0 || !f.allowWrite {
if !f.allowWrite {
return 0, os.ErrInvalid
}
@ -184,10 +184,6 @@ func (f *file) Close() error {
}
func (f *file) Stat() (os.FileInfo, error) {
if f.metaID == 0 {
return nil, os.ErrInvalid
}
fileMeta, err := findFileMetaByID(f.ctx, f.metaID)
if err != nil {
return nil, err
@ -232,15 +228,17 @@ func (f *file) open(flag int) (err error) {
if f.metaID != 0 {
return os.ErrExist
}
} else {
// create a new file if none exists.
if f.metaID == 0 {
if err = f.createEmpty(); err != nil {
return err
}
}
// create a new file if not exists.
if f.metaID == 0 {
if err = f.createEmpty(); err != nil {
return err
}
}
}
if f.metaID == 0 {
return os.ErrNotExist
}
if flag&os.O_TRUNC != 0 {
if err = f.truncate(); err != nil {
return err
@ -252,7 +250,7 @@ func (f *file) open(flag int) (err error) {
}
}
return nil
}
} // end if: allowWrite
// read only mode
if f.metaID == 0 {
@ -322,9 +320,6 @@ func (f *file) delete() error {
}
func (f *file) size() (int64, error) {
if f.metaID == 0 {
return 0, os.ErrNotExist
}
fileMeta, err := findFileMetaByID(f.ctx, f.metaID)
if err != nil {
return 0, err
@ -339,7 +334,7 @@ func findFileMetaByID(ctx context.Context, metaID int64) (*dbfsMeta, error) {
} else if ok {
return &fileMeta, nil
}
return nil, nil //nolint:nilnil // return nil to indicate that the object does not exist
return nil, os.ErrNotExist
}
func buildPath(path string) string {

View File

@ -40,6 +40,9 @@ The DBFS solution:
* In the future, when Gitea action needs to limit the log size (other CI/CD services also do so), it's easier to calculate the log file size.
* Even sometimes the UI needs to render the tailing lines, the tailing lines can be found be counting the "\n" from the end of the file by seek.
The seeking and finding is not the fastest way, but it's still acceptable and won't affect the performance too much.
Limitations of the DBFS solution:
* Not fully POSIX-compliant, some behaviors may be different from the real filesystem, especially for concurrent read/write
*/
type dbfsMeta struct {

View File

@ -9,19 +9,14 @@ import (
"os"
"testing"
"code.gitea.io/gitea/modules/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func changeDefaultFileBlockSize(n int64) (restore func()) {
old := defaultFileBlockSize
defaultFileBlockSize = n
return func() {
defaultFileBlockSize = old
}
}
func TestDbfsBasic(t *testing.T) {
defer changeDefaultFileBlockSize(4)()
defer test.MockVariableValue(&defaultFileBlockSize, 4)()
// test basic write/read
f, err := OpenFile(t.Context(), "test.txt", os.O_RDWR|os.O_CREATE)
@ -122,10 +117,55 @@ func TestDbfsBasic(t *testing.T) {
stat, err = f.Stat()
assert.NoError(t, err)
assert.EqualValues(t, 10, stat.Size())
t.Run("NonExisting", func(t *testing.T) {
f, err := OpenFile(t.Context(), "non-existing.txt", os.O_RDONLY)
assert.ErrorIs(t, err, os.ErrNotExist)
assert.Nil(t, f)
f, err = OpenFile(t.Context(), "non-existing.txt", os.O_WRONLY)
assert.ErrorIs(t, err, os.ErrNotExist)
assert.Nil(t, f)
f, err = OpenFile(t.Context(), "non-existing.txt", os.O_WRONLY|os.O_APPEND|os.O_TRUNC)
assert.ErrorIs(t, err, os.ErrNotExist)
assert.Nil(t, f)
})
t.Run("Existing", func(t *testing.T) {
assertFileContent := func(f File, expected string) {
_, err := f.Seek(0, io.SeekStart)
require.NoError(t, err)
buf, err := io.ReadAll(f)
require.NoError(t, err)
assert.Equal(t, expected, string(buf))
}
f, err := OpenFile(t.Context(), "existing.txt", os.O_RDWR|os.O_CREATE)
require.NoError(t, err)
_, _ = f.Write([]byte("test"))
assertFileContent(f, "test")
assert.NoError(t, f.Close())
f, err = OpenFile(t.Context(), "existing.txt", os.O_RDWR|os.O_CREATE|os.O_APPEND)
require.NoError(t, err)
_, _ = f.Write([]byte("\nnew"))
assertFileContent(f, "test\nnew")
assert.NoError(t, f.Close())
f, err = OpenFile(t.Context(), "existing.txt", os.O_RDWR|os.O_TRUNC)
require.NoError(t, err)
assertFileContent(f, "")
assert.NoError(t, f.Close())
f, err = OpenFile(t.Context(), "existing.txt", os.O_RDWR|os.O_CREATE|os.O_EXCL)
assert.ErrorIs(t, err, os.ErrExist)
assert.Nil(t, f)
})
}
func TestDbfsReadWrite(t *testing.T) {
defer changeDefaultFileBlockSize(4)()
defer test.MockVariableValue(&defaultFileBlockSize, 4)()
f1, err := OpenFile(t.Context(), "test.log", os.O_RDWR|os.O_CREATE)
assert.NoError(t, err)
@ -157,30 +197,32 @@ func TestDbfsReadWrite(t *testing.T) {
}
func TestDbfsSeekWrite(t *testing.T) {
defer changeDefaultFileBlockSize(4)()
defer test.MockVariableValue(&defaultFileBlockSize, 4)()
f, err := OpenFile(t.Context(), "test2.log", os.O_RDWR|os.O_CREATE)
assert.NoError(t, err)
defer f.Close()
// write something
fw, err := OpenFile(t.Context(), "test2.log", os.O_RDWR|os.O_CREATE)
require.NoError(t, err)
defer fw.Close()
n, err := f.Write([]byte("111"))
n, err := fw.Write([]byte("111"))
assert.NoError(t, err)
_, err = f.Seek(int64(n), io.SeekStart)
_, err = fw.Seek(int64(n), io.SeekStart)
assert.NoError(t, err)
_, err = f.Write([]byte("222"))
_, err = fw.Write([]byte("222"))
assert.NoError(t, err)
_, err = f.Seek(int64(n), io.SeekStart)
_, err = fw.Seek(int64(n), io.SeekStart)
assert.NoError(t, err)
_, err = f.Write([]byte("333"))
_, err = fw.Write([]byte("333"))
assert.NoError(t, err)
// then read it
fr, err := OpenFile(t.Context(), "test2.log", os.O_RDONLY)
assert.NoError(t, err)
defer f.Close()
require.NoError(t, err)
defer fr.Close()
buf, err := io.ReadAll(fr)
assert.NoError(t, err)

View File

@ -33,21 +33,22 @@ const (
// It doesn't respect the file format in the filename like ".zst", since it's difficult to reopen a closed compressed file and append new content.
// Why doesn't it store logs in object storage directly? Because it's not efficient to append content to object storage.
func WriteLogs(ctx context.Context, filename string, offset int64, rows []*runnerv1.LogRow) ([]int, error) {
flag := os.O_WRONLY
flag, openFileFor := os.O_WRONLY, "write-only"
if offset == 0 {
// Create file only if offset is 0, or it could result in content holes if the file doesn't exist.
flag |= os.O_CREATE
// Only allow to create file if offset is 0 (the first write), see #25560.
// Otherwise, it might result in content holes if the file has been deleted after transferred (actions.TransferLogs).
flag, openFileFor = os.O_WRONLY|os.O_CREATE, "write-create"
}
name := DBFSPrefix + filename
f, err := dbfs.OpenFile(ctx, name, flag)
if err != nil {
return nil, fmt.Errorf("dbfs OpenFile %q: %w", name, err)
return nil, fmt.Errorf("dbfs.OpenFile %q for %s: %w", name, openFileFor, err)
}
defer f.Close()
stat, err := f.Stat()
if err != nil {
return nil, fmt.Errorf("dbfs Stat %q: %w", name, err)
return nil, fmt.Errorf("dbfs.Stat %q: %w", name, err)
}
if stat.Size() < offset {
// If the size is less than offset, refuse to write, or it could result in content holes.
@ -56,7 +57,7 @@ func WriteLogs(ctx context.Context, filename string, offset int64, rows []*runne
}
if _, err := f.Seek(offset, io.SeekStart); err != nil {
return nil, fmt.Errorf("dbfs Seek %q: %w", name, err)
return nil, fmt.Errorf("dbfs.Seek %q: %w", name, err)
}
writer := bufio.NewWriterSize(f, defaultBufSize)
@ -121,16 +122,17 @@ const (
// TransferLogs transfers logs from DBFS to object storage.
// It happens when the file is complete and no more logs will be appended.
// It respects the file format in the filename like ".zst", and compresses the content if needed.
// The task log file must be marked as "log_in_storage=true" after the transfer.
func TransferLogs(ctx context.Context, filename string) (func(), error) {
name := DBFSPrefix + filename
remove := func() {
if err := dbfs.Remove(ctx, name); err != nil {
log.Warn("dbfs remove %q: %v", name, err)
log.Warn("dbfs.Remove %q: %v", name, err)
}
}
f, err := dbfs.Open(ctx, name)
if err != nil {
return nil, fmt.Errorf("dbfs open %q: %w", name, err)
return nil, fmt.Errorf("dbfs.Open %q: %w", name, err)
}
defer f.Close()
@ -164,7 +166,7 @@ func RemoveLogs(ctx context.Context, inStorage bool, filename string) error {
name := DBFSPrefix + filename
err := dbfs.Remove(ctx, name)
if err != nil {
return fmt.Errorf("dbfs remove %q: %w", name, err)
return fmt.Errorf("dbfs.Remove %q: %w", name, err)
}
return nil
}
@ -180,7 +182,7 @@ func OpenLogs(ctx context.Context, inStorage bool, filename string) (io.ReadSeek
name := DBFSPrefix + filename
f, err := dbfs.Open(ctx, name)
if err != nil {
return nil, fmt.Errorf("dbfs open %q: %w", name, err)
return nil, fmt.Errorf("dbfs.Open %q: %w", name, err)
}
return f, nil
}

View File

@ -84,6 +84,7 @@
"save": "Enregistrer",
"add": "Ajouter",
"add_all": "Tout Ajouter",
"dismiss": "Fermer",
"remove": "Retirer",
"remove_all": "Tout Retirer",
"remove_label_str": "Supprimer lélément « %s »",
@ -284,12 +285,6 @@
"install.register_confirm": "Exiger la confirmation du courriel lors de linscription",
"install.mail_notify": "Activer les notifications par courriel",
"install.server_service_title": "Paramètres Serveur et Tierce Parties",
"install.offline_mode": "Activer le mode hors-ligne",
"install.offline_mode_popup": "Désactiver l'utilisation de CDNs, et servir toutes les ressources localement.",
"install.disable_gravatar": "Désactiver Gravatar",
"install.disable_gravatar_popup": "Désactiver Gravatar et les autres sources d'avatars tierces. Un avatar par défaut sera utilisé pour les utilisateurs n'ayant pas téléversé un avatar personnalisé.",
"install.federated_avatar_lookup": "Activer les avatars unifiés",
"install.federated_avatar_lookup_popup": "Activer la recherche unifiée d'avatars en utilisant le service open source unifié basé sur libravatar.",
"install.disable_registration": "Désactiver le formulaire d'inscription",
"install.disable_registration_popup": "Désactiver les nouvelles inscriptions. Seuls les administrateurs pourront créer de nouveaux comptes utilisateurs.",
"install.allow_only_external_registration_popup": "N'autoriser l'inscription qu'à partir des services externes",
@ -871,7 +866,7 @@
"settings.permissions_list": "Autorisations :",
"settings.manage_oauth2_applications": "Gérer les applications OAuth2",
"settings.edit_oauth2_application": "Modifier l'application OAuth2",
"settings.oauth2_applications_desc": "Les applications OAuth2 permettent à votre application tierce d'authentifier en toute sécurité les utilisateurs de cette instance Gitea.",
"settings.oauth2_applications_desc": "OAuth2 permet a une application tierce dauthentifier les utilisateurs de cette instance Gitea.",
"settings.remove_oauth2_application": "Supprimer l'application OAuth2",
"settings.remove_oauth2_application_desc": "La suppression d'une application OAuth2 révoquera l'accès à tous les jetons d'accès signés. Continuer ?",
"settings.remove_oauth2_application_success": "L'application a été supprimée.",
@ -890,7 +885,7 @@
"settings.oauth2_regenerate_secret_hint": "Avez-vous perdu votre secret ?",
"settings.oauth2_client_secret_hint": "Le secret ne sera plus affiché après avoir quitté ou actualisé cette page. Veuillez vous assurer que vous l'avez enregistré.",
"settings.oauth2_application_edit": "Éditer",
"settings.oauth2_application_create_description": "Les applications OAuth2 permettent à votre application tierce d'accéder aux comptes d'utilisateurs de cette instance.",
"settings.oauth2_application_create_description": "OAuth2 permet à des applications tierces daccéder aux comptes utilisateurs de cette instance.",
"settings.oauth2_application_remove_description": "La suppression d'une application OAuth2 l'empêchera d'accéder aux comptes d'utilisateurs autorisés sur cette instance. Poursuivre ?",
"settings.oauth2_application_locked": "Gitea préinstalle des applications OAuth2 au démarrage si elles sont activées dans la configuration. Pour éviter des comportements inattendus, celles-ci ne peuvent être éditées ni supprimées. Veuillez vous référer à la documentation OAuth2 pour plus d'informations.",
"settings.authorized_oauth2_applications": "Applications OAuth2 autorisées",
@ -1524,6 +1519,7 @@
"repo.issues.commented_at": "a commenté <a href=\"#%s\"> %s</a>.",
"repo.issues.delete_comment_confirm": "Êtes-vous certain de vouloir supprimer ce commentaire?",
"repo.issues.context.copy_link": "Copier le lien",
"repo.issues.context.copy_source": "Copier la source",
"repo.issues.context.quote_reply": "Citer et répondre",
"repo.issues.context.reference_issue": "Référencer dans un nouveau ticket",
"repo.issues.context.edit": "Éditer",
@ -3192,7 +3188,6 @@
"admin.config.custom_conf": "Chemin du fichier de configuration",
"admin.config.custom_file_root_path": "Emplacement personnalisé du fichier racine",
"admin.config.domain": "Domaine du serveur",
"admin.config.offline_mode": "Mode hors-ligne",
"admin.config.disable_router_log": "Désactiver la Journalisation du Routeur",
"admin.config.run_user": "Exécuter avec l'utilisateur",
"admin.config.run_mode": "Mode d'Éxécution",
@ -3278,6 +3273,13 @@
"admin.config.cache_test_failed": "Impossible dinterroger le cache : %v.",
"admin.config.cache_test_slow": "Test du cache réussi, mais la réponse est lente : %s.",
"admin.config.cache_test_succeeded": "Test du cache réussi, réponse obtenue en %s.",
"admin.config.common.start_time": "Heure de début",
"admin.config.common.end_time": "Heure de fin",
"admin.config.common.skip_time_check": "Laisser le temps vide (effacer le champ) pour passer la vérification",
"admin.config.instance_maintenance": "Maintenance de linstance",
"admin.config.instance_maintenance_mode.admin_web_access_only": "Permettre uniquement aux administrateurs daccéder à linterface web",
"admin.config.instance_web_banner.enabled": "Afficher la bannière",
"admin.config.instance_web_banner.message_placeholder": "Message de bannière (supporte markdown)",
"admin.config.session_config": "Configuration de session",
"admin.config.session_provider": "Fournisseur de session",
"admin.config.provider_config": "Configuration du fournisseur",
@ -3288,7 +3290,7 @@
"admin.config.cookie_life_time": "Expiration du cookie",
"admin.config.picture_config": "Configuration de l'avatar",
"admin.config.picture_service": "Service d'Imagerie",
"admin.config.disable_gravatar": "Désactiver Gravatar",
"admin.config.enable_gravatar": "Activer Gravatar",
"admin.config.enable_federated_avatar": "Activer les avatars unifiés",
"admin.config.open_with_editor_app_help": "Les éditeurs disponibles via « Ouvrir avec ». Si laissé vide, la valeur par défaut sera utilisée. Développez pour voir la valeur par défaut.",
"admin.config.git_guide_remote_name": "Nom du dépôt distant pour les commandes git dans le guide",
@ -3672,6 +3674,8 @@
"actions.runners.reset_registration_token_confirm": "Voulez-vous révoquer le jeton actuel et en générer un nouveau ?",
"actions.runners.reset_registration_token_success": "Le jeton dinscription de lexécuteur a été réinitialisé avec succès",
"actions.runs.all_workflows": "Tous les flux de travail",
"actions.runs.workflow_run_count_1": "%d exécution du workflow",
"actions.runs.workflow_run_count_n": "%d exécutions du workflow",
"actions.runs.commit": "Révision",
"actions.runs.scheduled": "Planifié",
"actions.runs.pushed_by": "soumis par",

View File

@ -1399,17 +1399,17 @@
"repo.issues.new.clear_labels": "清除选中标签",
"repo.issues.new.projects": "项目",
"repo.issues.new.clear_projects": "清除项目",
"repo.issues.new.no_projects": "暂无项目",
"repo.issues.new.no_projects": "未选择项目",
"repo.issues.new.open_projects": "开启中的项目",
"repo.issues.new.closed_projects": "已关闭的项目",
"repo.issues.new.no_items": "无可选项",
"repo.issues.new.milestone": "里程碑",
"repo.issues.new.no_milestone": "未选择里程碑",
"repo.issues.new.clear_milestone": "取消选中里程碑",
"repo.issues.new.assignees": "指派成员",
"repo.issues.new.clear_assignees": "取消指派成员",
"repo.issues.new.no_assignees": "未指派员",
"repo.issues.new.no_reviewers": "评审人",
"repo.issues.new.assignees": "指派",
"repo.issues.new.clear_assignees": "取消指派",
"repo.issues.new.no_assignees": "未指派员",
"repo.issues.new.no_reviewers": "未指定评审人",
"repo.issues.new.blocked_user": "无法创建工单,因为您已被仓库所有者屏蔽。",
"repo.issues.edit.already_changed": "无法保存对工单的更改。其内容似乎已被其他用户更改。请刷新页面并重新编辑以避免覆盖他们的更改。",
"repo.issues.edit.blocked_user": "无法编辑内容,因为您已被仓库所有者或工单创建者屏蔽。",
@ -1465,9 +1465,9 @@
"repo.issues.filter_milestone_closed": "已关闭的里程碑",
"repo.issues.filter_project": "项目",
"repo.issues.filter_project_all": "所有项目",
"repo.issues.filter_project_none": "未加项目",
"repo.issues.filter_project_none": "项目",
"repo.issues.filter_assignee": "指派人筛选",
"repo.issues.filter_assignee_no_assignee": "未指派任何人",
"repo.issues.filter_assignee_no_assignee": "未指派任何人",
"repo.issues.filter_assignee_any_assignee": "已有指派",
"repo.issues.filter_poster": "作者",
"repo.issues.filter_user_placeholder": "搜索用户",
@ -1487,8 +1487,8 @@
"repo.issues.filter_sort.leastupdate": "最早更新",
"repo.issues.filter_sort.mostcomment": "最多评论",
"repo.issues.filter_sort.leastcomment": "最少评论",
"repo.issues.filter_sort.nearduedate": "到期日从近到远",
"repo.issues.filter_sort.farduedate": "到期日从远到近",
"repo.issues.filter_sort.nearduedate": "截止日期从近到远",
"repo.issues.filter_sort.farduedate": "截止日期从远到近",
"repo.issues.filter_sort.moststars": "点赞由多到少",
"repo.issues.filter_sort.feweststars": "点赞由少到多",
"repo.issues.filter_sort.mostforks": "派生由多到少",
@ -1519,6 +1519,7 @@
"repo.issues.commented_at": "评论于 <a href=\"#%s\">%s</a>",
"repo.issues.delete_comment_confirm": "您确定要删除该条评论吗?",
"repo.issues.context.copy_link": "复制链接",
"repo.issues.context.copy_source": "复制原文",
"repo.issues.context.quote_reply": "引用回复",
"repo.issues.context.reference_issue": "在新工单中引用",
"repo.issues.context.edit": "编辑",
@ -1927,8 +1928,8 @@
"repo.milestones.deletion_desc": "删除该里程碑将会移除所有工单中相关的信息。是否继续?",
"repo.milestones.deletion_success": "里程碑已删除。",
"repo.milestones.filter_sort.name": "名称",
"repo.milestones.filter_sort.earliest_due_data": "到期日从远到近",
"repo.milestones.filter_sort.latest_due_date": "到期日从近到远",
"repo.milestones.filter_sort.earliest_due_data": "截止日期从远到近",
"repo.milestones.filter_sort.latest_due_date": "截止日期从近到远",
"repo.milestones.filter_sort.least_complete": "完成度从低到高",
"repo.milestones.filter_sort.most_complete": "完成度从高到低",
"repo.milestones.filter_sort.most_issues": "工单从多到少",
@ -2011,7 +2012,7 @@
"repo.activity.title.issues_closed_from": "%[2]s 关闭了 %[1]s",
"repo.activity.title.issues_created_by": "%[2]s 创建了 %[1]s",
"repo.activity.closed_issue_label": "已关闭",
"repo.activity.new_issues_count_1": "开启的工单",
"repo.activity.new_issues_count_1": "开启的工单",
"repo.activity.new_issues_count_n": "已打开的工单",
"repo.activity.new_issue_label": "打开的",
"repo.activity.title.unresolved_conv_1": "%d 未解决的会话",
@ -3673,6 +3674,8 @@
"actions.runners.reset_registration_token_confirm": "是否吊销当前令牌并生成一个新令牌?",
"actions.runners.reset_registration_token_success": "成功重置运行器注册令牌",
"actions.runs.all_workflows": "所有工作流",
"actions.runs.workflow_run_count_1": "%d 次工作流运行",
"actions.runs.workflow_run_count_n": "%d 次工作流运行",
"actions.runs.commit": "提交",
"actions.runs.scheduled": "已计划的",
"actions.runs.pushed_by": "推送者",

View File

@ -270,7 +270,7 @@ func (s *Service) UpdateLog(
rows := req.Msg.Rows[ack-req.Msg.Index:]
ns, err := actions.WriteLogs(ctx, task.LogFilename, task.LogSize, rows)
if err != nil {
return nil, status.Errorf(codes.Internal, "write logs: %v", err)
return nil, status.Errorf(codes.Internal, "unable to append logs to dbfs file: %v", err)
}
task.LogLength += int64(len(rows))
for _, n := range ns {