diff --git a/models/repo/repo_indexer.go b/models/repo/repo_indexer.go index 6e19d8f937..c468fe3053 100644 --- a/models/repo/repo_indexer.go +++ b/models/repo/repo_indexer.go @@ -97,7 +97,7 @@ func UpdateIndexerStatus(ctx context.Context, repo *Repository, indexerType Repo return fmt.Errorf("UpdateIndexerStatus: Unable to getIndexerStatus for repo: %s Error: %w", repo.FullName(), err) } - if len(status.CommitSha) == 0 { + if status.ID == 0 { status.CommitSha = sha if err := db.Insert(ctx, status); err != nil { return fmt.Errorf("UpdateIndexerStatus: Unable to insert repoIndexerStatus for repo: %s Sha: %s Error: %w", repo.FullName(), sha, err) diff --git a/modules/indexer/code/bleve/bleve.go b/modules/indexer/code/bleve/bleve.go index 010ee39660..c1cfbe95c3 100644 --- a/modules/indexer/code/bleve/bleve.go +++ b/modules/indexer/code/bleve/bleve.go @@ -54,6 +54,7 @@ func addUnicodeNormalizeTokenFilter(m *mapping.IndexMappingImpl) error { // RepoIndexerData data stored in the repo indexer type RepoIndexerData struct { RepoID int64 + Archived bool CommitID string Content string Filename string @@ -71,7 +72,7 @@ const ( filenameIndexerAnalyzer = "filenameIndexerAnalyzer" filenameIndexerTokenizer = "filenameIndexerTokenizer" repoIndexerDocType = "repoIndexerDocType" - repoIndexerLatestVersion = 9 + repoIndexerLatestVersion = 10 ) // generateBleveIndexMapping generates a bleve index mapping for the repo indexer @@ -81,6 +82,10 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) { numericFieldMapping.IncludeInAll = false docMapping.AddFieldMappingsAt("RepoID", numericFieldMapping) + boolFieldMapping := bleve.NewBooleanFieldMapping() + boolFieldMapping.IncludeInAll = false + docMapping.AddFieldMappingsAt("Archived", boolFieldMapping) + textFieldMapping := bleve.NewTextFieldMapping() textFieldMapping.IncludeInAll = false docMapping.AddFieldMappingsAt("Content", textFieldMapping) @@ -195,6 +200,7 @@ func (b *Indexer) addUpdate(ctx context.Context, catFileBatch git.CatFileBatch, id := internal.FilenameIndexerID(repo.ID, update.Filename) return batch.Index(id, &RepoIndexerData{ RepoID: repo.ID, + Archived: repo.IsArchived, CommitID: commitSha, Filename: update.Filename, Content: string(charset.ToUTF8DropErrors(fileContents)), @@ -298,6 +304,13 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int indexerQuery = keywordQuery } + if opts.Archived.Has() { + indexerQuery = bleve.NewConjunctionQuery( + indexerQuery, + inner_bleve.BoolFieldQuery(opts.Archived.Value(), "Archived"), + ) + } + // Save for reuse without language filter facetQuery := indexerQuery if len(opts.Language) > 0 { diff --git a/modules/indexer/code/elasticsearch/elasticsearch.go b/modules/indexer/code/elasticsearch/elasticsearch.go index 9d170528ad..250a9b5e24 100644 --- a/modules/indexer/code/elasticsearch/elasticsearch.go +++ b/modules/indexer/code/elasticsearch/elasticsearch.go @@ -32,7 +32,7 @@ import ( ) const ( - esRepoIndexerLatestVersion = 3 + esRepoIndexerLatestVersion = 4 // multi-match-types, currently only 2 types are used // Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types esMultiMatchTypeBestFields = "best_fields" @@ -100,6 +100,10 @@ const ( "type": "long", "index": true }, + "archived": { + "type": "boolean", + "index": true + }, "filename": { "type": "text", "term_vector": "with_positions_offsets", @@ -185,6 +189,7 @@ func (b *Indexer) addUpdate(ctx context.Context, catFileBatch git.CatFileBatch, Id(id). Doc(map[string]any{ "repo_id": repo.ID, + "archived": repo.IsArchived, "filename": update.Filename, "content": string(charset.ToUTF8DropErrors(fileContents)), "commit_id": sha, @@ -377,6 +382,9 @@ func (b *Indexer) Search(ctx context.Context, opts *internal.SearchOptions) (int repoQuery := elastic.NewTermsQuery("repo_id", repoStrs...) query = query.Must(repoQuery) } + if opts.Archived.Has() { + query = query.Must(elastic.NewTermQuery("archived", opts.Archived.Value())) + } var ( start, pageSize = opts.GetSkipTake() diff --git a/modules/indexer/code/indexer_test.go b/modules/indexer/code/indexer_test.go index a884ab733a..ff43a22d0d 100644 --- a/modules/indexer/code/indexer_test.go +++ b/modules/indexer/code/indexer_test.go @@ -10,11 +10,13 @@ import ( "testing" "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" indexer_module "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/indexer/code/bleve" "code.gitea.io/gitea/modules/indexer/code/elasticsearch" "code.gitea.io/gitea/modules/indexer/code/internal" + "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/test" "code.gitea.io/gitea/modules/util" @@ -280,6 +282,110 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { assert.NoError(t, tearDownRepositoryIndexes(t.Context(), indexer)) }) + + t.Run(name+"_archived_filter", func(t *testing.T) { + assert.NoError(t, setupRepositoryIndexes(t.Context(), indexer)) + t.Cleanup(func() { + assert.NoError(t, tearDownRepositoryIndexes(context.Background(), indexer)) + }) + + repo1, err := repo_model.GetRepositoryByID(t.Context(), 1) + require.NoError(t, err) + repo62, err := repo_model.GetRepositoryByID(t.Context(), 62) + require.NoError(t, err) + + originalRepo1Archived := repo1.IsArchived + originalRepo62Archived := repo62.IsArchived + t.Cleanup(func() { + ctx := context.Background() + require.NoError(t, repo_model.SetArchiveRepoState(ctx, repo1, originalRepo1Archived)) + require.NoError(t, repo_model.SetArchiveRepoState(ctx, repo62, originalRepo62Archived)) + require.NoError(t, repo_model.UpdateIndexerStatus(ctx, repo1, repo_model.RepoIndexerTypeCode, "")) + require.NoError(t, repo_model.UpdateIndexerStatus(ctx, repo62, repo_model.RepoIndexerTypeCode, "")) + require.NoError(t, index(ctx, indexer, repo1.ID)) + require.NoError(t, index(ctx, indexer, repo62.ID)) + }) + + require.NoError(t, repo_model.SetArchiveRepoState(t.Context(), repo1, false)) + require.NoError(t, repo_model.SetArchiveRepoState(t.Context(), repo62, true)) + require.NoError(t, repo_model.UpdateIndexerStatus(t.Context(), repo1, repo_model.RepoIndexerTypeCode, "")) + require.NoError(t, repo_model.UpdateIndexerStatus(t.Context(), repo62, repo_model.RepoIndexerTypeCode, "")) + require.NoError(t, index(t.Context(), indexer, repo1.ID)) + require.NoError(t, index(t.Context(), indexer, repo62.ID)) + + testCases := []struct { + name string + keyword string + archived optional.Option[bool] + total int64 + filenames []string + }{ + { + name: "exclude_archived_repo_results", + keyword: "cucumber", + archived: optional.Some(false), + total: 0, + filenames: []string{}, + }, + { + name: "include_archived_repo_results", + keyword: "cucumber", + archived: optional.None[bool](), + total: 2, + filenames: []string{"cucumber.md", "avocado.md"}, + }, + { + name: "only_archived_repo_results", + keyword: "cucumber", + archived: optional.Some(true), + total: 2, + filenames: []string{"cucumber.md", "avocado.md"}, + }, + { + name: "exclude_keeps_non_archived_repo_results", + keyword: "Description", + archived: optional.Some(false), + total: 1, + filenames: []string{"README.md"}, + }, + { + name: "include_keeps_non_archived_repo_results", + keyword: "Description", + archived: optional.None[bool](), + total: 1, + filenames: []string{"README.md"}, + }, + { + name: "only_excludes_non_archived_repo_results", + keyword: "Description", + archived: optional.Some(true), + total: 0, + filenames: []string{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + total, res, _, err := indexer.Search(t.Context(), &internal.SearchOptions{ + Keyword: tc.keyword, + SearchMode: indexer_module.SearchModeWords, + Archived: tc.archived, + Paginator: &db.ListOptions{ + Page: 1, + PageSize: 10, + }, + }) + require.NoError(t, err) + assert.Equal(t, tc.total, total) + + filenames := make([]string, 0, len(res)) + for _, hit := range res { + filenames = append(filenames, hit.Filename) + } + assert.Equal(t, tc.filenames, filenames) + }) + } + }) } func TestBleveIndexAndSearch(t *testing.T) { @@ -332,6 +438,13 @@ func tearDownRepositoryIndexes(ctx context.Context, indexer internal.Indexer) er if err := indexer.Delete(ctx, repoID); err != nil { return err } + repo, err := repo_model.GetRepositoryByID(ctx, repoID) + if err != nil { + return err + } + if err := repo_model.UpdateIndexerStatus(ctx, repo, repo_model.RepoIndexerTypeCode, ""); err != nil { + return err + } } return nil } diff --git a/modules/indexer/code/internal/indexer.go b/modules/indexer/code/internal/indexer.go index d58b028124..b165f371fc 100644 --- a/modules/indexer/code/internal/indexer.go +++ b/modules/indexer/code/internal/indexer.go @@ -11,6 +11,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/indexer" "code.gitea.io/gitea/modules/indexer/internal" + "code.gitea.io/gitea/modules/optional" ) // Indexer defines an interface to index and search code contents @@ -28,6 +29,7 @@ type SearchOptions struct { Language string SearchMode indexer.SearchModeType + Archived optional.Option[bool] db.Paginator } diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index 4a5091fded..5434a22aa0 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -22,6 +22,7 @@ import ( unit_model "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/gitrepo" + code_indexer "code.gitea.io/gitea/modules/indexer/code" "code.gitea.io/gitea/modules/label" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/optional" @@ -994,6 +995,12 @@ func updateRepoArchivedState(ctx *context.APIContext, opts api.EditRepoOption) e if err := actions_service.CleanRepoScheduleTasks(ctx, repo); err != nil { log.Error("CleanRepoScheduleTasks for archived repo %s/%s: %v", ctx.Repo.Owner.Name, repo.Name, err) } + if setting.Indexer.RepoIndexerEnabled { + if err := repo_model.UpdateIndexerStatus(ctx, repo, repo_model.RepoIndexerTypeCode, ""); err != nil { + log.Error("Reset code indexer status for archived repo %s/%s: %v", ctx.Repo.Owner.Name, repo.Name, err) + } + code_indexer.UpdateRepoIndexer(repo) + } log.Trace("Repository was archived: %s/%s", ctx.Repo.Owner.Name, repo.Name) } else { if err := repo_model.SetArchiveRepoState(ctx, repo, *opts.Archived); err != nil { @@ -1006,6 +1013,12 @@ func updateRepoArchivedState(ctx *context.APIContext, opts api.EditRepoOption) e log.Error("DetectAndHandleSchedules for un-archived repo %s/%s: %v", ctx.Repo.Owner.Name, repo.Name, err) } } + if setting.Indexer.RepoIndexerEnabled { + if err := repo_model.UpdateIndexerStatus(ctx, repo, repo_model.RepoIndexerTypeCode, ""); err != nil { + log.Error("Reset code indexer status for un-archived repo %s/%s: %v", ctx.Repo.Owner.Name, repo.Name, err) + } + code_indexer.UpdateRepoIndexer(repo) + } log.Trace("Repository was un-archived: %s/%s", ctx.Repo.Owner.Name, repo.Name) } } diff --git a/routers/web/explore/code.go b/routers/web/explore/code.go index fc428d9ec9..0143d5bd40 100644 --- a/routers/web/explore/code.go +++ b/routers/web/explore/code.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" code_indexer "code.gitea.io/gitea/modules/indexer/code" + "code.gitea.io/gitea/modules/optional" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates" "code.gitea.io/gitea/routers/common" @@ -19,8 +20,27 @@ import ( const ( // tplExploreCode explore code page template tplExploreCode templates.TplName = "explore/code" + + archivedModeExclude = "exclude" + archivedModeInclude = "include" + archivedModeOnly = "only" ) +func parseArchivedMode(value string) (string, optional.Option[bool]) { + switch value { + case archivedModeInclude: + return archivedModeInclude, optional.None[bool]() + case "": + return archivedModeInclude, optional.None[bool]() + case archivedModeOnly: + return archivedModeOnly, optional.Some(true) + case archivedModeExclude: + return archivedModeExclude, optional.Some(false) + default: + return archivedModeInclude, optional.None[bool]() + } +} + // Code render explore code page func Code(ctx *context.Context) { if !setting.Indexer.RepoIndexerEnabled || setting.Service.Explore.DisableCodePage { @@ -36,6 +56,10 @@ func Code(ctx *context.Context) { ctx.Data["PageIsExploreCode"] = true ctx.Data["PageIsViewCode"] = true + archivedMode, archivedFilter := parseArchivedMode(ctx.FormTrim("archived")) + ctx.Data["ArchivedMode"] = archivedMode + ctx.Data["IsArchived"] = archivedFilter + prepareSearch := common.PrepareCodeSearch(ctx) if prepareSearch.Keyword == "" { ctx.HTML(http.StatusOK, tplExploreCode) @@ -77,6 +101,7 @@ func Code(ctx *context.Context) { Keyword: prepareSearch.Keyword, SearchMode: prepareSearch.SearchMode, Language: prepareSearch.Language, + Archived: archivedFilter, Paginator: &db.ListOptions{ Page: page, PageSize: setting.UI.RepoSearchPagingNum, diff --git a/routers/web/repo/setting/setting.go b/routers/web/repo/setting/setting.go index 5a5137a1a7..4b0a05e7bf 100644 --- a/routers/web/repo/setting/setting.go +++ b/routers/web/repo/setting/setting.go @@ -962,6 +962,12 @@ func handleSettingsPostArchive(ctx *context.Context) { // update issue indexer issue_indexer.UpdateRepoIndexer(ctx, repo.ID) + if setting.Indexer.RepoIndexerEnabled { + if err := repo_model.UpdateIndexerStatus(ctx, repo, repo_model.RepoIndexerTypeCode, ""); err != nil { + log.Error("Reset code indexer status for archived repo %s/%s: %v", ctx.Repo.Owner.Name, repo.Name, err) + } + code.UpdateRepoIndexer(repo) + } ctx.Flash.Success(ctx.Tr("repo.settings.archive.success")) @@ -991,6 +997,12 @@ func handleSettingsPostUnarchive(ctx *context.Context) { // update issue indexer issue_indexer.UpdateRepoIndexer(ctx, repo.ID) + if setting.Indexer.RepoIndexerEnabled { + if err := repo_model.UpdateIndexerStatus(ctx, repo, repo_model.RepoIndexerTypeCode, ""); err != nil { + log.Error("Reset code indexer status for un-archived repo %s/%s: %v", ctx.Repo.Owner.Name, repo.Name, err) + } + code.UpdateRepoIndexer(repo) + } ctx.Flash.Success(ctx.Tr("repo.settings.unarchive.success")) diff --git a/templates/shared/search/code/results.tmpl b/templates/shared/search/code/results.tmpl index 42f7a181a3..4d67b660c5 100644 --- a/templates/shared/search/code/results.tmpl +++ b/templates/shared/search/code/results.tmpl @@ -1,7 +1,7 @@
{{range $term := .SearchResultLanguages}} + href="?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}{{if $.SelectedSearchMode}}&search_mode={{$.SelectedSearchMode}}{{end}}{{if $.ArchivedMode}}&archived={{$.ArchivedMode}}{{end}}"> {{$term.Language}}
{{$term.Count}}
diff --git a/templates/shared/search/code/search.tmpl b/templates/shared/search/code/search.tmpl index 2041213e19..35e4fce31c 100644 --- a/templates/shared/search/code/search.tmpl +++ b/templates/shared/search/code/search.tmpl @@ -1,12 +1,29 @@ -
- {{template "shared/search/combo" (dict - "Disabled" .CodeIndexerUnavailable - "Value" .Keyword - "Placeholder" (ctx.Locale.Tr "search.code_kind") - "SearchModes" .SearchModes - "SelectedSearchMode" .SelectedSearchMode - )}} -
+
+
+ {{if .Language}}{{end}} +
+ {{template "shared/search/combo" (dict + "Disabled" .CodeIndexerUnavailable + "Value" .Keyword + "Placeholder" (ctx.Locale.Tr "search.code_kind") + "SearchModes" .SearchModes + "SelectedSearchMode" .SelectedSearchMode + )}} +
+ {{if .PageIsExploreCode}} +
+ {{ctx.Locale.Tr "filter_title"}} + {{svg "octicon-triangle-down" 14 "dropdown icon"}} +
+ +
+ + +
+
+ {{end}} +
+
{{template "base/alert" .}}