From 72243bead799653dd0ac32138aeb8c7ea069ef8e Mon Sep 17 00:00:00 2001 From: Rafail Giavrimis <47496212+grafail@users.noreply.github.com> Date: Thu, 24 Sep 2026 08:57:42 +0100 Subject: [PATCH] fix(indexer): index full file paths and real offsets in bleve (#39405) The bleve path token filter added in https://github.com/go-gitea/gitea/pull/32210 never generated the full path of a file, so searching a file by its path (e.g. `potato/ham`) found nothing. It also gave the path tokens made-up offsets instead of their position in the path. The filter is replaced by a tokenizer that emits the raw path suffixes starting at each segment and word (`potato/ham.md`, `ham.md`). Paths below the root now match too, as do names containing `-` or spaces and dotfiles. An exact file name also ranks above files inside a directory with the same name. --------- Co-authored-by: silverwind Co-authored-by: wxiaoguang --- modules/indexer/code/bleve/bleve.go | 16 ++- modules/indexer/code/bleve/bleve_test.go | 6 +- modules/indexer/code/bleve/token/path/path.go | 102 ------------------ .../code/bleve/token/path/path_test.go | 76 ------------- .../bleve/{code_token.go => token_code.go} | 7 -- modules/indexer/code/bleve/token_path.go | 38 +++++++ modules/indexer/code/indexer_test.go | 12 +++ 7 files changed, 66 insertions(+), 191 deletions(-) delete mode 100644 modules/indexer/code/bleve/token/path/path.go delete mode 100644 modules/indexer/code/bleve/token/path/path_test.go rename modules/indexer/code/bleve/{code_token.go => token_code.go} (87%) create mode 100644 modules/indexer/code/bleve/token_path.go diff --git a/modules/indexer/code/bleve/bleve.go b/modules/indexer/code/bleve/bleve.go index 46f97624a8c..d22f735571a 100644 --- a/modules/indexer/code/bleve/bleve.go +++ b/modules/indexer/code/bleve/bleve.go @@ -17,7 +17,6 @@ import ( "gitea.dev/modules/git" "gitea.dev/modules/git/gitcmd" "gitea.dev/modules/indexer" - path_filter "gitea.dev/modules/indexer/code/bleve/token/path" "gitea.dev/modules/indexer/code/internal" indexer_internal "gitea.dev/modules/indexer/internal" inner_bleve "gitea.dev/modules/indexer/internal/bleve" @@ -32,8 +31,8 @@ import ( analyzer_keyword "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" "github.com/blevesearch/bleve/v2/analysis/token/lowercase" "github.com/blevesearch/bleve/v2/analysis/token/unicodenorm" - "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/registry" "github.com/blevesearch/bleve/v2/search/query" "github.com/go-enry/go-enry/v2" ) @@ -69,7 +68,7 @@ const ( repoIndexerAnalyzer = "repoIndexerAnalyzer" filenameIndexerAnalyzer = "filenameIndexerAnalyzer" repoIndexerDocType = "repoIndexerDocType" - repoIndexerLatestVersion = 10 + repoIndexerLatestVersion = 11 ) // generateBleveIndexMapping generates a bleve index mapping for the repo indexer @@ -114,8 +113,8 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) { if err := mapping.AddCustomAnalyzer(filenameIndexerAnalyzer, map[string]any{ "type": analyzer_custom.Name, "char_filters": []string{}, - "tokenizer": unicode.Name, - "token_filters": []string{unicodeNormalizeName, path_filter.Name, lowercase.Name}, + "tokenizer": pathTokenizerName, + "token_filters": []string{unicodeNormalizeName, lowercase.Name}, }); err != nil { return nil, err } @@ -139,6 +138,13 @@ func (b *Indexer) SupportedSearchModes() []indexer.SearchMode { return indexer.SearchModesExactWords() } +func init() { + // due to bleve's design problem, the "Register" must be done in the main goroutine, otherwise data-race + util.MustNoError(registry.RegisterTokenizer(codeTokenizerName, codeTokenizerConstructor)) + util.MustNoError(registry.RegisterTokenFilter(codeTokenFilterName, codeTokenFilterConstructor)) + util.MustNoError(registry.RegisterTokenizer(pathTokenizerName, pathTokenizerConstructor)) +} + // NewIndexer creates a new bleve local indexer func NewIndexer(indexDir string) *Indexer { inner := inner_bleve.NewIndexer(indexDir, repoIndexerLatestVersion, generateBleveIndexMapping) diff --git a/modules/indexer/code/bleve/bleve_test.go b/modules/indexer/code/bleve/bleve_test.go index 0d0fac0f312..0a59b677e13 100644 --- a/modules/indexer/code/bleve/bleve_test.go +++ b/modules/indexer/code/bleve/bleve_test.go @@ -24,7 +24,7 @@ func TestBleveIndexerTokenFilter(t *testing.T) { require.NoError(t, err) batch := inner_bleve.NewFlushingBatch(indexer.inner.Indexer, maxBatchSize) - batch.Index("2", &RepoIndexerData{RepoID: 2, Content: "mDNS.port2=12345", UpdatedAt: time.Now()}) + batch.Index("2", &RepoIndexerData{RepoID: 2, Filename: ".sub-dir/.sub-filename", Content: "mDNS.port2=12345", UpdatedAt: time.Now()}) batch.Flush() testCases := []struct { @@ -36,6 +36,10 @@ func TestBleveIndexerTokenFilter(t *testing.T) { {keyword: "mdns", expectedIDs: []int64{2}}, {keyword: "port", expectedIDs: []int64{2}}, {keyword: "port2", expectedIDs: []int64{2}}, + {keyword: ".sub-dir", expectedIDs: []int64{2}}, + {keyword: "sub-file", expectedIDs: []int64{2}}, // prefix matching? not sure whether this behavior is wanted + {keyword: "filename", expectedIDs: []int64{2}}, + {keyword: "name", expectedIDs: []int64{}}, } for _, testCase := range testCases { diff --git a/modules/indexer/code/bleve/token/path/path.go b/modules/indexer/code/bleve/token/path/path.go deleted file mode 100644 index 09dbe0ad4dd..00000000000 --- a/modules/indexer/code/bleve/token/path/path.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package path - -import ( - "slices" - "strings" - - "gitea.dev/modules/util" - - "github.com/blevesearch/bleve/v2/analysis" - "github.com/blevesearch/bleve/v2/registry" -) - -const Name = "gitea/path" - -type TokenFilter struct{} - -func NewTokenFilter() *TokenFilter { - return &TokenFilter{} -} - -func TokenFilterConstructor(config map[string]any, cache *registry.Cache) (analysis.TokenFilter, error) { - return NewTokenFilter(), nil -} - -func (s *TokenFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - if len(input) == 1 { - // if there is only one token, we don't need to generate the reversed chain - return generatePathTokens(input, false) - } - - normal := generatePathTokens(input, false) - reversed := generatePathTokens(input, true) - - return append(normal, reversed...) -} - -// Generates path tokens from the input tokens. -// This mimics the behavior of the path hierarchy tokenizer in ES. It takes the input tokens and combine them, generating a term for each component -// in tree (e.g., foo/bar/baz.md will generate foo, foo/bar, and foo/bar/baz.md). -// -// If the reverse flag is set, the order of the tokens is reversed (the same input will generate baz.md, baz.md/bar, baz.md/bar/foo). This is useful -// to efficiently search for filenames without supplying the fullpath. -func generatePathTokens(input analysis.TokenStream, reversed bool) analysis.TokenStream { - terms := make([]string, 0, len(input)) - longestTerm := 0 - - if reversed { - slices.Reverse(input) - } - - for i := range input { - var sb strings.Builder - sb.Write(input[0].Term) - - for j := 1; j < i; j++ { - sb.WriteString("/") - sb.Write(input[j].Term) - } - - term := sb.String() - - if longestTerm < len(term) { - longestTerm = len(term) - } - - terms = append(terms, term) - } - - output := make(analysis.TokenStream, 0, len(terms)) - - for _, term := range terms { - var start, end int - - if reversed { - start = 0 - end = len(term) - } else { - start = longestTerm - len(term) - end = longestTerm - } - - token := analysis.Token{ - Position: 1, - Start: start, - End: end, - Type: analysis.AlphaNumeric, - Term: []byte(term), - } - - output = append(output, &token) - } - - return output -} - -func init() { - // FIXME: move it to the bleve's init function, but do not call it in global init - util.MustNoError(registry.RegisterTokenFilter(Name, TokenFilterConstructor)) -} diff --git a/modules/indexer/code/bleve/token/path/path_test.go b/modules/indexer/code/bleve/token/path/path_test.go deleted file mode 100644 index cc52021ef7f..00000000000 --- a/modules/indexer/code/bleve/token/path/path_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package path - -import ( - "fmt" - "testing" - - "github.com/blevesearch/bleve/v2/analysis" - "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" - "github.com/stretchr/testify/assert" -) - -type Scenario struct { - Input string - Tokens []string -} - -func TestTokenFilter(t *testing.T) { - scenarios := []struct { - Input string - Terms []string - }{ - { - Input: "Dockerfile", - Terms: []string{"Dockerfile"}, - }, - { - Input: "Dockerfile.rootless", - Terms: []string{"Dockerfile.rootless"}, - }, - { - Input: "a/b/c/Dockerfile.rootless", - Terms: []string{"a", "a/b", "a/b/c", "a/b/c/Dockerfile.rootless", "Dockerfile.rootless", "Dockerfile.rootless/c", "Dockerfile.rootless/c/b", "Dockerfile.rootless/c/b/a"}, - }, - { - Input: "", - Terms: []string{}, - }, - } - - for _, scenario := range scenarios { - t.Run(fmt.Sprintf("ensure terms of '%s'", scenario.Input), func(t *testing.T) { - terms := extractTerms(scenario.Input) - - assert.Len(t, terms, len(scenario.Terms)) - - for _, term := range terms { - assert.Contains(t, scenario.Terms, term) - } - }) - } -} - -func extractTerms(input string) []string { - tokens := tokenize(input) - filteredTokens := filter(tokens) - terms := make([]string, 0, len(filteredTokens)) - - for _, token := range filteredTokens { - terms = append(terms, string(token.Term)) - } - - return terms -} - -func filter(input analysis.TokenStream) analysis.TokenStream { - filter := NewTokenFilter() - return filter.Filter(input) -} - -func tokenize(input string) analysis.TokenStream { - tokenizer := unicode.NewUnicodeTokenizer() - return tokenizer.Tokenize([]byte(input)) -} diff --git a/modules/indexer/code/bleve/code_token.go b/modules/indexer/code/bleve/token_code.go similarity index 87% rename from modules/indexer/code/bleve/code_token.go rename to modules/indexer/code/bleve/token_code.go index 3a14ddf3981..53a6c401a0e 100644 --- a/modules/indexer/code/bleve/code_token.go +++ b/modules/indexer/code/bleve/token_code.go @@ -7,8 +7,6 @@ import ( "regexp" "unicode" - "gitea.dev/modules/util" - "github.com/blevesearch/bleve/v2/analysis" "github.com/blevesearch/bleve/v2/analysis/tokenizer/character" "github.com/blevesearch/bleve/v2/registry" @@ -58,8 +56,3 @@ func codeTokenFilterConstructor(_ map[string]any, _ *registry.Cache) (analysis.T re: regexp.MustCompile("[a-zA-Z]+|[0-9]+"), }, nil } - -func init() { - util.MustNoError(registry.RegisterTokenizer(codeTokenizerName, codeTokenizerConstructor)) - util.MustNoError(registry.RegisterTokenFilter(codeTokenFilterName, codeTokenFilterConstructor)) -} diff --git a/modules/indexer/code/bleve/token_path.go b/modules/indexer/code/bleve/token_path.go new file mode 100644 index 00000000000..c80233fc09e --- /dev/null +++ b/modules/indexer/code/bleve/token_path.go @@ -0,0 +1,38 @@ +// Copyright 2026 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package bleve + +import ( + "github.com/blevesearch/bleve/v2/analysis" + unicode_tokenizer "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" + "github.com/blevesearch/bleve/v2/registry" +) + +const pathTokenizerName = "pathTokenizer" + +// pathTokenizer emits the path suffixes starting at each segment and word, for the prefix query in Search +type pathTokenizer struct{} + +func (pathTokenizer) Tokenize(input []byte) (ret analysis.TokenStream) { + words := unicode_tokenizer.NewUnicodeTokenizer().Tokenize(input) + for start := range input { + isWordStart := len(words) > 0 && words[0].Start == start + if isWordStart { + words = words[1:] + } + if start == 0 || input[start-1] == '/' || isWordStart { + ret = append(ret, &analysis.Token{ + Start: start, + End: len(input), + Term: input[start:], + Position: len(ret) + 1, + }) + } + } + return ret +} + +func pathTokenizerConstructor(_ map[string]any, _ *registry.Cache) (analysis.Tokenizer, error) { + return pathTokenizer{}, nil +} diff --git a/modules/indexer/code/indexer_test.go b/modules/indexer/code/indexer_test.go index baf9edd8f9c..3a5cad97d76 100644 --- a/modules/indexer/code/indexer_test.go +++ b/modules/indexer/code/indexer_test.go @@ -178,6 +178,18 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) { }, }, }, + // Search for a match on the full path of a file within the repo '62'. + { + RepoIDs: []int64{62}, + Keyword: "potato/ham", + Langs: 1, + Results: []codeSearchResult{ + { + Filename: "potato/ham.md", + Content: "This is not cheese", + }, + }, + }, // Search for matches on the contents of files within the repo '62'. // This scenario yields two results (both are based on contents, the first one is an exact match where as the second is a 'fuzzy' one) {