Use git attributes to determine generated and vendored status for language stats and diffs (#16773)

Replaces #16262
Replaces #16250
Replaces #14833

This PR first implements a `git check-attr` pipe reader - using `git check-attr --stdin -z --cached` - taking account of the change in the output format in git 1.8.5 and creates a helper function to read a tree into a temporary index file for that pipe reader.

It then wires this in to the language stats helper and into the git diff generation.

Files which are marked generated will be folded by default.

Fixes #14786
Fixes #12653

modules/git/repo_index.go

@@ -6,11 +6,17 @@ package git
 
 import (
 	"bytes"
+	"context"
+	"io/ioutil"
+	"os"
 	"strings"
+
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/util"
 )
 
 // ReadTreeToIndex reads a treeish to the index
-func (repo *Repository) ReadTreeToIndex(treeish string) error {
+func (repo *Repository) ReadTreeToIndex(treeish string, indexFilename ...string) error {
 	if len(treeish) != 40 {
 		res, err := NewCommand("rev-parse", "--verify", treeish).RunInDir(repo.Path)
 		if err != nil {
@@ -24,17 +30,42 @@ func (repo *Repository) ReadTreeToIndex(treeish string) error {
 	if err != nil {
 		return err
 	}
-	return repo.readTreeToIndex(id)
+	return repo.readTreeToIndex(id, indexFilename...)
 }
 
-func (repo *Repository) readTreeToIndex(id SHA1) error {
-	_, err := NewCommand("read-tree", id.String()).RunInDir(repo.Path)
+func (repo *Repository) readTreeToIndex(id SHA1, indexFilename ...string) error {
+	var env []string
+	if len(indexFilename) > 0 {
+		env = append(os.Environ(), "GIT_INDEX_FILE="+indexFilename[0])
+	}
+	_, err := NewCommand("read-tree", id.String()).RunInDirWithEnv(repo.Path, env)
 	if err != nil {
 		return err
 	}
 	return nil
 }
 
+// ReadTreeToTemporaryIndex reads a treeish to a temporary index file
+func (repo *Repository) ReadTreeToTemporaryIndex(treeish string) (filename string, cancel context.CancelFunc, err error) {
+	tmpIndex, err := ioutil.TempFile("", "index")
+	if err != nil {
+		return
+	}
+	filename = tmpIndex.Name()
+	cancel = func() {
+		err := util.Remove(filename)
+		if err != nil {
+			log.Error("failed to remove tmp index file: %v", err)
+		}
+	}
+	err = repo.ReadTreeToIndex(treeish, filename)
+	if err != nil {
+		defer cancel()
+		return "", func() {}, err
+	}
+	return
+}
+
 // EmptyIndex empties the index
 func (repo *Repository) EmptyIndex() error {
 	_, err := NewCommand("read-tree", "--empty").RunInDir(repo.Path)