Browse Source

Move blame to gitrepo (#36161)

pull/35550/merge
Lunny Xiao 18 hours ago committed by GitHub
parent
commit
4c67aac23b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 218
      modules/git/blame.go
  2. 208
      modules/gitrepo/blame.go
  3. 15
      modules/gitrepo/blame_sha256_test.go
  4. 14
      modules/gitrepo/blame_test.go
  5. 19
      routers/web/repo/blame.go

218
modules/git/blame.go

@ -1,218 +0,0 @@ @@ -1,218 +0,0 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package git
import (
"bufio"
"bytes"
"context"
"io"
"os"
"code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)
// BlamePart represents block of blame - continuous lines with one sha
type BlamePart struct {
Sha string
Lines []string
PreviousSha string
PreviousPath string
}
// BlameReader returns part of file blame one by one
type BlameReader struct {
output io.WriteCloser
reader io.ReadCloser
bufferedReader *bufio.Reader
done chan error
lastSha *string
ignoreRevsFile string
objectFormat ObjectFormat
cleanupFuncs []func()
}
func (r *BlameReader) UsesIgnoreRevs() bool {
return r.ignoreRevsFile != ""
}
// NextPart returns next part of blame (sequential code lines with the same commit)
func (r *BlameReader) NextPart() (*BlamePart, error) {
var blamePart *BlamePart
if r.lastSha != nil {
blamePart = &BlamePart{
Sha: *r.lastSha,
Lines: make([]string, 0),
}
}
const previousHeader = "previous "
var lineBytes []byte
var isPrefix bool
var err error
for err != io.EOF {
lineBytes, isPrefix, err = r.bufferedReader.ReadLine()
if err != nil && err != io.EOF {
return blamePart, err
}
if len(lineBytes) == 0 {
// isPrefix will be false
continue
}
var objectID string
objectFormatLength := r.objectFormat.FullLength()
if len(lineBytes) > objectFormatLength && lineBytes[objectFormatLength] == ' ' && r.objectFormat.IsValid(string(lineBytes[0:objectFormatLength])) {
objectID = string(lineBytes[0:objectFormatLength])
}
if len(objectID) > 0 {
if blamePart == nil {
blamePart = &BlamePart{
Sha: objectID,
Lines: make([]string, 0),
}
}
if blamePart.Sha != objectID {
r.lastSha = &objectID
// need to munch to end of line...
for isPrefix {
_, isPrefix, err = r.bufferedReader.ReadLine()
if err != nil && err != io.EOF {
return blamePart, err
}
}
return blamePart, nil
}
} else if lineBytes[0] == '\t' {
blamePart.Lines = append(blamePart.Lines, string(lineBytes[1:]))
} else if bytes.HasPrefix(lineBytes, []byte(previousHeader)) {
offset := len(previousHeader) // already includes a space
blamePart.PreviousSha = string(lineBytes[offset : offset+objectFormatLength])
offset += objectFormatLength + 1 // +1 for space
blamePart.PreviousPath = string(lineBytes[offset:])
}
// need to munch to end of line...
for isPrefix {
_, isPrefix, err = r.bufferedReader.ReadLine()
if err != nil && err != io.EOF {
return blamePart, err
}
}
}
r.lastSha = nil
return blamePart, nil
}
// Close BlameReader - don't run NextPart after invoking that
func (r *BlameReader) Close() error {
if r.bufferedReader == nil {
return nil
}
err := <-r.done
r.bufferedReader = nil
_ = r.reader.Close()
_ = r.output.Close()
for _, cleanup := range r.cleanupFuncs {
if cleanup != nil {
cleanup()
}
}
return err
}
// CreateBlameReader creates reader for given repository, commit and file
func CreateBlameReader(ctx context.Context, objectFormat ObjectFormat, repoPath string, commit *Commit, file string, bypassBlameIgnore bool) (rd *BlameReader, err error) {
var ignoreRevsFileName string
var ignoreRevsFileCleanup func()
defer func() {
if err != nil && ignoreRevsFileCleanup != nil {
ignoreRevsFileCleanup()
}
}()
cmd := gitcmd.NewCommand("blame", "--porcelain")
if DefaultFeatures().CheckVersionAtLeast("2.23") && !bypassBlameIgnore {
ignoreRevsFileName, ignoreRevsFileCleanup, err = tryCreateBlameIgnoreRevsFile(commit)
if err != nil && !IsErrNotExist(err) {
return nil, err
}
if ignoreRevsFileName != "" {
// Possible improvement: use --ignore-revs-file /dev/stdin on unix
// There is no equivalent on Windows. May be implemented if Gitea uses an external git backend.
cmd.AddOptionValues("--ignore-revs-file", ignoreRevsFileName)
}
}
cmd.AddDynamicArguments(commit.ID.String()).AddDashesAndList(file)
done := make(chan error, 1)
reader, stdout, err := os.Pipe()
if err != nil {
return nil, err
}
go func() {
stderr := bytes.Buffer{}
// TODO: it doesn't work for directories (the directories shouldn't be "blamed"), and the "err" should be returned by "Read" but not by "Close"
err := cmd.WithDir(repoPath).
WithUseContextTimeout(true).
WithStdout(stdout).
WithStderr(&stderr).
Run(ctx)
done <- err
_ = stdout.Close()
if err != nil {
log.Error("Error running git blame (dir: %v): %v, stderr: %v", repoPath, err, stderr.String())
}
}()
bufferedReader := bufio.NewReader(reader)
return &BlameReader{
output: stdout,
reader: reader,
bufferedReader: bufferedReader,
done: done,
ignoreRevsFile: ignoreRevsFileName,
objectFormat: objectFormat,
cleanupFuncs: []func(){ignoreRevsFileCleanup},
}, nil
}
func tryCreateBlameIgnoreRevsFile(commit *Commit) (string, func(), error) {
entry, err := commit.GetTreeEntryByPath(".git-blame-ignore-revs")
if err != nil {
return "", nil, err
}
r, err := entry.Blob().DataAsync()
if err != nil {
return "", nil, err
}
defer r.Close()
f, cleanup, err := setting.AppDataTempDir("git-repo-content").CreateTempFileRandom("git-blame-ignore-revs")
if err != nil {
return "", nil, err
}
filename := f.Name()
_, err = io.Copy(f, r)
_ = f.Close()
if err != nil {
cleanup()
return "", nil, err
}
return filename, cleanup, nil
}

208
modules/gitrepo/blame.go

@ -4,9 +4,16 @@ @@ -4,9 +4,16 @@
package gitrepo
import (
"bufio"
"bytes"
"context"
"io"
"os"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/git/gitcmd"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
)
func LineBlame(ctx context.Context, repo Repository, revision, file string, line uint) (string, error) {
@ -16,3 +23,204 @@ func LineBlame(ctx context.Context, repo Repository, revision, file string, line @@ -16,3 +23,204 @@ func LineBlame(ctx context.Context, repo Repository, revision, file string, line
AddOptionValues("-p", revision).
AddDashesAndList(file))
}
// BlamePart represents block of blame - continuous lines with one sha
type BlamePart struct {
Sha string
Lines []string
PreviousSha string
PreviousPath string
}
// BlameReader returns part of file blame one by one
type BlameReader struct {
output io.WriteCloser
reader io.ReadCloser
bufferedReader *bufio.Reader
done chan error
lastSha *string
ignoreRevsFile string
objectFormat git.ObjectFormat
cleanupFuncs []func()
}
func (r *BlameReader) UsesIgnoreRevs() bool {
return r.ignoreRevsFile != ""
}
// NextPart returns next part of blame (sequential code lines with the same commit)
func (r *BlameReader) NextPart() (*BlamePart, error) {
var blamePart *BlamePart
if r.lastSha != nil {
blamePart = &BlamePart{
Sha: *r.lastSha,
Lines: make([]string, 0),
}
}
const previousHeader = "previous "
var lineBytes []byte
var isPrefix bool
var err error
for err != io.EOF {
lineBytes, isPrefix, err = r.bufferedReader.ReadLine()
if err != nil && err != io.EOF {
return blamePart, err
}
if len(lineBytes) == 0 {
// isPrefix will be false
continue
}
var objectID string
objectFormatLength := r.objectFormat.FullLength()
if len(lineBytes) > objectFormatLength && lineBytes[objectFormatLength] == ' ' && r.objectFormat.IsValid(string(lineBytes[0:objectFormatLength])) {
objectID = string(lineBytes[0:objectFormatLength])
}
if len(objectID) > 0 {
if blamePart == nil {
blamePart = &BlamePart{
Sha: objectID,
Lines: make([]string, 0),
}
}
if blamePart.Sha != objectID {
r.lastSha = &objectID
// need to munch to end of line...
for isPrefix {
_, isPrefix, err = r.bufferedReader.ReadLine()
if err != nil && err != io.EOF {
return blamePart, err
}
}
return blamePart, nil
}
} else if lineBytes[0] == '\t' {
blamePart.Lines = append(blamePart.Lines, string(lineBytes[1:]))
} else if bytes.HasPrefix(lineBytes, []byte(previousHeader)) {
offset := len(previousHeader) // already includes a space
blamePart.PreviousSha = string(lineBytes[offset : offset+objectFormatLength])
offset += objectFormatLength + 1 // +1 for space
blamePart.PreviousPath = string(lineBytes[offset:])
}
// need to munch to end of line...
for isPrefix {
_, isPrefix, err = r.bufferedReader.ReadLine()
if err != nil && err != io.EOF {
return blamePart, err
}
}
}
r.lastSha = nil
return blamePart, nil
}
// Close BlameReader - don't run NextPart after invoking that
func (r *BlameReader) Close() error {
if r.bufferedReader == nil {
return nil
}
err := <-r.done
r.bufferedReader = nil
_ = r.reader.Close()
_ = r.output.Close()
for _, cleanup := range r.cleanupFuncs {
if cleanup != nil {
cleanup()
}
}
return err
}
// CreateBlameReader creates reader for given repository, commit and file
func CreateBlameReader(ctx context.Context, objectFormat git.ObjectFormat, repo Repository, commit *git.Commit, file string, bypassBlameIgnore bool) (rd *BlameReader, err error) {
var ignoreRevsFileName string
var ignoreRevsFileCleanup func()
defer func() {
if err != nil && ignoreRevsFileCleanup != nil {
ignoreRevsFileCleanup()
}
}()
cmd := gitcmd.NewCommand("blame", "--porcelain")
if git.DefaultFeatures().CheckVersionAtLeast("2.23") && !bypassBlameIgnore {
ignoreRevsFileName, ignoreRevsFileCleanup, err = tryCreateBlameIgnoreRevsFile(commit)
if err != nil && !git.IsErrNotExist(err) {
return nil, err
}
if ignoreRevsFileName != "" {
// Possible improvement: use --ignore-revs-file /dev/stdin on unix
// There is no equivalent on Windows. May be implemented if Gitea uses an external git backend.
cmd.AddOptionValues("--ignore-revs-file", ignoreRevsFileName)
}
}
cmd.AddDynamicArguments(commit.ID.String()).AddDashesAndList(file)
done := make(chan error, 1)
reader, stdout, err := os.Pipe()
if err != nil {
return nil, err
}
go func() {
stderr := bytes.Buffer{}
// TODO: it doesn't work for directories (the directories shouldn't be "blamed"), and the "err" should be returned by "Read" but not by "Close"
err := RunCmd(ctx, repo, cmd.WithUseContextTimeout(true).
WithStdout(stdout).
WithStderr(&stderr),
)
done <- err
_ = stdout.Close()
if err != nil {
log.Error("Error running git blame (dir: %v): %v, stderr: %v", repoPath, err, stderr.String())
}
}()
bufferedReader := bufio.NewReader(reader)
return &BlameReader{
output: stdout,
reader: reader,
bufferedReader: bufferedReader,
done: done,
ignoreRevsFile: ignoreRevsFileName,
objectFormat: objectFormat,
cleanupFuncs: []func(){ignoreRevsFileCleanup},
}, nil
}
func tryCreateBlameIgnoreRevsFile(commit *git.Commit) (string, func(), error) {
entry, err := commit.GetTreeEntryByPath(".git-blame-ignore-revs")
if err != nil {
return "", nil, err
}
r, err := entry.Blob().DataAsync()
if err != nil {
return "", nil, err
}
defer r.Close()
f, cleanup, err := setting.AppDataTempDir("git-repo-content").CreateTempFileRandom("git-blame-ignore-revs")
if err != nil {
return "", nil, err
}
filename := f.Name()
_, err = io.Copy(f, r)
_ = f.Close()
if err != nil {
cleanup()
return "", nil, err
}
return filename, cleanup, nil
}

15
modules/git/blame_sha256_test.go → modules/gitrepo/blame_sha256_test.go

@ -1,12 +1,13 @@ @@ -1,12 +1,13 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package git
package gitrepo
import (
"context"
"testing"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/setting"
"github.com/stretchr/testify/assert"
@ -17,13 +18,14 @@ func TestReadingBlameOutputSha256(t *testing.T) { @@ -17,13 +18,14 @@ func TestReadingBlameOutputSha256(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
if isGogit {
if git.DefaultFeatures().UsingGogit {
t.Skip("Skipping test since gogit does not support sha256")
return
}
t.Run("Without .git-blame-ignore-revs", func(t *testing.T) {
repo, err := OpenRepository(ctx, "./tests/repos/repo5_pulls_sha256")
storage := &mockRepository{path: "repo5_pulls_sha256"}
repo, err := OpenRepository(ctx, storage)
assert.NoError(t, err)
defer repo.Close()
@ -47,7 +49,7 @@ func TestReadingBlameOutputSha256(t *testing.T) { @@ -47,7 +49,7 @@ func TestReadingBlameOutputSha256(t *testing.T) {
}
for _, bypass := range []bool{false, true} {
blameReader, err := CreateBlameReader(ctx, Sha256ObjectFormat, "./tests/repos/repo5_pulls_sha256", commit, "README.md", bypass)
blameReader, err := CreateBlameReader(ctx, git.Sha256ObjectFormat, storage, commit, "README.md", bypass)
assert.NoError(t, err)
assert.NotNil(t, blameReader)
defer blameReader.Close()
@ -68,7 +70,8 @@ func TestReadingBlameOutputSha256(t *testing.T) { @@ -68,7 +70,8 @@ func TestReadingBlameOutputSha256(t *testing.T) {
})
t.Run("With .git-blame-ignore-revs", func(t *testing.T) {
repo, err := OpenRepository(ctx, "./tests/repos/repo6_blame_sha256")
storage := &mockRepository{path: "repo6_blame_sha256"}
repo, err := OpenRepository(ctx, storage)
assert.NoError(t, err)
defer repo.Close()
@ -131,7 +134,7 @@ func TestReadingBlameOutputSha256(t *testing.T) { @@ -131,7 +134,7 @@ func TestReadingBlameOutputSha256(t *testing.T) {
for _, c := range cases {
commit, err := repo.GetCommit(c.CommitID)
assert.NoError(t, err)
blameReader, err := CreateBlameReader(ctx, objectFormat, "./tests/repos/repo6_blame_sha256", commit, "blame.txt", c.Bypass)
blameReader, err := CreateBlameReader(ctx, objectFormat, storage, commit, "blame.txt", c.Bypass)
assert.NoError(t, err)
assert.NotNil(t, blameReader)
defer blameReader.Close()

14
modules/git/blame_test.go → modules/gitrepo/blame_test.go

@ -1,12 +1,13 @@ @@ -1,12 +1,13 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package git
package gitrepo
import (
"context"
"testing"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/setting"
"github.com/stretchr/testify/assert"
@ -18,10 +19,10 @@ func TestReadingBlameOutput(t *testing.T) { @@ -18,10 +19,10 @@ func TestReadingBlameOutput(t *testing.T) {
defer cancel()
t.Run("Without .git-blame-ignore-revs", func(t *testing.T) {
repo, err := OpenRepository(ctx, "./tests/repos/repo5_pulls")
storage := &mockRepository{path: "repo5_pulls"}
repo, err := OpenRepository(ctx, storage)
assert.NoError(t, err)
defer repo.Close()
commit, err := repo.GetCommit("f32b0a9dfd09a60f616f29158f772cedd89942d2")
assert.NoError(t, err)
@ -42,7 +43,7 @@ func TestReadingBlameOutput(t *testing.T) { @@ -42,7 +43,7 @@ func TestReadingBlameOutput(t *testing.T) {
}
for _, bypass := range []bool{false, true} {
blameReader, err := CreateBlameReader(ctx, Sha1ObjectFormat, "./tests/repos/repo5_pulls", commit, "README.md", bypass)
blameReader, err := CreateBlameReader(ctx, git.Sha1ObjectFormat, storage, commit, "README.md", bypass)
assert.NoError(t, err)
assert.NotNil(t, blameReader)
defer blameReader.Close()
@ -63,7 +64,8 @@ func TestReadingBlameOutput(t *testing.T) { @@ -63,7 +64,8 @@ func TestReadingBlameOutput(t *testing.T) {
})
t.Run("With .git-blame-ignore-revs", func(t *testing.T) {
repo, err := OpenRepository(ctx, "./tests/repos/repo6_blame")
storage := &mockRepository{path: "repo6_blame"}
repo, err := OpenRepository(ctx, storage)
assert.NoError(t, err)
defer repo.Close()
@ -127,7 +129,7 @@ func TestReadingBlameOutput(t *testing.T) { @@ -127,7 +129,7 @@ func TestReadingBlameOutput(t *testing.T) {
commit, err := repo.GetCommit(c.CommitID)
assert.NoError(t, err)
blameReader, err := CreateBlameReader(ctx, objectFormat, "./tests/repos/repo6_blame", commit, "blame.txt", c.Bypass)
blameReader, err := CreateBlameReader(ctx, objectFormat, storage, commit, "blame.txt", c.Bypass)
assert.NoError(t, err)
assert.NotNil(t, blameReader)
defer blameReader.Close()

19
routers/web/repo/blame.go

@ -17,6 +17,7 @@ import ( @@ -17,6 +17,7 @@ import (
"code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/git/languagestats"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/highlight"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
@ -99,7 +100,7 @@ func RefBlame(ctx *context.Context) { @@ -99,7 +100,7 @@ func RefBlame(ctx *context.Context) {
}
type blameResult struct {
Parts []*git.BlamePart
Parts []*gitrepo.BlamePart
UsesIgnoreRevs bool
FaultyIgnoreRevsFile bool
}
@ -107,7 +108,7 @@ type blameResult struct { @@ -107,7 +108,7 @@ type blameResult struct {
func performBlame(ctx *context.Context, repo *repo_model.Repository, commit *git.Commit, file string, bypassBlameIgnore bool) (*blameResult, error) {
objectFormat := ctx.Repo.GetObjectFormat()
blameReader, err := git.CreateBlameReader(ctx, objectFormat, repo.RepoPath(), commit, file, bypassBlameIgnore)
blameReader, err := gitrepo.CreateBlameReader(ctx, objectFormat, repo, commit, file, bypassBlameIgnore)
if err != nil {
return nil, err
}
@ -123,7 +124,7 @@ func performBlame(ctx *context.Context, repo *repo_model.Repository, commit *git @@ -123,7 +124,7 @@ func performBlame(ctx *context.Context, repo *repo_model.Repository, commit *git
if len(r.Parts) == 0 && r.UsesIgnoreRevs {
// try again without ignored revs
blameReader, err = git.CreateBlameReader(ctx, objectFormat, repo.RepoPath(), commit, file, true)
blameReader, err = gitrepo.CreateBlameReader(ctx, objectFormat, repo, commit, file, true)
if err != nil {
return nil, err
}
@ -143,12 +144,12 @@ func performBlame(ctx *context.Context, repo *repo_model.Repository, commit *git @@ -143,12 +144,12 @@ func performBlame(ctx *context.Context, repo *repo_model.Repository, commit *git
return r, nil
}
func fillBlameResult(br *git.BlameReader, r *blameResult) error {
func fillBlameResult(br *gitrepo.BlameReader, r *blameResult) error {
r.UsesIgnoreRevs = br.UsesIgnoreRevs()
previousHelper := make(map[string]*git.BlamePart)
previousHelper := make(map[string]*gitrepo.BlamePart)
r.Parts = make([]*git.BlamePart, 0, 5)
r.Parts = make([]*gitrepo.BlamePart, 0, 5)
for {
blamePart, err := br.NextPart()
if err != nil {
@ -173,7 +174,7 @@ func fillBlameResult(br *git.BlameReader, r *blameResult) error { @@ -173,7 +174,7 @@ func fillBlameResult(br *git.BlameReader, r *blameResult) error {
return nil
}
func processBlameParts(ctx *context.Context, blameParts []*git.BlamePart) map[string]*user_model.UserCommit {
func processBlameParts(ctx *context.Context, blameParts []*gitrepo.BlamePart) map[string]*user_model.UserCommit {
// store commit data by SHA to look up avatar info etc
commitNames := make(map[string]*user_model.UserCommit)
// and as blameParts can reference the same commits multiple
@ -220,7 +221,7 @@ func processBlameParts(ctx *context.Context, blameParts []*git.BlamePart) map[st @@ -220,7 +221,7 @@ func processBlameParts(ctx *context.Context, blameParts []*git.BlamePart) map[st
return commitNames
}
func renderBlameFillFirstBlameRow(repoLink string, avatarUtils *templates.AvatarUtils, part *git.BlamePart, commit *user_model.UserCommit, br *blameRow) {
func renderBlameFillFirstBlameRow(repoLink string, avatarUtils *templates.AvatarUtils, part *gitrepo.BlamePart, commit *user_model.UserCommit, br *blameRow) {
if commit.User != nil {
br.Avatar = avatarUtils.Avatar(commit.User, 18)
} else {
@ -234,7 +235,7 @@ func renderBlameFillFirstBlameRow(repoLink string, avatarUtils *templates.Avatar @@ -234,7 +235,7 @@ func renderBlameFillFirstBlameRow(repoLink string, avatarUtils *templates.Avatar
br.CommitSince = templates.TimeSince(commit.Author.When)
}
func renderBlame(ctx *context.Context, blameParts []*git.BlamePart, commitNames map[string]*user_model.UserCommit) {
func renderBlame(ctx *context.Context, blameParts []*gitrepo.BlamePart, commitNames map[string]*user_model.UserCommit) {
language, err := languagestats.GetFileLanguage(ctx, ctx.Repo.GitRepo, ctx.Repo.CommitID, ctx.Repo.TreePath)
if err != nil {
log.Error("Unable to get file language for %-v:%s. Error: %v", ctx.Repo.Repository, ctx.Repo.TreePath, err)

Loading…
Cancel
Save