Unverified Commit 511f6138 authored by zeripath's avatar zeripath Committed by GitHub
Browse files

Use native git variants by default with go-git variants as build tag (#13673)


* Move last commit cache back into modules/git
Signed-off-by: default avatarAndrew Thornton <art27@cantab.net>

* Remove go-git from the interface for last commit cache
Signed-off-by: default avatarAndrew Thornton <art27@cantab.net>

* move cacheref to last_commit_cache
Signed-off-by: default avatarAndrew Thornton <art27@cantab.net>

* Remove go-git from routers/private/hook
Signed-off-by: default avatarAndrew Thornton <art27@cantab.net>

* Move FindLFSFiles to pipeline
Signed-off-by: default avatarAndrew Thornton <art27@cantab.net>

* Make no-go-git variants
Signed-off-by: default avatarAndrew Thornton <art27@cantab.net>

* Submodule RefID
Signed-off-by: default avatarAndrew Thornton <art27@cantab.net>

* fix issue with GetCommitsInfo
Signed-off-by: default avatarAndrew Thornton <art27@cantab.net>

* fix GetLastCommitForPaths
Signed-off-by: default avatarAndrew Thornton <art27@cantab.net>

* Improve efficiency
Signed-off-by: default avatarAndrew Thornton <art27@cantab.net>

* More efficiency
Signed-off-by: default avatarAndrew Thornton <art27@cantab.net>

* even faster
Signed-off-by: default avatarAndrew Thornton <art27@cantab.net>

* Reduce duplication

* As per @lunny
Signed-off-by: default avatarAndrew Thornton <art27@cantab.net>

* attempt to fix drone
Signed-off-by: default avatarAndrew Thornton <art27@cantab.net>

* fix test-tags
Signed-off-by: default avatarAndrew Thornton <art27@cantab.net>

* default to use no-go-git variants and add gogit build tag
Signed-off-by: default avatarAndrew Thornton <art27@cantab.net>

* placate lint
Signed-off-by: default avatarAndrew Thornton <art27@cantab.net>

* as per @6543
Signed-off-by: default avatarAndrew Thornton <art27@cantab.net>
Co-authored-by: default avatar6543 <6543@obermui.de>
Co-authored-by: default avatartechknowlogick <techknowlogick@gitea.io>
parent 0851a895
Showing with 1432 additions and 403 deletions
+1432 -403
......@@ -33,6 +33,16 @@ steps:
GOSUMDB: sum.golang.org
TAGS: bindata sqlite sqlite_unlock_notify
- name: lint-backend-gogit
pull: always
image: golang:1.15
commands:
- make lint-backend
environment:
GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not
GOSUMDB: sum.golang.org
TAGS: bindata gogit sqlite sqlite_unlock_notify
- name: checks-frontend
image: node:14
commands:
......@@ -69,7 +79,7 @@ steps:
GOPROXY: off
GOOS: linux
GOARCH: arm64
TAGS: bindata
TAGS: bindata gogit
commands:
- make backend # test cross compile
- rm ./gitea # clean
......@@ -173,6 +183,17 @@ steps:
GITHUB_READ_TOKEN:
from_secret: github_read_token
- name: unit-test-gogit
pull: always
image: golang:1.15
commands:
- make unit-test-coverage test-check
environment:
GOPROXY: off
TAGS: bindata gogit sqlite sqlite_unlock_notify
GITHUB_READ_TOKEN:
from_secret: github_read_token
- name: test-mysql
image: golang:1.15
commands:
......@@ -305,7 +326,8 @@ steps:
- timeout -s ABRT 40m make test-sqlite-migration test-sqlite
environment:
GOPROXY: off
TAGS: bindata
TAGS: bindata gogit sqlite sqlite_unlock_notify
TEST_TAGS: gogit sqlite sqlite_unlock_notify
USE_REPO_TEST_DIR: 1
depends_on:
- build
......@@ -318,7 +340,8 @@ steps:
- timeout -s ABRT 40m make test-pgsql-migration test-pgsql
environment:
GOPROXY: off
TAGS: bindata
TAGS: bindata gogit
TEST_TAGS: gogit
TEST_LDAP: 1
USE_REPO_TEST_DIR: 1
depends_on:
......
......@@ -110,7 +110,10 @@ TAGS ?=
TAGS_SPLIT := $(subst $(COMMA), ,$(TAGS))
TAGS_EVIDENCE := $(MAKE_EVIDENCE_DIR)/tags
TEST_TAGS ?= sqlite sqlite_unlock_notify
GO_DIRS := cmd integrations models modules routers build services vendor tools
GO_SOURCES := $(wildcard *.go)
GO_SOURCES += $(shell find $(GO_DIRS) -type f -name "*.go" -not -path modules/options/bindata.go -not -path modules/public/bindata.go -not -path modules/templates/bindata.go)
......@@ -339,8 +342,8 @@ watch-backend: go-check
.PHONY: test
test:
@echo "Running go test..."
@$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='sqlite sqlite_unlock_notify' $(GO_PACKAGES)
@echo "Running go test with -tags '$(TEST_TAGS)'..."
@$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='$(TEST_TAGS)' $(GO_PACKAGES)
.PHONY: test-check
test-check:
......@@ -356,8 +359,8 @@ test-check:
.PHONY: test\#%
test\#%:
@echo "Running go test..."
@$(GO) test -mod=vendor -tags='sqlite sqlite_unlock_notify' -run $(subst .,/,$*) $(GO_PACKAGES)
@echo "Running go test with -tags '$(TEST_TAGS)'..."
@$(GO) test -mod=vendor -tags='$(TEST_TAGS)' -run $(subst .,/,$*) $(GO_PACKAGES)
.PHONY: coverage
coverage:
......@@ -365,8 +368,8 @@ coverage:
.PHONY: unit-test-coverage
unit-test-coverage:
@echo "Running unit-test-coverage..."
@$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='sqlite sqlite_unlock_notify' -cover -coverprofile coverage.out $(GO_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1
@echo "Running unit-test-coverage -tags '$(TEST_TAGS)'..."
@$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='$(TEST_TAGS)' -cover -coverprofile coverage.out $(GO_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1
.PHONY: vendor
vendor:
......@@ -511,7 +514,7 @@ integrations.mssql.test: git-check $(GO_SOURCES)
$(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.mssql.test
integrations.sqlite.test: git-check $(GO_SOURCES)
$(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.sqlite.test -tags 'sqlite sqlite_unlock_notify'
$(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.sqlite.test -tags '$(TEST_TAGS)'
integrations.cover.test: git-check $(GO_SOURCES)
$(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -coverpkg $(shell echo $(GO_PACKAGES) | tr ' ' ',') -o integrations.cover.test
......@@ -534,7 +537,7 @@ migrations.mssql.test: $(GO_SOURCES)
.PHONY: migrations.sqlite.test
migrations.sqlite.test: $(GO_SOURCES)
$(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/integrations/migration-test -o migrations.sqlite.test -tags 'sqlite sqlite_unlock_notify'
$(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/integrations/migration-test -o migrations.sqlite.test -tags '$(TEST_TAGS)'
.PHONY: check
check: test
......
......@@ -101,6 +101,7 @@ Depending on requirements, the following build tags can be included.
- `pam`: Enable support for PAM (Linux Pluggable Authentication Modules). Can
be used to authenticate local users or extend authentication to methods
available to PAM.
* `gogit`: (EXPERIMENTAL) Use go-git variants of git commands.
Bundling assets into the binary using the `bindata` build tag is recommended for
production deployments. It is possible to serve the static assets directly via a reverse proxy,
......
......@@ -27,6 +27,24 @@ func newCache(cacheConfig setting.Cache) (mc.Cache, error) {
})
}
// Cache is the interface that operates the cache data.
type Cache interface {
// Put puts value into cache with key and expire time.
Put(key string, val interface{}, timeout int64) error
// Get gets cached value by given key.
Get(key string) interface{}
// Delete deletes cached value by given key.
Delete(key string) error
// Incr increases cached int-type value by given key as a counter.
Incr(key string) error
// Decr decreases cached int-type value by given key as a counter.
Decr(key string) error
// IsExist returns true if cached value exists.
IsExist(key string) bool
// Flush deletes all cached data.
Flush() error
}
// NewContext start cache service
func NewContext() error {
var err error
......@@ -40,6 +58,11 @@ func NewContext() error {
return err
}
// GetCache returns the currently configured cache
func GetCache() Cache {
return conn
}
// GetString returns the key value from cache with callback when no key exists in cache
func GetString(key string, getFunc func() (string, error)) (string, error) {
if conn == nil || setting.CacheService.TTL == 0 {
......
......@@ -13,7 +13,6 @@ import (
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/stretchr/testify/assert"
)
......@@ -21,7 +20,7 @@ func TestToCommitMeta(t *testing.T) {
assert.NoError(t, models.PrepareTestDatabase())
headRepo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository)
sha1, _ := git.NewIDFromString("0000000000000000000000000000000000000000")
signature := &object.Signature{Name: "Test Signature", Email: "test@email.com", When: time.Unix(0, 0)}
signature := &git.Signature{Name: "Test Signature", Email: "test@email.com", When: time.Unix(0, 0)}
tag := &git.Tag{
Name: "Test Tag",
ID: sha1,
......
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bufio"
"bytes"
"math"
"strconv"
)
// ReadBatchLine reads the header line from cat-file --batch
// We expect:
// <sha> SP <type> SP <size> LF
func ReadBatchLine(rd *bufio.Reader) (sha []byte, typ string, size int64, err error) {
sha, err = rd.ReadBytes(' ')
if err != nil {
return
}
sha = sha[:len(sha)-1]
typ, err = rd.ReadString(' ')
if err != nil {
return
}
typ = typ[:len(typ)-1]
var sizeStr string
sizeStr, err = rd.ReadString('\n')
if err != nil {
return
}
size, err = strconv.ParseInt(sizeStr[:len(sizeStr)-1], 10, 64)
return
}
// ReadTagObjectID reads a tag object ID hash from a cat-file --batch stream, throwing away the rest of the stream.
func ReadTagObjectID(rd *bufio.Reader, size int64) (string, error) {
id := ""
var n int64
headerLoop:
for {
line, err := rd.ReadBytes('\n')
if err != nil {
return "", err
}
n += int64(len(line))
idx := bytes.Index(line, []byte{' '})
if idx < 0 {
continue
}
if string(line[:idx]) == "object" {
id = string(line[idx+1 : len(line)-1])
break headerLoop
}
}
// Discard the rest of the tag
discard := size - n
for discard > math.MaxInt32 {
_, err := rd.Discard(math.MaxInt32)
if err != nil {
return id, err
}
discard -= math.MaxInt32
}
_, err := rd.Discard(int(discard))
return id, err
}
// ReadTreeID reads a tree ID from a cat-file --batch stream, throwing away the rest of the stream.
func ReadTreeID(rd *bufio.Reader, size int64) (string, error) {
id := ""
var n int64
headerLoop:
for {
line, err := rd.ReadBytes('\n')
if err != nil {
return "", err
}
n += int64(len(line))
idx := bytes.Index(line, []byte{' '})
if idx < 0 {
continue
}
if string(line[:idx]) == "tree" {
id = string(line[idx+1 : len(line)-1])
break headerLoop
}
}
// Discard the rest of the commit
discard := size - n
for discard > math.MaxInt32 {
_, err := rd.Discard(math.MaxInt32)
if err != nil {
return id, err
}
discard -= math.MaxInt32
}
_, err := rd.Discard(int(discard))
return id, err
}
// git tree files are a list:
// <mode-in-ascii> SP <fname> NUL <20-byte SHA>
//
// Unfortunately this 20-byte notation is somewhat in conflict to all other git tools
// Therefore we need some method to convert these 20-byte SHAs to a 40-byte SHA
// constant hextable to help quickly convert between 20byte and 40byte hashes
const hextable = "0123456789abcdef"
// to40ByteSHA converts a 20-byte SHA in a 40-byte slice into a 40-byte sha in place
// without allocations. This is at least 100x quicker that hex.EncodeToString
// NB This requires that sha is a 40-byte slice
func to40ByteSHA(sha []byte) []byte {
for i := 19; i >= 0; i-- {
v := sha[i]
vhi, vlo := v>>4, v&0x0f
shi, slo := hextable[vhi], hextable[vlo]
sha[i*2], sha[i*2+1] = shi, slo
}
return sha
}
// ParseTreeLineSkipMode reads an entry from a tree in a cat-file --batch stream
// This simply skips the mode - saving a substantial amount of time and carefully avoids allocations - except where fnameBuf is too small.
// It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations
//
// Each line is composed of:
// <mode-in-ascii-dropping-initial-zeros> SP <fname> NUL <20-byte SHA>
//
// We don't attempt to convert the 20-byte SHA to 40-byte SHA to save a lot of time
func ParseTreeLineSkipMode(rd *bufio.Reader, fnameBuf, shaBuf []byte) (fname, sha []byte, n int, err error) {
var readBytes []byte
// Skip the Mode
readBytes, err = rd.ReadSlice(' ') // NB: DOES NOT ALLOCATE SIMPLY RETURNS SLICE WITHIN READER BUFFER
if err != nil {
return
}
n += len(readBytes)
// Deal with the fname
readBytes, err = rd.ReadSlice('\x00')
copy(fnameBuf, readBytes)
if len(fnameBuf) > len(readBytes) {
fnameBuf = fnameBuf[:len(readBytes)] // cut the buf the correct size
} else {
fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...) // extend the buf and copy in the missing bits
}
for err == bufio.ErrBufferFull { // Then we need to read more
readBytes, err = rd.ReadSlice('\x00')
fnameBuf = append(fnameBuf, readBytes...) // there is little point attempting to avoid allocations here so just extend
}
n += len(fnameBuf)
if err != nil {
return
}
fnameBuf = fnameBuf[:len(fnameBuf)-1] // Drop the terminal NUL
fname = fnameBuf // set the returnable fname to the slice
// Now deal with the 20-byte SHA
idx := 0
for idx < 20 {
read := 0
read, err = rd.Read(shaBuf[idx:20])
n += read
if err != nil {
return
}
idx += read
}
sha = shaBuf
return
}
// ParseTreeLine reads an entry from a tree in a cat-file --batch stream
// This carefully avoids allocations - except where fnameBuf is too small.
// It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations
//
// Each line is composed of:
// <mode-in-ascii-dropping-initial-zeros> SP <fname> NUL <20-byte SHA>
//
// We don't attempt to convert the 20-byte SHA to 40-byte SHA to save a lot of time
func ParseTreeLine(rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) {
var readBytes []byte
// Read the Mode
readBytes, err = rd.ReadSlice(' ')
if err != nil {
return
}
n += len(readBytes)
copy(modeBuf, readBytes)
if len(modeBuf) > len(readBytes) {
modeBuf = modeBuf[:len(readBytes)]
} else {
modeBuf = append(modeBuf, readBytes[len(modeBuf):]...)
}
mode = modeBuf[:len(modeBuf)-1] // Drop the SP
// Deal with the fname
readBytes, err = rd.ReadSlice('\x00')
copy(fnameBuf, readBytes)
if len(fnameBuf) > len(readBytes) {
fnameBuf = fnameBuf[:len(readBytes)]
} else {
fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...)
}
for err == bufio.ErrBufferFull {
readBytes, err = rd.ReadSlice('\x00')
fnameBuf = append(fnameBuf, readBytes...)
}
n += len(fnameBuf)
if err != nil {
return
}
fnameBuf = fnameBuf[:len(fnameBuf)-1]
fname = fnameBuf
// Deal with the 20-byte SHA
idx := 0
for idx < 20 {
read := 0
read, err = rd.Read(shaBuf[idx:20])
n += read
if err != nil {
return
}
idx += read
}
sha = shaBuf
return
}
......@@ -10,28 +10,9 @@ import (
"encoding/base64"
"io"
"io/ioutil"
"github.com/go-git/go-git/v5/plumbing"
)
// Blob represents a Git object.
type Blob struct {
ID SHA1
gogitEncodedObj plumbing.EncodedObject
name string
}
// DataAsync gets a ReadCloser for the contents of a blob without reading it all.
// Calling the Close function on the result will discard all unread output.
func (b *Blob) DataAsync() (io.ReadCloser, error) {
return b.gogitEncodedObj.Reader()
}
// Size returns the uncompressed size of the blob
func (b *Blob) Size() int64 {
return b.gogitEncodedObj.Size()
}
// This file contains common functions between the gogit and !gogit variants for git Blobs
// Name returns name of the tree entry this blob object was created from (or empty string)
func (b *Blob) Name() string {
......
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"io"
"github.com/go-git/go-git/v5/plumbing"
)
// Blob represents a Git object.
type Blob struct {
ID SHA1
gogitEncodedObj plumbing.EncodedObject
name string
}
// DataAsync gets a ReadCloser for the contents of a blob without reading it all.
// Calling the Close function on the result will discard all unread output.
func (b *Blob) DataAsync() (io.ReadCloser, error) {
return b.gogitEncodedObj.Reader()
}
// Size returns the uncompressed size of the blob
func (b *Blob) Size() int64 {
return b.gogitEncodedObj.Size()
}
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bufio"
"io"
"strconv"
"strings"
)
// Blob represents a Git object.
type Blob struct {
ID SHA1
gotSize bool
size int64
repoPath string
name string
}
// DataAsync gets a ReadCloser for the contents of a blob without reading it all.
// Calling the Close function on the result will discard all unread output.
func (b *Blob) DataAsync() (io.ReadCloser, error) {
stdoutReader, stdoutWriter := io.Pipe()
var err error
go func() {
stderr := &strings.Builder{}
err = NewCommand("cat-file", "--batch").RunInDirFullPipeline(b.repoPath, stdoutWriter, stderr, strings.NewReader(b.ID.String()+"\n"))
if err != nil {
err = ConcatenateError(err, stderr.String())
_ = stdoutWriter.CloseWithError(err)
} else {
_ = stdoutWriter.Close()
}
}()
bufReader := bufio.NewReader(stdoutReader)
_, _, size, err := ReadBatchLine(bufReader)
if err != nil {
stdoutReader.Close()
return nil, err
}
return &LimitedReaderCloser{
R: bufReader,
C: stdoutReader,
N: int64(size),
}, err
}
// Size returns the uncompressed size of the blob
func (b *Blob) Size() int64 {
if b.gotSize {
return b.size
}
size, err := NewCommand("cat-file", "-s", b.ID.String()).RunInDir(b.repoPath)
if err != nil {
log("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repoPath, err)
return 0
}
b.size, err = strconv.ParseInt(size[:len(size)-1], 10, 64)
if err != nil {
log("error whilst parsing size %s for %s in %s. Error: %v", size, b.ID.String(), b.repoPath, err)
return 0
}
b.gotSize = true
return b.size
}
......@@ -189,7 +189,7 @@ func (c *Command) RunInDirTimeoutEnv(env []string, timeout time.Duration, dir st
stdout := new(bytes.Buffer)
stderr := new(bytes.Buffer)
if err := c.RunInDirTimeoutEnvPipeline(env, timeout, dir, stdout, stderr); err != nil {
return nil, concatenateError(err, stderr.String())
return nil, ConcatenateError(err, stderr.String())
}
if stdout.Len() > 0 {
......
......@@ -19,8 +19,6 @@ import (
"net/http"
"strconv"
"strings"
"github.com/go-git/go-git/v5/plumbing/object"
)
// Commit represents a git commit.
......@@ -43,61 +41,6 @@ type CommitGPGSignature struct {
Payload string //TODO check if can be reconstruct from the rest of commit information to not have duplicate data
}
func convertPGPSignature(c *object.Commit) *CommitGPGSignature {
if c.PGPSignature == "" {
return nil
}
var w strings.Builder
var err error
if _, err = fmt.Fprintf(&w, "tree %s\n", c.TreeHash.String()); err != nil {
return nil
}
for _, parent := range c.ParentHashes {
if _, err = fmt.Fprintf(&w, "parent %s\n", parent.String()); err != nil {
return nil
}
}
if _, err = fmt.Fprint(&w, "author "); err != nil {
return nil
}
if err = c.Author.Encode(&w); err != nil {
return nil
}
if _, err = fmt.Fprint(&w, "\ncommitter "); err != nil {
return nil
}
if err = c.Committer.Encode(&w); err != nil {
return nil
}
if _, err = fmt.Fprintf(&w, "\n\n%s", c.Message); err != nil {
return nil
}
return &CommitGPGSignature{
Signature: c.PGPSignature,
Payload: w.String(),
}
}
func convertCommit(c *object.Commit) *Commit {
return &Commit{
ID: c.Hash,
CommitMessage: c.Message,
Committer: &c.Committer,
Author: &c.Author,
Signature: convertPGPSignature(c),
Parents: c.ParentHashes,
}
}
// Message returns the commit message. Same as retrieving CommitMessage directly.
func (c *Commit) Message() string {
return c.CommitMessage
......@@ -576,7 +519,7 @@ func GetCommitFileStatus(repoPath, commitID string) (*CommitFileStatus, error) {
err := NewCommand("show", "--name-status", "--pretty=format:''", commitID).RunInDirPipeline(repoPath, w, stderr)
w.Close() // Close writer to exit parsing goroutine
if err != nil {
return nil, concatenateError(err, stderr.String())
return nil, ConcatenateError(err, stderr.String())
}
<-done
......
// Copyright 2015 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"fmt"
"strings"
"github.com/go-git/go-git/v5/plumbing/object"
)
func convertPGPSignature(c *object.Commit) *CommitGPGSignature {
if c.PGPSignature == "" {
return nil
}
var w strings.Builder
var err error
if _, err = fmt.Fprintf(&w, "tree %s\n", c.TreeHash.String()); err != nil {
return nil
}
for _, parent := range c.ParentHashes {
if _, err = fmt.Fprintf(&w, "parent %s\n", parent.String()); err != nil {
return nil
}
}
if _, err = fmt.Fprint(&w, "author "); err != nil {
return nil
}
if err = c.Author.Encode(&w); err != nil {
return nil
}
if _, err = fmt.Fprint(&w, "\ncommitter "); err != nil {
return nil
}
if err = c.Committer.Encode(&w); err != nil {
return nil
}
if _, err = fmt.Fprintf(&w, "\n\n%s", c.Message); err != nil {
return nil
}
return &CommitGPGSignature{
Signature: c.PGPSignature,
Payload: w.String(),
}
}
func convertCommit(c *object.Commit) *Commit {
return &Commit{
ID: c.Hash,
CommitMessage: c.Message,
Committer: &c.Committer,
Author: &c.Author,
Signature: convertPGPSignature(c),
Parents: c.ParentHashes,
}
}
......@@ -4,286 +4,9 @@
package git
import (
"path"
"github.com/emirpasic/gods/trees/binaryheap"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph"
)
// GetCommitsInfo gets information of all commits that are corresponding to these entries
func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache LastCommitCache) ([][]interface{}, *Commit, error) {
entryPaths := make([]string, len(tes)+1)
// Get the commit for the treePath itself
entryPaths[0] = ""
for i, entry := range tes {
entryPaths[i+1] = entry.Name()
}
commitNodeIndex, commitGraphFile := commit.repo.CommitNodeIndex()
if commitGraphFile != nil {
defer commitGraphFile.Close()
}
c, err := commitNodeIndex.Get(commit.ID)
if err != nil {
return nil, nil, err
}
var revs map[string]*object.Commit
if cache != nil {
var unHitPaths []string
revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache)
if err != nil {
return nil, nil, err
}
if len(unHitPaths) > 0 {
revs2, err := GetLastCommitForPaths(c, treePath, unHitPaths)
if err != nil {
return nil, nil, err
}
for k, v := range revs2 {
if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil {
return nil, nil, err
}
revs[k] = v
}
}
} else {
revs, err = GetLastCommitForPaths(c, treePath, entryPaths)
}
if err != nil {
return nil, nil, err
}
commit.repo.gogitStorage.Close()
commitsInfo := make([][]interface{}, len(tes))
for i, entry := range tes {
if rev, ok := revs[entry.Name()]; ok {
entryCommit := convertCommit(rev)
if entry.IsSubModule() {
subModuleURL := ""
var fullPath string
if len(treePath) > 0 {
fullPath = treePath + "/" + entry.Name()
} else {
fullPath = entry.Name()
}
if subModule, err := commit.GetSubModule(fullPath); err != nil {
return nil, nil, err
} else if subModule != nil {
subModuleURL = subModule.URL
}
subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String())
commitsInfo[i] = []interface{}{entry, subModuleFile}
} else {
commitsInfo[i] = []interface{}{entry, entryCommit}
}
} else {
commitsInfo[i] = []interface{}{entry, nil}
}
}
// Retrieve the commit for the treePath itself (see above). We basically
// get it for free during the tree traversal and it's used for listing
// pages to display information about newest commit for a given path.
var treeCommit *Commit
if treePath == "" {
treeCommit = commit
} else if rev, ok := revs[""]; ok {
treeCommit = convertCommit(rev)
treeCommit.repo = commit.repo
}
return commitsInfo, treeCommit, nil
}
type commitAndPaths struct {
commit cgobject.CommitNode
// Paths that are still on the branch represented by commit
paths []string
// Set of hashes for the paths
hashes map[string]plumbing.Hash
}
func getCommitTree(c cgobject.CommitNode, treePath string) (*object.Tree, error) {
tree, err := c.Tree()
if err != nil {
return nil, err
}
// Optimize deep traversals by focusing only on the specific tree
if treePath != "" {
tree, err = tree.Tree(treePath)
if err != nil {
return nil, err
}
}
return tree, nil
}
func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[string]plumbing.Hash, error) {
tree, err := getCommitTree(c, treePath)
if err == object.ErrDirectoryNotFound {
// The whole tree didn't exist, so return empty map
return make(map[string]plumbing.Hash), nil
}
if err != nil {
return nil, err
}
hashes := make(map[string]plumbing.Hash)
for _, path := range paths {
if path != "" {
entry, err := tree.FindEntry(path)
if err == nil {
hashes[path] = entry.Hash
}
} else {
hashes[path] = tree.Hash
}
}
return hashes, nil
}
func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache LastCommitCache) (map[string]*object.Commit, []string, error) {
var unHitEntryPaths []string
var results = make(map[string]*object.Commit)
for _, p := range paths {
lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
if err != nil {
return nil, nil, err
}
if lastCommit != nil {
results[p] = lastCommit
continue
}
unHitEntryPaths = append(unHitEntryPaths, p)
}
return results, unHitEntryPaths, nil
}
// GetLastCommitForPaths returns last commit information
func GetLastCommitForPaths(c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
// We do a tree traversal with nodes sorted by commit time
heap := binaryheap.NewWith(func(a, b interface{}) int {
if a.(*commitAndPaths).commit.CommitTime().Before(b.(*commitAndPaths).commit.CommitTime()) {
return 1
}
return -1
})
resultNodes := make(map[string]cgobject.CommitNode)
initialHashes, err := getFileHashes(c, treePath, paths)
if err != nil {
return nil, err
}
// Start search from the root commit and with full set of paths
heap.Push(&commitAndPaths{c, paths, initialHashes})
for {
cIn, ok := heap.Pop()
if !ok {
break
}
current := cIn.(*commitAndPaths)
// Load the parent commits for the one we are currently examining
numParents := current.commit.NumParents()
var parents []cgobject.CommitNode
for i := 0; i < numParents; i++ {
parent, err := current.commit.ParentNode(i)
if err != nil {
break
}
parents = append(parents, parent)
}
// Examine the current commit and set of interesting paths
pathUnchanged := make([]bool, len(current.paths))
parentHashes := make([]map[string]plumbing.Hash, len(parents))
for j, parent := range parents {
parentHashes[j], err = getFileHashes(parent, treePath, current.paths)
if err != nil {
break
}
for i, path := range current.paths {
if parentHashes[j][path] == current.hashes[path] {
pathUnchanged[i] = true
}
}
}
var remainingPaths []string
for i, path := range current.paths {
// The results could already contain some newer change for the same path,
// so don't override that and bail out on the file early.
if resultNodes[path] == nil {
if pathUnchanged[i] {
// The path existed with the same hash in at least one parent so it could
// not have been changed in this commit directly.
remainingPaths = append(remainingPaths, path)
} else {
// There are few possible cases how can we get here:
// - The path didn't exist in any parent, so it must have been created by
// this commit.
// - The path did exist in the parent commit, but the hash of the file has
// changed.
// - We are looking at a merge commit and the hash of the file doesn't
// match any of the hashes being merged. This is more common for directories,
// but it can also happen if a file is changed through conflict resolution.
resultNodes[path] = current.commit
}
}
}
if len(remainingPaths) > 0 {
// Add the parent nodes along with remaining paths to the heap for further
// processing.
for j, parent := range parents {
// Combine remainingPath with paths available on the parent branch
// and make union of them
remainingPathsForParent := make([]string, 0, len(remainingPaths))
newRemainingPaths := make([]string, 0, len(remainingPaths))
for _, path := range remainingPaths {
if parentHashes[j][path] == current.hashes[path] {
remainingPathsForParent = append(remainingPathsForParent, path)
} else {
newRemainingPaths = append(newRemainingPaths, path)
}
}
if remainingPathsForParent != nil {
heap.Push(&commitAndPaths{parent, remainingPathsForParent, parentHashes[j]})
}
if len(newRemainingPaths) == 0 {
break
} else {
remainingPaths = newRemainingPaths
}
}
}
}
// Post-processing
result := make(map[string]*object.Commit)
for path, commitNode := range resultNodes {
var err error
result[path], err = commitNode.Commit()
if err != nil {
return nil, err
}
}
return result, nil
// CommitInfo describes the first commit with the provided entry
type CommitInfo struct {
Entry *TreeEntry
Commit *Commit
SubModuleFile *SubModuleFile
}
// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build gogit
package git
import (
"path"
"github.com/emirpasic/gods/trees/binaryheap"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph"
)
// GetCommitsInfo gets information of all commits that are corresponding to these entries
func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) {
entryPaths := make([]string, len(tes)+1)
// Get the commit for the treePath itself
entryPaths[0] = ""
for i, entry := range tes {
entryPaths[i+1] = entry.Name()
}
commitNodeIndex, commitGraphFile := commit.repo.CommitNodeIndex()
if commitGraphFile != nil {
defer commitGraphFile.Close()
}
c, err := commitNodeIndex.Get(commit.ID)
if err != nil {
return nil, nil, err
}
var revs map[string]*object.Commit
if cache != nil {
var unHitPaths []string
revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache)
if err != nil {
return nil, nil, err
}
if len(unHitPaths) > 0 {
revs2, err := GetLastCommitForPaths(c, treePath, unHitPaths)
if err != nil {
return nil, nil, err
}
for k, v := range revs2 {
if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil {
return nil, nil, err
}
revs[k] = v
}
}
} else {
revs, err = GetLastCommitForPaths(c, treePath, entryPaths)
}
if err != nil {
return nil, nil, err
}
commit.repo.gogitStorage.Close()
commitsInfo := make([]CommitInfo, len(tes))
for i, entry := range tes {
commitsInfo[i] = CommitInfo{
Entry: entry,
}
if rev, ok := revs[entry.Name()]; ok {
entryCommit := convertCommit(rev)
commitsInfo[i].Commit = entryCommit
if entry.IsSubModule() {
subModuleURL := ""
var fullPath string
if len(treePath) > 0 {
fullPath = treePath + "/" + entry.Name()
} else {
fullPath = entry.Name()
}
if subModule, err := commit.GetSubModule(fullPath); err != nil {
return nil, nil, err
} else if subModule != nil {
subModuleURL = subModule.URL
}
subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String())
commitsInfo[i].SubModuleFile = subModuleFile
}
}
}
// Retrieve the commit for the treePath itself (see above). We basically
// get it for free during the tree traversal and it's used for listing
// pages to display information about newest commit for a given path.
var treeCommit *Commit
if treePath == "" {
treeCommit = commit
} else if rev, ok := revs[""]; ok {
treeCommit = convertCommit(rev)
treeCommit.repo = commit.repo
}
return commitsInfo, treeCommit, nil
}
type commitAndPaths struct {
commit cgobject.CommitNode
// Paths that are still on the branch represented by commit
paths []string
// Set of hashes for the paths
hashes map[string]plumbing.Hash
}
func getCommitTree(c cgobject.CommitNode, treePath string) (*object.Tree, error) {
tree, err := c.Tree()
if err != nil {
return nil, err
}
// Optimize deep traversals by focusing only on the specific tree
if treePath != "" {
tree, err = tree.Tree(treePath)
if err != nil {
return nil, err
}
}
return tree, nil
}
func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[string]plumbing.Hash, error) {
tree, err := getCommitTree(c, treePath)
if err == object.ErrDirectoryNotFound {
// The whole tree didn't exist, so return empty map
return make(map[string]plumbing.Hash), nil
}
if err != nil {
return nil, err
}
hashes := make(map[string]plumbing.Hash)
for _, path := range paths {
if path != "" {
entry, err := tree.FindEntry(path)
if err == nil {
hashes[path] = entry.Hash
}
} else {
hashes[path] = tree.Hash
}
}
return hashes, nil
}
func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*object.Commit, []string, error) {
var unHitEntryPaths []string
var results = make(map[string]*object.Commit)
for _, p := range paths {
lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
if err != nil {
return nil, nil, err
}
if lastCommit != nil {
results[p] = lastCommit.(*object.Commit)
continue
}
unHitEntryPaths = append(unHitEntryPaths, p)
}
return results, unHitEntryPaths, nil
}
// GetLastCommitForPaths returns last commit information
func GetLastCommitForPaths(c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) {
// We do a tree traversal with nodes sorted by commit time
heap := binaryheap.NewWith(func(a, b interface{}) int {
if a.(*commitAndPaths).commit.CommitTime().Before(b.(*commitAndPaths).commit.CommitTime()) {
return 1
}
return -1
})
resultNodes := make(map[string]cgobject.CommitNode)
initialHashes, err := getFileHashes(c, treePath, paths)
if err != nil {
return nil, err
}
// Start search from the root commit and with full set of paths
heap.Push(&commitAndPaths{c, paths, initialHashes})
for {
cIn, ok := heap.Pop()
if !ok {
break
}
current := cIn.(*commitAndPaths)
// Load the parent commits for the one we are currently examining
numParents := current.commit.NumParents()
var parents []cgobject.CommitNode
for i := 0; i < numParents; i++ {
parent, err := current.commit.ParentNode(i)
if err != nil {
break
}
parents = append(parents, parent)
}
// Examine the current commit and set of interesting paths
pathUnchanged := make([]bool, len(current.paths))
parentHashes := make([]map[string]plumbing.Hash, len(parents))
for j, parent := range parents {
parentHashes[j], err = getFileHashes(parent, treePath, current.paths)
if err != nil {
break
}
for i, path := range current.paths {
if parentHashes[j][path] == current.hashes[path] {
pathUnchanged[i] = true
}
}
}
var remainingPaths []string
for i, path := range current.paths {
// The results could already contain some newer change for the same path,
// so don't override that and bail out on the file early.
if resultNodes[path] == nil {
if pathUnchanged[i] {
// The path existed with the same hash in at least one parent so it could
// not have been changed in this commit directly.
remainingPaths = append(remainingPaths, path)
} else {
// There are few possible cases how can we get here:
// - The path didn't exist in any parent, so it must have been created by
// this commit.
// - The path did exist in the parent commit, but the hash of the file has
// changed.
// - We are looking at a merge commit and the hash of the file doesn't
// match any of the hashes being merged. This is more common for directories,
// but it can also happen if a file is changed through conflict resolution.
resultNodes[path] = current.commit
}
}
}
if len(remainingPaths) > 0 {
// Add the parent nodes along with remaining paths to the heap for further
// processing.
for j, parent := range parents {
// Combine remainingPath with paths available on the parent branch
// and make union of them
remainingPathsForParent := make([]string, 0, len(remainingPaths))
newRemainingPaths := make([]string, 0, len(remainingPaths))
for _, path := range remainingPaths {
if parentHashes[j][path] == current.hashes[path] {
remainingPathsForParent = append(remainingPathsForParent, path)
} else {
newRemainingPaths = append(newRemainingPaths, path)
}
}
if remainingPathsForParent != nil {
heap.Push(&commitAndPaths{parent, remainingPathsForParent, parentHashes[j]})
}
if len(newRemainingPaths) == 0 {
break
} else {
remainingPaths = newRemainingPaths
}
}
}
}
// Post-processing
result := make(map[string]*object.Commit)
for path, commitNode := range resultNodes {
var err error
result[path], err = commitNode.Commit()
if err != nil {
return nil, err
}
}
return result, nil
}
// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"bufio"
"bytes"
"fmt"
"io"
"math"
"path"
"sort"
"strings"
)
// GetCommitsInfo gets information of all commits that are corresponding to these entries
func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) {
entryPaths := make([]string, len(tes)+1)
// Get the commit for the treePath itself
entryPaths[0] = ""
for i, entry := range tes {
entryPaths[i+1] = entry.Name()
}
var err error
var revs map[string]*Commit
if cache != nil {
var unHitPaths []string
revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache)
if err != nil {
return nil, nil, err
}
if len(unHitPaths) > 0 {
sort.Strings(unHitPaths)
commits, err := GetLastCommitForPaths(commit, treePath, unHitPaths)
if err != nil {
return nil, nil, err
}
for i, found := range commits {
if err := cache.Put(commit.ID.String(), path.Join(treePath, unHitPaths[i]), found.ID.String()); err != nil {
return nil, nil, err
}
revs[unHitPaths[i]] = found
}
}
} else {
sort.Strings(entryPaths)
revs = map[string]*Commit{}
var foundCommits []*Commit
foundCommits, err = GetLastCommitForPaths(commit, treePath, entryPaths)
for i, found := range foundCommits {
revs[entryPaths[i]] = found
}
}
if err != nil {
return nil, nil, err
}
commitsInfo := make([]CommitInfo, len(tes))
for i, entry := range tes {
commitsInfo[i] = CommitInfo{
Entry: entry,
}
if entryCommit, ok := revs[entry.Name()]; ok {
commitsInfo[i].Commit = entryCommit
if entry.IsSubModule() {
subModuleURL := ""
var fullPath string
if len(treePath) > 0 {
fullPath = treePath + "/" + entry.Name()
} else {
fullPath = entry.Name()
}
if subModule, err := commit.GetSubModule(fullPath); err != nil {
return nil, nil, err
} else if subModule != nil {
subModuleURL = subModule.URL
}
subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String())
commitsInfo[i].SubModuleFile = subModuleFile
}
}
}
// Retrieve the commit for the treePath itself (see above). We basically
// get it for free during the tree traversal and it's used for listing
// pages to display information about newest commit for a given path.
var treeCommit *Commit
var ok bool
if treePath == "" {
treeCommit = commit
} else if treeCommit, ok = revs[""]; ok {
treeCommit.repo = commit.repo
}
return commitsInfo, treeCommit, nil
}
func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*Commit, []string, error) {
var unHitEntryPaths []string
var results = make(map[string]*Commit)
for _, p := range paths {
lastCommit, err := cache.Get(commitID, path.Join(treePath, p))
if err != nil {
return nil, nil, err
}
if lastCommit != nil {
results[p] = lastCommit.(*Commit)
continue
}
unHitEntryPaths = append(unHitEntryPaths, p)
}
return results, unHitEntryPaths, nil
}
// GetLastCommitForPaths returns last commit information
func GetLastCommitForPaths(commit *Commit, treePath string, paths []string) ([]*Commit, error) {
// We read backwards from the commit to obtain all of the commits
// We'll do this by using rev-list to provide us with parent commits in order
revListReader, revListWriter := io.Pipe()
defer func() {
_ = revListWriter.Close()
_ = revListReader.Close()
}()
go func() {
stderr := strings.Builder{}
err := NewCommand("rev-list", "--format=%T", commit.ID.String()).RunInDirPipeline(commit.repo.Path, revListWriter, &stderr)
if err != nil {
_ = revListWriter.CloseWithError(ConcatenateError(err, (&stderr).String()))
} else {
_ = revListWriter.Close()
}
}()
// We feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary.
// so let's create a batch stdin and stdout
batchStdinReader, batchStdinWriter := io.Pipe()
batchStdoutReader, batchStdoutWriter := io.Pipe()
defer func() {
_ = batchStdinReader.Close()
_ = batchStdinWriter.Close()
_ = batchStdoutReader.Close()
_ = batchStdoutWriter.Close()
}()
go func() {
stderr := strings.Builder{}
err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(commit.repo.Path, batchStdoutWriter, &stderr, batchStdinReader)
if err != nil {
_ = revListWriter.CloseWithError(ConcatenateError(err, (&stderr).String()))
} else {
_ = revListWriter.Close()
}
}()
// For simplicities sake we'll us a buffered reader
batchReader := bufio.NewReader(batchStdoutReader)
mapsize := 4096
if len(paths) > mapsize {
mapsize = len(paths)
}
path2idx := make(map[string]int, mapsize)
for i, path := range paths {
path2idx[path] = i
}
fnameBuf := make([]byte, 4096)
modeBuf := make([]byte, 40)
allShaBuf := make([]byte, (len(paths)+1)*20)
shaBuf := make([]byte, 20)
tmpTreeID := make([]byte, 40)
// commits is the returnable commits matching the paths provided
commits := make([]string, len(paths))
// ids are the blob/tree ids for the paths
ids := make([][]byte, len(paths))
// We'll use a scanner for the revList because it's simpler than a bufio.Reader
scan := bufio.NewScanner(revListReader)
revListLoop:
for scan.Scan() {
// Get the next parent commit ID
commitID := scan.Text()
if !scan.Scan() {
break revListLoop
}
commitID = commitID[7:]
rootTreeID := scan.Text()
// push the tree to the cat-file --batch process
_, err := batchStdinWriter.Write([]byte(rootTreeID + "\n"))
if err != nil {
return nil, err
}
currentPath := ""
// OK if the target tree path is "" and the "" is in the paths just set this now
if treePath == "" && paths[0] == "" {
// If this is the first time we see this set the id appropriate for this paths to this tree and set the last commit to curCommit
if len(ids[0]) == 0 {
ids[0] = []byte(rootTreeID)
commits[0] = string(commitID)
} else if bytes.Equal(ids[0], []byte(rootTreeID)) {
commits[0] = string(commitID)
}
}
treeReadingLoop:
for {
_, _, size, err := ReadBatchLine(batchReader)
if err != nil {
return nil, err
}
// Handle trees
// n is counter for file position in the tree file
var n int64
// Two options: currentPath is the targetTreepath
if treePath == currentPath {
// We are in the right directory
// Parse each tree line in turn. (don't care about mode here.)
for n < size {
fname, sha, count, err := ParseTreeLineSkipMode(batchReader, fnameBuf, shaBuf)
shaBuf = sha
if err != nil {
return nil, err
}
n += int64(count)
idx, ok := path2idx[string(fname)]
if ok {
// Now if this is the first time round set the initial Blob(ish) SHA ID and the commit
if len(ids[idx]) == 0 {
copy(allShaBuf[20*(idx+1):20*(idx+2)], shaBuf)
ids[idx] = allShaBuf[20*(idx+1) : 20*(idx+2)]
commits[idx] = string(commitID)
} else if bytes.Equal(ids[idx], shaBuf) {
commits[idx] = string(commitID)
}
}
// FIXME: is there any order to the way strings are emitted from cat-file?
// if there is - then we could skip once we've passed all of our data
}
break treeReadingLoop
}
var treeID []byte
// We're in the wrong directory
// Find target directory in this directory
idx := len(currentPath)
if idx > 0 {
idx++
}
target := strings.SplitN(treePath[idx:], "/", 2)[0]
for n < size {
// Read each tree entry in turn
mode, fname, sha, count, err := ParseTreeLine(batchReader, modeBuf, fnameBuf, shaBuf)
if err != nil {
return nil, err
}
n += int64(count)
// if we have found the target directory
if bytes.Equal(fname, []byte(target)) && bytes.Equal(mode, []byte("40000")) {
copy(tmpTreeID, sha)
treeID = tmpTreeID
break
}
}
if n < size {
// Discard any remaining entries in the current tree
discard := size - n
for discard > math.MaxInt32 {
_, err := batchReader.Discard(math.MaxInt32)
if err != nil {
return nil, err
}
discard -= math.MaxInt32
}
_, err := batchReader.Discard(int(discard))
if err != nil {
return nil, err
}
}
// if we haven't found a treeID for the target directory our search is over
if len(treeID) == 0 {
break treeReadingLoop
}
// add the target to the current path
if idx > 0 {
currentPath += "/"
}
currentPath += target
// if we've now found the current path check its sha id and commit status
if treePath == currentPath && paths[0] == "" {
if len(ids[0]) == 0 {
copy(allShaBuf[0:20], treeID)
ids[0] = allShaBuf[0:20]
commits[0] = string(commitID)
} else if bytes.Equal(ids[0], treeID) {
commits[0] = string(commitID)
}
}
treeID = to40ByteSHA(treeID)
_, err = batchStdinWriter.Write(treeID)
if err != nil {
return nil, err
}
_, err = batchStdinWriter.Write([]byte("\n"))
if err != nil {
return nil, err
}
}
}
commitsMap := make(map[string]*Commit, len(commits))
commitsMap[commit.ID.String()] = commit
commitCommits := make([]*Commit, len(commits))
for i, commitID := range commits {
c, ok := commitsMap[commitID]
if ok {
commitCommits[i] = c
continue
}
if len(commitID) == 0 {
continue
}
_, err := batchStdinWriter.Write([]byte(commitID + "\n"))
if err != nil {
return nil, err
}
_, typ, size, err := ReadBatchLine(batchReader)
if err != nil {
return nil, err
}
if typ != "commit" {
return nil, fmt.Errorf("unexpected type: %s for commit id: %s", typ, commitID)
}
c, err = CommitFromReader(commit.repo, MustIDFromString(string(commitID)), io.LimitReader(batchReader, int64(size)))
if err != nil {
return nil, err
}
commitCommits[i] = c
}
return commitCommits, scan.Err()
}
......@@ -58,17 +58,27 @@ func testGetCommitsInfo(t *testing.T, repo1 *Repository) {
for _, testCase := range testCases {
commit, err := repo1.GetCommit(testCase.CommitID)
assert.NoError(t, err)
assert.NotNil(t, commit)
assert.NotNil(t, commit.Tree)
assert.NotNil(t, commit.Tree.repo)
tree, err := commit.Tree.SubTree(testCase.Path)
assert.NotNil(t, tree, "tree is nil for testCase CommitID %s in Path %s", testCase.CommitID, testCase.Path)
assert.NotNil(t, tree.repo, "repo is nil for testCase CommitID %s in Path %s", testCase.CommitID, testCase.Path)
assert.NoError(t, err)
entries, err := tree.ListEntries()
assert.NoError(t, err)
commitsInfo, treeCommit, err := entries.GetCommitsInfo(commit, testCase.Path, nil)
assert.Equal(t, testCase.ExpectedTreeCommit, treeCommit.ID.String())
assert.NoError(t, err)
if err != nil {
t.FailNow()
}
assert.Equal(t, testCase.ExpectedTreeCommit, treeCommit.ID.String())
assert.Len(t, commitsInfo, len(testCase.ExpectedIDs))
for _, commitInfo := range commitsInfo {
entry := commitInfo[0].(*TreeEntry)
commit := commitInfo[1].(*Commit)
entry := commitInfo.Entry
commit := commitInfo.Commit
expectedID, ok := testCase.ExpectedIDs[entry.Name()]
if !assert.True(t, ok) {
continue
......
......@@ -9,13 +9,13 @@ import (
"bytes"
"io"
"strings"
"github.com/go-git/go-git/v5/plumbing"
)
// CommitFromReader will generate a Commit from a provided reader
// We will need this to interpret commits from cat-file
func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader) (*Commit, error) {
// We need this to interpret commits from cat-file or cat-file --batch
//
// If used as part of a cat-file --batch stream you need to limit the reader to the correct size
func CommitFromReader(gitRepo *Repository, sha SHA1, reader io.Reader) (*Commit, error) {
commit := &Commit{
ID: sha,
}
......@@ -26,26 +26,20 @@ func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader)
message := false
pgpsig := false
scanner := bufio.NewScanner(reader)
// Split by '\n' but include the '\n'
scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
return 0, nil, nil
}
if i := bytes.IndexByte(data, '\n'); i >= 0 {
// We have a full newline-terminated line.
return i + 1, data[0 : i+1], nil
}
// If we're at EOF, we have a final, non-terminated line. Return it.
if atEOF {
return len(data), data, nil
}
// Request more data.
return 0, nil, nil
})
bufReader, ok := reader.(*bufio.Reader)
if !ok {
bufReader = bufio.NewReader(reader)
}
for scanner.Scan() {
line := scanner.Bytes()
readLoop:
for {
line, err := bufReader.ReadBytes('\n')
if err != nil {
if err == io.EOF {
break readLoop
}
return nil, err
}
if pgpsig {
if len(line) > 0 && line[0] == ' ' {
_, _ = signatureSB.Write(line[1:])
......@@ -72,10 +66,10 @@ func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader)
switch string(split[0]) {
case "tree":
commit.Tree = *NewTree(gitRepo, plumbing.NewHash(string(data)))
commit.Tree = *NewTree(gitRepo, MustIDFromString(string(data)))
_, _ = payloadSB.Write(line)
case "parent":
commit.Parents = append(commit.Parents, plumbing.NewHash(string(data)))
commit.Parents = append(commit.Parents, MustIDFromString(string(data)))
_, _ = payloadSB.Write(line)
case "author":
commit.Author = &Signature{}
......@@ -104,5 +98,5 @@ func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader)
commit.Signature = nil
}
return commit, scanner.Err()
return commit, nil
}
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package git
import (
"crypto/sha256"
"fmt"
)
// Cache represents a caching interface
type Cache interface {
// Put puts value into cache with key and expire time.
Put(key string, val interface{}, timeout int64) error
// Get gets cached value by given key.
Get(key string) interface{}
}
func (c *LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string {
hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, ref, entryPath)))
return fmt.Sprintf("last_commit:%x", hashBytes)
}
// Put put the last commit id with commit and entry path
func (c *LastCommitCache) Put(ref, entryPath, commitID string) error {
log("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID)
return c.cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl)
}
......@@ -2,51 +2,47 @@
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package cache
// +build gogit
import (
"crypto/sha256"
"fmt"
package git
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
import (
"path"
mc "gitea.com/macaron/cache"
"github.com/go-git/go-git/v5/plumbing/object"
cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph"
)
// LastCommitCache represents a cache to store last commit
type LastCommitCache struct {
repoPath string
ttl int64
repo *git.Repository
repo *Repository
commitCache map[string]*object.Commit
mc.Cache
cache Cache
}
// NewLastCommitCache creates a new last commit cache for repo
func NewLastCommitCache(repoPath string, gitRepo *git.Repository, ttl int64) *LastCommitCache {
func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl int64, cache Cache) *LastCommitCache {
if cache == nil {
return nil
}
return &LastCommitCache{
repoPath: repoPath,
repo: gitRepo,
commitCache: make(map[string]*object.Commit),
ttl: ttl,
Cache: conn,
cache: cache,
}
}
func (c LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string {
hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, ref, entryPath)))
return fmt.Sprintf("last_commit:%x", hashBytes)
}
// Get get the last commit information by commit id and entry path
func (c LastCommitCache) Get(ref, entryPath string) (*object.Commit, error) {
v := c.Cache.Get(c.getCacheKey(c.repoPath, ref, entryPath))
func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) {
v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath))
if vs, ok := v.(string); ok {
log.Trace("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs)
log("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs)
if commit, ok := c.commitCache[vs]; ok {
log.Trace("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs)
log("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs)
return commit, nil
}
id, err := c.repo.ConvertToSHA1(vs)
......@@ -63,8 +59,55 @@ func (c LastCommitCache) Get(ref, entryPath string) (*object.Commit, error) {
return nil, nil
}
// Put put the last commit id with commit and entry path
func (c LastCommitCache) Put(ref, entryPath, commitID string) error {
log.Trace("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID)
return c.Cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl)
// CacheCommit will cache the commit from the gitRepository
func (c *LastCommitCache) CacheCommit(commit *Commit) error {
commitNodeIndex, _ := commit.repo.CommitNodeIndex()
index, err := commitNodeIndex.Get(commit.ID)
if err != nil {
return err
}
return c.recursiveCache(index, &commit.Tree, "", 1)
}
func (c *LastCommitCache) recursiveCache(index cgobject.CommitNode, tree *Tree, treePath string, level int) error {
if level == 0 {
return nil
}
entries, err := tree.ListEntries()
if err != nil {
return err
}
entryPaths := make([]string, len(entries))
entryMap := make(map[string]*TreeEntry)
for i, entry := range entries {
entryPaths[i] = entry.Name()
entryMap[entry.Name()] = entry
}
commits, err := GetLastCommitForPaths(index, treePath, entryPaths)
if err != nil {
return err
}
for entry, cm := range commits {
if err := c.Put(index.ID().String(), path.Join(treePath, entry), cm.ID().String()); err != nil {
return err
}
if entryMap[entry].IsDir() {
subTree, err := tree.SubTree(entry)
if err != nil {
return err
}
if err := c.recursiveCache(index, subTree, entry, level-1); err != nil {
return err
}
}
}
return nil
}
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
// +build !gogit
package git
import (
"path"
)
// LastCommitCache represents a cache to store last commit
type LastCommitCache struct {
repoPath string
ttl int64
repo *Repository
commitCache map[string]*Commit
cache Cache
}
// NewLastCommitCache creates a new last commit cache for repo
func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl int64, cache Cache) *LastCommitCache {
if cache == nil {
return nil
}
return &LastCommitCache{
repoPath: repoPath,
repo: gitRepo,
commitCache: make(map[string]*Commit),
ttl: ttl,
cache: cache,
}
}
// Get get the last commit information by commit id and entry path
func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) {
v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath))
if vs, ok := v.(string); ok {
log("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs)
if commit, ok := c.commitCache[vs]; ok {
log("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs)
return commit, nil
}
id, err := c.repo.ConvertToSHA1(vs)
if err != nil {
return nil, err
}
commit, err := c.repo.getCommit(id)
if err != nil {
return nil, err
}
c.commitCache[vs] = commit
return commit, nil
}
return nil, nil
}
// CacheCommit will cache the commit from the gitRepository
func (c *LastCommitCache) CacheCommit(commit *Commit) error {
return c.recursiveCache(commit, &commit.Tree, "", 1)
}
func (c *LastCommitCache) recursiveCache(commit *Commit, tree *Tree, treePath string, level int) error {
if level == 0 {
return nil
}
entries, err := tree.ListEntries()
if err != nil {
return err
}
entryPaths := make([]string, len(entries))
entryMap := make(map[string]*TreeEntry)
for i, entry := range entries {
entryPaths[i] = entry.Name()
entryMap[entry.Name()] = entry
}
commits, err := GetLastCommitForPaths(commit, treePath, entryPaths)
if err != nil {
return err
}
for i, entryCommit := range commits {
entry := entryPaths[i]
if err := c.Put(commit.ID.String(), path.Join(treePath, entryPaths[i]), entryCommit.ID.String()); err != nil {
return err
}
if entryMap[entry].IsDir() {
subTree, err := tree.SubTree(entry)
if err != nil {
return err
}
if err := c.recursiveCache(commit, subTree, entry, level-1); err != nil {
return err
}
}
}
return nil
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment