Adds support to open local repositories and to use file-based object …

…storage (#55) * remove some comments * idx writer/reader * Shut up ssh tests, they are annoying * Add file scheme test to clients * Add dummy file client * Add test fot file client * Make tests use fixture endpoint * add parser for packed-refs format * add parser for packed-refs format * WIP adding dir.Refs() tests * Add test for fixture refs * refs parser for the refs directory * Documentation * Add Capabilities to file client * tgz.Exatract now accpets a path instead of a Reader * fix bug in idxfile fanout calculation * remove dead code * packfile documentation * clean packfile parser code * add core.Object.Content() and returns errors for core.ObjectStorage.Iter() * add seekable storage * add dir repos to NewRepository * clean prints * Add dir client documentation to README * Organize the README * README * Clean tgz package * Clean temp dirs after tgz tests * Gometalinter on gitdir * Clean pattern function * metalinter tgz * metalinter gitdir * gitdir coverage and remove seekable packfile filedescriptor leak * gitdir Idxfile tests and remove file descriptor leak * gitdir Idxfile tests when no idx is found * clean storage/seekable/internal/index and some formats/idxfile API issues * clean storage/seekable * clean formats/idx * turn packfile/doc.go into packfile/doc.txt * move formats/packfile/reader to decoder * fix packfile decoder error names * improve documentation * comment packfile decoder errors * comment public API (format/packfile) * remve duplicated code in packfile decoder test * move tracking_reader into an internal package and clean it * use iota for packfile format * rename packfile parse.go to packfile object_at.go * clean packfile deltas * fix delta header size bug * improve delta documentation * clean packfile deltas * clean packfiles deltas * clean repository.go * Remove go 1.5 from Travis CI Because go 1.5 does not suport internal packages. * change local repo scheme to local:// * change "local://" to "file://" as the local scheme * fix broken indentation * shortens names of variables in short scopes * more shortening of variable names * more shortening of variable names * Rename git dir client to "file", as the scheme used for it * Fix file format ctor name, now that the package name has change * Sortcut local repo constructor to not use remotes The object storage is build directly in the repository ctor, instead of creating a remote and waiting for the user to pull it. * update README and fix some errors in it * remove file scheme client * Local respositories has now a new ctor This is, they are no longer identified by the scheme of the URL, but are created different from inception. * remove unused URL field form Repository * move all git dir logic to seekable sotrage ctor * fix documentation * Make formats/file/dir an internal package to storage/seekable * change package storage/seekable to storage/fs * clean storage/fs * overall storage/fs clean * more cleaning * some metalinter fixes * upgrade cshared to last changes * remove dead code * fix test error info * remove file scheme check from clients * fix test error message * fix test error message * fix error messages * style changes * fix comments everywhere * style changes * style changes * scaffolding and tests for local packfiles without ifx files * outsource index building from packfile to the packfile decoder * refactor packfile header reading into a new function * move code to generate index from packfile back to index package * add header parsing * fix documentation errata * add undeltified and OFS delta support for index building from the packfile * add tests for packfile with ref-deltas * support for packfiles with ref-deltas and no idx * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * WIP refactor packfile format parser to reuse code * refactor packfile format parser to reuse code * remove prints from tests * remove prints from tests * refactor packfile.core into packfile.parser * rename packfile reader to something that shows it is a recaller * rename cannot recall error * rename packfile.Reader to packfile.ReadRecaller and document * speed up test by using StreamReader instead of SeekableReader when possible * clean packfile StreamReader * stream_reader tests * refactor packfile.StreamReader into packfile.StreamReadRecaller * refactor packfile.SeekableReader into packfile.SeekableReadRecaller and document it * generalize packfile.StreamReadRecaller test to all packfile.ReadRecaller implementations * speed up storage/fs tests * speed up tests in . by loading packfiles in memory * speed up repository tests by using and smaller fixture * restore doc.go files * rename packfile.ReadRecaller implementations to shorter names * update comments to type changes * packfile.Parser test (WIP) * packfile.Parser tests and add ForgetAll() to packfile.ReadRecaller * add test for packfile.ReadRecaller.ForgetAll() * clarify seekable being able to recallByOffset forgetted objects * use better names for internal maps * metalinter packfile package * speed up some tests * documentation fixes * change storage.fs package name to storage.proxy to avoid confusion with new filesystem support * New fs package and os transparent implementation Now NewRepositoryFromFS receives a fs and a path and tests are modified accordingly, but it is still not using for anything. * add fs to gitdir and proxy.store * reduce fs interface for easier implementation * remove garbage dirs from tgz tests * change file name gitdir/dir.go to gitdir/gitdir.go * fs.OS tests * metalinter utils/fs * add NewRepositoryFromFS documentation to README * Readability fixes to README * move tgz to an external dependency * move filesystem impl. example to example dir * rename proxy/store.go to proxy/storage.go for coherence with memory/storage.go * rename proxy package to seekable
src-d · Jul 4, 2016 · 5e73f01 · 5e73f01
1 parent 808076a
commit 5e73f01
Show file tree

Hide file tree

Showing 58 changed files with 6,630 additions and 924 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -2,7 +2,6 @@ language: go
 
 go:
   - 1.4
-  - 1.5
   - 1.6
   - tip
 

diff --git a/README.md b/README.md
@@ -49,7 +49,10 @@ if err := r.PullDefault(); err != nil {
 	panic(err)
 }
 
-iter := r.Commits()
+iter, err := r.Commits()
+if err != nil {
+	panic(err)
+}
 defer iter.Close()
 
 for {
@@ -112,6 +115,61 @@ if err != nil {
 fmt.Println(commit)
 ```
 
+Creating a repository from an ordinary local git directory (that has been
+previously prepared by running `git gc` on it).
+
+```go
+// Download any git repository and prepare it as as follows:
+//
+//   $ git clone https://github.com/src-d/go-git /tmp/go-git
+//   $ pushd /tmp/go-git ; git gc ; popd
+//
+// Then, create a go-git repository from the local content
+// and print its commits as follows:
+
+package main
+
+import (
+	"fmt"
+	"io"
+
+	"gopkg.in/src-d/go-git.v3"
+	"gopkg.in/src-d/go-git.v3/utils/fs"
+)
+
+func main() {
+	fs := fs.NewOS() // a simple proxy for the local host filesystem
+	path := "/tmp/go-git/.git"
+
+	repo, err := git.NewRepositoryFromFS(fs, path)
+	if err != nil {
+		panic(err)
+	}
+
+	iter, err := repo.Commits()
+	if err != nil {
+		panic(err)
+	}
+	defer iter.Close()
+
+	for {
+		commit, err := iter.Next()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			panic(err)
+		}
+
+		fmt.Println(commit)
+	}
+}
+```
+
+Implementing your own filesystem will let you access repositories stored on
+remote services (e.g. amazon S3), see the
+[examples](https://github.com/src-d/go-git/tree/master/examples/fs_implementation/)
+directory for a simple filesystem implementation and usage.
 
 Wrapping
 --------

diff --git a/blame_test.go b/blame_test.go
@@ -1,6 +1,8 @@
 package git
 
 import (
+	"bytes"
+	"io/ioutil"
 	"os"
 
 	"gopkg.in/src-d/go-git.v3/core"
@@ -19,24 +21,23 @@ var _ = Suite(&BlameCommon{})
 func (s *BlameCommon) SetUpSuite(c *C) {
 	s.repos = make(map[string]*Repository, 0)
 	for _, fixRepo := range fixtureRepos {
-		repo := NewPlainRepository()
-		repo.URL = fixRepo.url
+		r := NewPlainRepository()
 
-		d, err := os.Open(fixRepo.packfile)
+		f, err := os.Open(fixRepo.packfile)
 		c.Assert(err, IsNil)
 
-		r := packfile.NewReader(d)
-		// TODO: how to know the format of a pack file ahead of time?
-		// Some info at:
-		// https://codewords.recurse.com/issues/three/unpacking-git-packfiles
-		r.Format = packfile.OFSDeltaFormat
+		data, err := ioutil.ReadAll(f)
+		c.Assert(err, IsNil)
+
+		stream := packfile.NewStream(bytes.NewReader(data))
 
-		_, err = r.Read(repo.Storage)
+		d := packfile.NewDecoder(stream)
+		err = d.Decode(r.Storage)
 		c.Assert(err, IsNil)
 
-		c.Assert(d.Close(), IsNil)
+		c.Assert(f.Close(), IsNil)
 
-		s.repos[fixRepo.url] = repo
+		s.repos[fixRepo.url] = r
 	}
 }
 
@@ -48,22 +49,22 @@ type blameTest struct {
 }
 
 func (s *BlameCommon) mockBlame(t blameTest, c *C) (blame *Blame) {
-	repo, ok := s.repos[t.repo]
+	r, ok := s.repos[t.repo]
 	c.Assert(ok, Equals, true)
 
-	commit, err := repo.Commit(core.NewHash(t.rev))
-	c.Assert(err, IsNil, Commentf("%v: repo=%s, rev=%s", err, repo, t.rev))
+	commit, err := r.Commit(core.NewHash(t.rev))
+	c.Assert(err, IsNil, Commentf("%v: repo=%s, rev=%s", err, r, t.rev))
 
-	file, err := commit.File(t.path)
+	f, err := commit.File(t.path)
 	c.Assert(err, IsNil)
-	lines, err := file.Lines()
+	lines, err := f.Lines()
 	c.Assert(err, IsNil)
 	c.Assert(len(t.blames), Equals, len(lines), Commentf(
 		"repo=%s, path=%s, rev=%s: the number of lines in the file and the number of expected blames differ (len(blames)=%d, len(lines)=%d)\nblames=%#q\nlines=%#q", t.repo, t.path, t.rev, len(t.blames), len(lines), t.blames, lines))
 
 	blamedLines := make([]*line, 0, len(t.blames))
 	for i := range t.blames {
-		commit, err := repo.Commit(core.NewHash(t.blames[i]))
+		commit, err := r.Commit(core.NewHash(t.blames[i]))
 		c.Assert(err, IsNil)
 		l := &line{
 			author: commit.Author.Email,
@@ -82,17 +83,17 @@ func (s *BlameCommon) mockBlame(t blameTest, c *C) (blame *Blame) {
 // run a blame on all the suite's tests
 func (s *BlameCommon) TestBlame(c *C) {
 	for _, t := range blameTests {
-		expected := s.mockBlame(t, c)
+		exp := s.mockBlame(t, c)
 
-		repo, ok := s.repos[t.repo]
+		r, ok := s.repos[t.repo]
 		c.Assert(ok, Equals, true)
 
-		commit, err := repo.Commit(core.NewHash(t.rev))
+		commit, err := r.Commit(core.NewHash(t.rev))
 		c.Assert(err, IsNil)
 
-		obtained, err := commit.Blame(t.path)
+		obt, err := commit.Blame(t.path)
 		c.Assert(err, IsNil)
-		c.Assert(obtained, DeepEquals, expected)
+		c.Assert(obt, DeepEquals, exp)
 	}
 }
 
@@ -105,16 +106,18 @@ func repeat(s string, n int) []string {
 	for i := 0; i < n; i++ {
 		r = append(r, s)
 	}
+
 	return r
 }
 
 // utility function to concat slices
 func concat(vargs ...[]string) []string {
-	var result []string
+	var r []string
 	for _, ss := range vargs {
-		result = append(result, ss...)
+		r = append(r, ss...)
 	}
-	return result
+
+	return r
 }
 
 var blameTests = [...]blameTest{

diff --git a/clients/common.go b/clients/common.go
@@ -55,10 +55,10 @@ func NewGitUploadPackService(repoURL string) (common.GitUploadPackService, error
 	if err != nil {
 		return nil, fmt.Errorf("invalid url %q", repoURL)
 	}
-	service, ok := KnownProtocols[u.Scheme]
+	s, ok := KnownProtocols[u.Scheme]
 	if !ok {
 		return nil, fmt.Errorf("unsupported scheme %q", u.Scheme)
 	}
 
-	return service, nil
+	return s, nil
 }
diff --git a/clients/common/common.go b/clients/common/common.go
@@ -236,11 +236,7 @@ func (r *GitUploadPackInfo) decodeHeaderLine(line string) {
 }
 
 func (r *GitUploadPackInfo) isValidLine(line string) bool {
-	if line[0] == '#' {
-		return false
-	}
-
-	return true
+	return line[0] != '#'
 }
 
 func (r *GitUploadPackInfo) readLine(line string) {

diff --git a/clients/common_test.go b/clients/common_test.go
@@ -3,23 +3,41 @@ package clients
 import (
 	"fmt"
 	"io"
+	"os"
 	"testing"
 
-	. "gopkg.in/check.v1"
 	"gopkg.in/src-d/go-git.v3/clients/common"
+
+	"github.com/alcortesm/tgz"
+	. "gopkg.in/check.v1"
 )
 
 func Test(t *testing.T) { TestingT(t) }
 
-type SuiteCommon struct{}
+type SuiteCommon struct {
+	dirFixturePath string
+}
 
 var _ = Suite(&SuiteCommon{})
 
+const fixtureTGZ = "../storage/seekable/internal/gitdir/fixtures/spinnaker-gc.tgz"
+
+func (s *SuiteCommon) SetUpSuite(c *C) {
+	var err error
+	s.dirFixturePath, err = tgz.Extract(fixtureTGZ)
+	c.Assert(err, IsNil)
+}
+
+func (s *SuiteCommon) TearDownSuite(c *C) {
+	err := os.RemoveAll(s.dirFixturePath)
+	c.Assert(err, IsNil)
+}
+
 func (s *SuiteCommon) TestNewGitUploadPackService(c *C) {
 	var tests = [...]struct {
-		input    string
-		err      bool
-		expected string
+		input string
+		err   bool
+		exp   string
 	}{
 		{"://example.com", true, "<nil>"},
 		{"badscheme://github.com/src-d/go-git", true, "<nil>"},
@@ -30,8 +48,10 @@ func (s *SuiteCommon) TestNewGitUploadPackService(c *C) {
 
 	for i, t := range tests {
 		output, err := NewGitUploadPackService(t.input)
-		c.Assert(err != nil, Equals, t.err, Commentf("%d) %q: wrong error value", i, t.input))
-		c.Assert(typeAsString(output), Equals, t.expected, Commentf("%d) %q: wrong type", i, t.input))
+		c.Assert(err != nil, Equals, t.err,
+			Commentf("%d) %q: wrong error value (was: %s)", i, t.input, err))
+		c.Assert(typeAsString(output), Equals, t.exp,
+			Commentf("%d) %q: wrong type", i, t.input))
 	}
 }
 
@@ -70,7 +90,6 @@ func (s *SuiteCommon) TestInstallProtocol(c *C) {
 
 	for i, t := range tests {
 		if t.panic {
-			fmt.Println(t.service == nil)
 			c.Assert(func() { InstallProtocol(t.scheme, t.service) }, PanicMatches, `nil service`)
 			continue
 		}

diff --git a/clients/ssh/git_upload_pack_test.go b/clients/ssh/git_upload_pack_test.go
@@ -1,3 +1,5 @@
+// +build ssh
+
 package ssh
 
 import (

diff --git a/common_test.go b/common_test.go
@@ -1,7 +1,9 @@
 package git
 
 import (
+	"bytes"
 	"io"
+	"io/ioutil"
 	"os"
 	"testing"
 
@@ -29,21 +31,22 @@ func (s *MockGitUploadPackService) ConnectWithAuth(url common.Endpoint, auth com
 }
 
 func (s *MockGitUploadPackService) Info() (*common.GitUploadPackInfo, error) {
-	hash := core.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")
+	h := core.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")
 
-	cap := common.NewCapabilities()
-	cap.Decode("6ecf0ef2c2dffb796033e5a02219af86ec6584e5 HEADmulti_ack thin-pack side-band side-band-64k ofs-delta shallow no-progress include-tag multi_ack_detailed no-done symref=HEAD:refs/heads/master agent=git/2:2.4.8~dbussink-fix-enterprise-tokens-compilation-1167-gc7006cf")
+	c := common.NewCapabilities()
+	c.Decode("6ecf0ef2c2dffb796033e5a02219af86ec6584e5 HEADmulti_ack thin-pack side-band side-band-64k ofs-delta shallow no-progress include-tag multi_ack_detailed no-done symref=HEAD:refs/heads/master agent=git/2:2.4.8~dbussink-fix-enterprise-tokens-compilation-1167-gc7006cf")
 
 	return &common.GitUploadPackInfo{
-		Capabilities: cap,
-		Head:         hash,
-		Refs:         map[string]core.Hash{"refs/heads/master": hash},
+		Capabilities: c,
+		Head:         h,
+		Refs:         map[string]core.Hash{"refs/heads/master": h},
 	}, nil
 }
 
 func (s *MockGitUploadPackService) Fetch(*common.GitUploadPackRequest) (io.ReadCloser, error) {
 	var err error
 	s.RC, err = os.Open("formats/packfile/fixtures/git-fixture.ref-delta")
+
 	return s.RC, err
 }
 
@@ -65,20 +68,28 @@ func unpackFixtures(c *C, fixtures ...[]packedFixture) map[string]*Repository {
 			if _, existing := repos[fixture.url]; existing {
 				continue
 			}
-			repos[fixture.url] = NewPlainRepository()
 
-			d, err := os.Open(fixture.packfile)
-			c.Assert(err, IsNil)
+			comment := Commentf("fixture packfile: %q", fixture.packfile)
 
-			r := packfile.NewReader(d)
-			r.Format = packfile.OFSDeltaFormat // This is hardcoded because we don't have a good way to sniff the format
+			repos[fixture.url] = NewPlainRepository()
 
-			_, err = r.Read(repos[fixture.url].Storage)
+			f, err := os.Open(fixture.packfile)
+			c.Assert(err, IsNil, comment)
+
+			// increase memory consumption to speed up tests
+			data, err := ioutil.ReadAll(f)
 			c.Assert(err, IsNil)
+			memStream := bytes.NewReader(data)
+			r := packfile.NewStream(memStream)
+
+			d := packfile.NewDecoder(r)
+			err = d.Decode(repos[fixture.url].Storage)
+			c.Assert(err, IsNil, comment)
 
-			c.Assert(d.Close(), IsNil)
+			c.Assert(f.Close(), IsNil, comment)
 		}
 	}
+
 	return repos
 }
 

diff --git a/core/object.go b/core/object.go
@@ -34,16 +34,16 @@ type Object interface {
 	SetType(ObjectType)
 	Size() int64
 	SetSize(int64)
+	Content() []byte
 	Reader() (ObjectReader, error)
 	Writer() (ObjectWriter, error)
 }
 
 // ObjectStorage generic storage of objects
 type ObjectStorage interface {
-	New() (Object, error)
 	Set(Object) (Hash, error)
 	Get(Hash) (Object, error)
-	Iter(ObjectType) ObjectIter
+	Iter(ObjectType) (ObjectIter, error)
 }
 
 // ObjectType internal object type's