Merge branch 'master' into integration_test/883

Author: ajnavarro

CHANGELOG.md

@@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- git libraries bare or non bare format is automatically detected ([#897](https://github.com/src-d/gitbase/pull/897))
+- Fix bug that created multiple object cache with incorrect size ([#898](https://github.com/src-d/gitbase/pull/898))
+
 ## [0.22.0-beta1] - 2019-06-20
 
 ### Added

cmd/gitbase/command/server.go

@@ -59,6 +59,7 @@ type Server struct {
 	Format        string         `long:"format" default:"git" choice:"git" choice:"siva" description:"Library format"`
 	Bucket        int            `long:"bucket" default:"2" description:"Bucketing level to use with siva libraries"`
 	Bare          bool           `long:"bare" description:"Sets the library to use bare git repositories, used only with git format libraries"`
+	NonBare       bool           `long:"non-bare" description:"Sets the library to use non bare git repositories, used only with git format libraries"`
 	NonRooted     bool           `long:"non-rooted" description:"Disables treating siva files as rooted repositories"`
 	Host          string         `long:"host" default:"localhost" description:"Host where the server is going to listen"`
 	Port          int            `short:"p" long:"port" default:"3306" description:"Port where the server is going to listen"`
@@ -122,6 +123,10 @@ func (c *Server) Execute(args []string) error {
 		logrus.SetLevel(logrus.DebugLevel)
 	}
 
+	if c.Bare && c.NonBare {
+		return fmt.Errorf("cannot use both --bare and --non-bare")
+	}
+
 	// info is the default log level
 	if c.LogLevel != "info" {
 		level, err := logrus.ParseLevel(c.LogLevel)
@@ -220,10 +225,10 @@ func (c *Server) buildDatabase() error {
 		)
 	}
 
-	c.rootLibrary = libraries.New(libraries.Options{})
-	c.pool = gitbase.NewRepositoryPool(c.CacheSize*cache.MiByte, c.rootLibrary)
+	c.sharedCache = cache.NewObjectLRU(c.CacheSize * cache.MiByte)
 
-	c.sharedCache = cache.NewObjectLRU(512 * cache.MiByte)
+	c.rootLibrary = libraries.New(libraries.Options{})
+	c.pool = gitbase.NewRepositoryPool(c.sharedCache, c.rootLibrary)
 
 	if err := c.addDirectories(); err != nil {
 		return err
@@ -270,11 +275,19 @@ func (c *Server) addDirectories() error {
 		logrus.Error("at least one folder should be provided.")
 	}
 
+	defaultBare := bareAuto
+	switch {
+	case c.Bare:
+		defaultBare = bareOn
+	case c.NonBare:
+		defaultBare = bareOff
+	}
+
 	for _, d := range c.Directories {
 		dir := directory{
 			Path:   d,
 			Format: c.Format,
-			Bare:   c.Bare,
+			Bare:   defaultBare,
 			Bucket: c.Bucket,
 			Rooted: !c.NonRooted,
 		}
@@ -327,10 +340,15 @@ func (c *Server) addDirectory(d directory) error {
 		return nil
 	}
 
+	bare, err := discoverBare(d)
+	if err != nil {
+		return err
+	}
+
 	plainOpts := &plain.LocationOptions{
 		Cache:       c.sharedCache,
 		Performance: true,
-		Bare:        d.Bare,
+		Bare:        bare,
 	}
 
 	if c.plainLibrary == nil {
@@ -354,12 +372,20 @@ func (c *Server) addDirectory(d directory) error {
 	return nil
 }
 
+type bareOpt int
+
+const (
+	bareAuto bareOpt = iota
+	bareOn
+	bareOff
+)
+
 type directory struct {
 	Path   string
 	Format string
 	Bucket int
 	Rooted bool
-	Bare   bool
+	Bare   bareOpt
 }
 
 var (
@@ -405,12 +431,18 @@ func parseDirectory(dir directory) (directory, error) {
 			dir.Format = val
 
 		case "bare":
-			if val != "true" && val != "false" {
+			switch val {
+			case "true":
+				dir.Bare = bareOn
+			case "false":
+				dir.Bare = bareOff
+			case "auto":
+				dir.Bare = bareAuto
+			default:
 				logrus.Errorf("invalid value in bare, it can only "+
-					"be true or false %v", val)
+					"be true, false, or auto %v", val)
 				return dir, ErrInvalid
 			}
-			dir.Bare = (val == "true")
 
 		case "rooted":
 			if val != "true" && val != "false" {
@@ -436,3 +468,24 @@ func parseDirectory(dir directory) (directory, error) {
 
 	return dir, nil
 }
+
+func discoverBare(d directory) (bool, error) {
+	fs := osfs.New(d.Path)
+
+	var bare bool
+	if d.Bare == bareAuto {
+		b, err := plain.IsFirstRepositoryBare(fs, "/")
+		if plain.ErrRepositoriesNotFound.Is(err) {
+			logrus.WithField("path", d.Path).
+				Errorf("could not find repositories, assuming non bare format")
+		} else if err != nil {
+			return false, err
+		}
+
+		bare = b
+	} else {
+		bare = d.Bare == bareOn
+	}
+
+	return bare, nil
+}

cmd/gitbase/command/server_test.go

@@ -1,9 +1,15 @@
 package command
 
 import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
 	"testing"
 
+	fixtures "github.com/src-d/go-git-fixtures"
 	"github.com/stretchr/testify/require"
+	"gopkg.in/src-d/go-git.v4/plumbing"
 )
 
 func TestDirectories(t *testing.T) {
@@ -70,14 +76,14 @@ func TestDirectories(t *testing.T) {
 			path: "file:///siva/path?bare=true",
 			expected: directory{
 				Path: "/siva/path",
-				Bare: true,
+				Bare: bareOn,
 			},
 		},
 		{
 			path: "file:///siva/path?bare=false",
 			expected: directory{
 				Path: "/siva/path",
-				Bare: false,
+				Bare: bareOff,
 			},
 		},
 		{
@@ -118,7 +124,7 @@ func TestDirectories(t *testing.T) {
 			expected: directory{
 				Path:   "/siva/path",
 				Format: "git",
-				Bare:   false,
+				Bare:   bareOff,
 			},
 		},
 		{
@@ -147,3 +153,169 @@ func TestDirectories(t *testing.T) {
 		})
 	}
 }
+
+func TestDiscoverBare(t *testing.T) {
+	defer func() {
+		require.NoError(t, fixtures.Clean())
+	}()
+
+	tmpDir, err := ioutil.TempDir("", "gitbase")
+	require.NoError(t, err)
+	defer os.RemoveAll(tmpDir)
+
+	emptyDir := filepath.Join(tmpDir, "empty")
+	err = os.Mkdir(emptyDir, 0777)
+	require.NoError(t, err)
+
+	bareDir := filepath.Join(tmpDir, "bare")
+	err = os.Mkdir(bareDir, 0777)
+	require.NoError(t, err)
+	dir := fixtures.ByTag("worktree").One().DotGit().Root()
+	err = os.Rename(dir, filepath.Join(bareDir, "repo"))
+	require.NoError(t, err)
+
+	nonBareDir := filepath.Join(tmpDir, "non_bare")
+	err = os.Mkdir(nonBareDir, 0777)
+	require.NoError(t, err)
+	dir = fixtures.ByTag("worktree").One().Worktree().Root()
+	err = os.Rename(dir, filepath.Join(nonBareDir, "repo"))
+	require.NoError(t, err)
+
+	tests := []struct {
+		path     string
+		bare     bareOpt
+		expected bool
+		err      bool
+	}{
+		{
+			path: "/does/not/exist",
+			err:  true,
+		},
+		{
+			path:     emptyDir,
+			bare:     bareAuto,
+			expected: false,
+		},
+		{
+			path:     emptyDir,
+			bare:     bareOn,
+			expected: true,
+		},
+		{
+			path:     emptyDir,
+			bare:     bareOff,
+			expected: false,
+		},
+		{
+			path:     bareDir,
+			bare:     bareAuto,
+			expected: true,
+		},
+		{
+			path:     bareDir,
+			bare:     bareOn,
+			expected: true,
+		},
+		{
+			path:     bareDir,
+			bare:     bareOff,
+			expected: false,
+		},
+		{
+			path:     nonBareDir,
+			bare:     bareAuto,
+			expected: false,
+		},
+		{
+			path:     nonBareDir,
+			bare:     bareOn,
+			expected: true,
+		},
+		{
+			path:     nonBareDir,
+			bare:     bareOff,
+			expected: false,
+		},
+	}
+
+	for _, test := range tests {
+		dir := directory{
+			Path: test.path,
+			Bare: test.bare,
+		}
+
+		t.Run(bareTestName(dir, test.err), func(t *testing.T) {
+			bare, err := discoverBare(dir)
+			if test.err {
+				require.Error(t, err)
+				return
+			}
+
+			require.NoError(t, err)
+			require.Equal(t, test.expected, bare)
+		})
+	}
+}
+
+func bareTestName(d directory, err bool) string {
+	bare := ""
+	switch d.Bare {
+	case bareOn:
+		bare = "bare"
+	case bareOff:
+		bare = "non-bare"
+	case bareAuto:
+		bare = "auto"
+	}
+
+	if err {
+		bare = "error"
+	}
+
+	return fmt.Sprintf("%s_%s", d.Path, bare)
+}
+
+func TestCache(t *testing.T) {
+	require := require.New(t)
+
+	tmpDir, err := ioutil.TempDir("", "gitbase")
+	require.NoError(err)
+	func() {
+		require.NoError(os.RemoveAll(tmpDir))
+	}()
+
+	server := &Server{
+		CacheSize:   512,
+		Format:      "siva",
+		Bucket:      0,
+		LogLevel:    "debug",
+		Directories: []string{"../../../_testdata"},
+		IndexDir:    tmpDir,
+	}
+
+	err = server.buildDatabase()
+	require.NoError(err)
+
+	cache := server.sharedCache
+	pool := server.pool
+	hash := plumbing.NewHash("dbfab055c70379219cbcf422f05316fdf4e1aed3")
+
+	_, ok := cache.Get(hash)
+	require.False(ok)
+
+	repo, err := pool.GetRepo("015da2f4-6d89-7ec8-5ac9-a38329ea875b")
+	require.NoError(err)
+
+	_, ok = repo.Cache().Get(hash)
+	require.False(ok)
+	require.Equal(cache, repo.Cache())
+
+	_, err = repo.CommitObject(hash)
+	require.NoError(err)
+
+	_, ok = cache.Get(hash)
+	require.True(ok)
+
+	_, ok = repo.Cache().Get(hash)
+	require.True(ok)
+}

common_test.go

@@ -49,7 +49,7 @@ func buildSession(
 	lib, err := newMultiLibrary()
 	require.NoError(err)
 
-	pool := NewRepositoryPool(cache.DefaultMaxSize, lib)
+	pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib)
 	for _, fixture := range repos {
 		path := fixture.Worktree().Root()
 		ok, err := IsGitRepo(path)
@@ -173,7 +173,7 @@ func setupSivaCloseRepos(t *testing.T, dir string) (*sql.Context, *closedLibrary
 	require.NoError(err)
 
 	closedLib := &closedLibrary{multiLibrary: lib}
-	pool := NewRepositoryPool(cache.DefaultMaxSize, closedLib)
+	pool := NewRepositoryPool(cache.NewObjectLRUDefault(), closedLib)
 
 	cwd, err := os.Getwd()
 	require.NoError(err)
@@ -298,7 +298,7 @@ func newMultiPool() (*multiLibrary, *RepositoryPool, error) {
 		return nil, nil, err
 	}
 
-	pool := NewRepositoryPool(cache.DefaultMaxSize, lib)
+	pool := NewRepositoryPool(cache.NewObjectLRUDefault(), lib)
 
 	return lib, pool, err
 }