default to using a filesystem loopback instead of a symlink to local repositories

fixes #13

config.example.yaml

@@ -6,6 +6,10 @@ fs:
   # See mount.fuse(8) for the full list of options.
   #mountoptions: nodev,nosuid
 
+  # Use a symlink to point to the real location of the repository instead of doing a loopback
+  # Faster and allow cloning to be asynchronous, but may cause compatibility issues
+  # use_symlinks: false
+
   # The git forge to use as the backend.
   # Must be one of "gitlab", "github", or "gitea"
   forge: gitlab

config/loader.go

@@ -32,6 +32,7 @@ type (
 	FSConfig struct {
 		Mountpoint   string `yaml:"mountpoint,omitempty"`
 		MountOptions string `yaml:"mountoptions,omitempty"`
+		UseSymlinks  bool   `yaml:"use_symlinks,omitempty"`
 		Forge        string `yaml:"forge,omitempty"`
 	}
 	GitlabClientConfig struct {
@@ -89,6 +90,7 @@ func LoadConfig(configPath string) (*Config, error) {
 		FS: FSConfig{
 			Mountpoint:   "",
 			MountOptions: "nodev,nosuid",
+			UseSymlinks:  false,
 			Forge:        "",
 		},
 		Gitlab: GitlabClientConfig{

fstree/group.go

@@ -31,7 +31,7 @@ var _ = (fs.NodeReaddirer)((*groupNode)(nil))
 // Ensure we are implementing the NodeLookuper interface
 var _ = (fs.NodeLookuper)((*groupNode)(nil))
 
-func newGroupNodeFromSource(source GroupSource, param *FSParam) (*groupNode, error) {
+func newGroupNodeFromSource(ctx context.Context, source GroupSource, param *FSParam) (fs.InodeEmbedder, error) {
 	node := &groupNode{
 		param:  param,
 		source: source,
@@ -85,7 +85,7 @@ func (n *groupNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut)
 				Ino:  group.GetGroupID() + groupBaseInode,
 				Mode: fuse.S_IFDIR,
 			}
-			groupNode, _ := newGroupNodeFromSource(group, n.param)
+			groupNode, _ := newGroupNodeFromSource(ctx, group, n.param)
 			return n.NewInode(ctx, groupNode, attrs), 0
 		}
 
@@ -93,10 +93,17 @@ func (n *groupNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut)
 		repository, found := repositories[name]
 		if found {
 			attrs := fs.StableAttr{
-				Ino:  repository.GetRepositoryID() + repositoryBaseInode,
-				Mode: fuse.S_IFLNK,
+				Ino: repository.GetRepositoryID() + repositoryBaseInode,
+			}
+			if n.param.UseSymlinks {
+				attrs.Mode = fuse.S_IFLNK
+			} else {
+				attrs.Mode = fuse.S_IFDIR
+			}
+			repositoryNode, err := newRepositoryNodeFromSource(ctx, repository, n.param)
+			if err != nil {
+				panic(err)
 			}
-			repositoryNode, _ := newRepositoryNodeFromSource(repository, n.param)
 			return n.NewInode(ctx, repositoryNode, attrs), 0
 		}
 

fstree/repository.go

@@ -2,7 +2,11 @@ package fstree
 
 import (
 	"context"
+	"errors"
+	"fmt"
+	"os"
 	"syscall"
+	"time"
 
 	"github.com/hanwen/go-fuse/v2/fs"
 )
@@ -11,7 +15,7 @@ const (
 	repositoryBaseInode = 2_000_000_000
 )
 
-type repositoryNode struct {
+type repositorySymlinkNode struct {
 	fs.Inode
 	param *FSParam
 
@@ -26,24 +30,53 @@ type RepositorySource interface {
 }
 
 // Ensure we are implementing the NodeReaddirer interface
-var _ = (fs.NodeReadlinker)((*repositoryNode)(nil))
+var _ = (fs.NodeReadlinker)((*repositorySymlinkNode)(nil))
 
-func newRepositoryNodeFromSource(source RepositorySource, param *FSParam) (*repositoryNode, error) {
-	node := &repositoryNode{
-		param:  param,
-		source: source,
+func newRepositoryNodeFromSource(ctx context.Context, source RepositorySource, param *FSParam) (fs.InodeEmbedder, error) {
+	if param.UseSymlinks {
+		return &repositorySymlinkNode{
+			param:  param,
+			source: source,
+		}, nil
+	} else {
+		localRepositoryPath, err := param.GitClient.FetchLocalRepositoryPath(ctx, source)
+		if err != nil {
+			return nil, fmt.Errorf("failed to fetch local repository path: %w", err)
+		}
+		// The path must exist to successfully create a loopback. We poll the filesystem until its created.
+		// This of course add latency, maybe we should think of a way of mitigating it in the future.
+		// We do not care in the case of a symlink. A symlink pointing on nothing is still a valid symlink.
+		for ctx.Err() == nil {
+			_, err := os.Stat(localRepositoryPath)
+			if err == nil {
+				return fs.NewLoopbackRoot(localRepositoryPath)
+			} else if errors.Is(err, os.ErrNotExist) {
+				// wait for the file to be created
+				// TODO: think of a more efficient way of archiving this
+				time.Sleep(100 * time.Millisecond)
+			} else {
+				// error, filesystem
+				return nil, fmt.Errorf("error while waiting for the local repository to be created: %w", err)
+			}
+		}
+		// error, context cancelled
+		return nil, fmt.Errorf("context cancelled while waiting for the local repository to be created: %w", ctx.Err())
 	}
-	// Passthrough the error if there is one, nothing to add here
-	// Errors on clone/pull are non-fatal
-	return node, nil
 }
 
-func (n *repositoryNode) Readlink(ctx context.Context) ([]byte, syscall.Errno) {
+func (n *repositorySymlinkNode) Readlink(ctx context.Context) ([]byte, syscall.Errno) {
 	// Create the local copy of the repo
 	// TODO: cleanup
-	localRepositoryPath, err := n.param.GitClient.FetchLocalRepositoryPath(n.source)
+	localRepositoryPath, err := n.param.GitClient.FetchLocalRepositoryPath(ctx, n.source)
 	if err != nil {
 		n.param.logger.Error(err.Error())
 	}
 	return []byte(localRepositoryPath), 0
 }
+
+type repositoryLoopbackNode struct {
+	fs.LoopbackNode
+	param *FSParam
+
+	source RepositorySource
+}

fstree/root.go

@@ -19,7 +19,7 @@ type staticNode interface {
 }
 
 type GitClient interface {
-	FetchLocalRepositoryPath(source RepositorySource) (string, error)
+	FetchLocalRepositoryPath(ctx context.Context, source RepositorySource) (string, error)
 }
 
 type GitForge interface {
@@ -28,6 +28,8 @@ type GitForge interface {
 }
 
 type FSParam struct {
+	UseSymlinks bool
+
 	GitClient GitClient
 	GitForge  GitForge
 
@@ -75,7 +77,7 @@ func (n *rootNode) OnAdd(ctx context.Context) {
 	}
 
 	for groupName, group := range rootGroups {
-		groupNode, _ := newGroupNodeFromSource(group, n.param)
+		groupNode, _ := newGroupNodeFromSource(ctx, group, n.param)
 		persistentInode := n.NewPersistentInode(
 			ctx,
 			groupNode,

git/client.go

@@ -84,7 +84,7 @@ func NewClient(logger *slog.Logger, p config.GitClientConfig) (*gitClient, error
 	return c, nil
 }
 
-func (c *gitClient) FetchLocalRepositoryPath(source fstree.RepositorySource) (localRepoLoc string, err error) {
+func (c *gitClient) FetchLocalRepositoryPath(ctx context.Context, source fstree.RepositorySource) (localRepoLoc string, err error) {
 	rid := source.GetRepositoryID()
 	cloneUrl := source.GetCloneURL()
 	defaultBranch := source.GetDefaultBranch()
@@ -98,12 +98,12 @@ func (c *gitClient) FetchLocalRepositoryPath(source fstree.RepositorySource) (lo
 	localRepoLoc = filepath.Join(c.CloneLocation, hostname, strconv.Itoa(int(rid)))
 	if _, err := os.Stat(localRepoLoc); os.IsNotExist(err) {
 		// Dispatch clone msg
-		msg := c.cloneTask.WithArgs(context.Background(), cloneUrl, defaultBranch, localRepoLoc)
+		msg := c.cloneTask.WithArgs(ctx, cloneUrl, defaultBranch, localRepoLoc)
 		msg.OnceInPeriod(time.Second, rid)
 		c.queue.Add(msg)
 	} else if c.AutoPull {
 		// Dispatch pull msg
-		msg := c.pullTask.WithArgs(context.Background(), localRepoLoc, defaultBranch)
+		msg := c.pullTask.WithArgs(ctx, localRepoLoc, defaultBranch)
 		msg.OnceInPeriod(time.Second, rid)
 		c.queue.Add(msg)
 	}

go.mod

@@ -5,7 +5,7 @@ go 1.21
 require (
 	code.gitea.io/sdk/gitea v0.19.0
 	github.com/google/go-github/v63 v63.0.0
-	github.com/hanwen/go-fuse/v2 v2.5.1
+	github.com/hanwen/go-fuse/v2 v2.7.2
 	github.com/vmihailenco/taskq/v3 v3.2.9
 	github.com/xanzy/go-gitlab v0.107.0
 	gopkg.in/yaml.v2 v2.4.0

go.sum

@@ -59,6 +59,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hanwen/go-fuse/v2 v2.5.1 h1:OQBE8zVemSocRxA4OaFJbjJ5hlpCmIWbGr7r0M4uoQQ=
 github.com/hanwen/go-fuse/v2 v2.5.1/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1rlMJc+8IJs=
+github.com/hanwen/go-fuse/v2 v2.7.2 h1:SbJP1sUP+n1UF8NXBA14BuojmTez+mDgOk0bC057HQw=
+github.com/hanwen/go-fuse/v2 v2.7.2/go.mod h1:ugNaD/iv5JYyS1Rcvi57Wz7/vrLQJo10mmketmoef48=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=

main.go

@@ -98,7 +98,11 @@ func main() {
 		logger,
 		mountpoint,
 		parsedMountoptions,
-		&fstree.FSParam{GitClient: gitClient, GitForge: gitForgeClient},
+		&fstree.FSParam{
+			UseSymlinks: loadedConfig.FS.UseSymlinks,
+			GitClient:   gitClient,
+			GitForge:    gitForgeClient,
+		},
 		*debug,
 	)
 	if err != nil {