diff --git a/config.example.yaml b/config.example.yaml index 9d4568b..2925b97 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -6,6 +6,10 @@ fs: # See mount.fuse(8) for the full list of options. #mountoptions: nodev,nosuid + # Use a symlink to point to the real location of the repository instead of doing a loopback + # Faster and allow cloning to be asynchronous, but may cause compatibility issues + # use_symlinks: false + # The git forge to use as the backend. # Must be one of "gitlab", "github", or "gitea" forge: gitlab diff --git a/config/loader.go b/config/loader.go index 48043d8..456e8d8 100644 --- a/config/loader.go +++ b/config/loader.go @@ -32,6 +32,7 @@ type ( FSConfig struct { Mountpoint string `yaml:"mountpoint,omitempty"` MountOptions string `yaml:"mountoptions,omitempty"` + UseSymlinks bool `yaml:"use_symlinks,omitempty"` Forge string `yaml:"forge,omitempty"` } GitlabClientConfig struct { @@ -89,6 +90,7 @@ func LoadConfig(configPath string) (*Config, error) { FS: FSConfig{ Mountpoint: "", MountOptions: "nodev,nosuid", + UseSymlinks: false, Forge: "", }, Gitlab: GitlabClientConfig{ diff --git a/fstree/group.go b/fstree/group.go index 7be1bfa..943798f 100644 --- a/fstree/group.go +++ b/fstree/group.go @@ -31,7 +31,7 @@ var _ = (fs.NodeReaddirer)((*groupNode)(nil)) // Ensure we are implementing the NodeLookuper interface var _ = (fs.NodeLookuper)((*groupNode)(nil)) -func newGroupNodeFromSource(source GroupSource, param *FSParam) (*groupNode, error) { +func newGroupNodeFromSource(ctx context.Context, source GroupSource, param *FSParam) (fs.InodeEmbedder, error) { node := &groupNode{ param: param, source: source, @@ -85,7 +85,7 @@ func (n *groupNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) Ino: group.GetGroupID() + groupBaseInode, Mode: fuse.S_IFDIR, } - groupNode, _ := newGroupNodeFromSource(group, n.param) + groupNode, _ := newGroupNodeFromSource(ctx, group, n.param) return n.NewInode(ctx, groupNode, attrs), 0 } @@ -93,10 +93,17 @@ func (n *groupNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) repository, found := repositories[name] if found { attrs := fs.StableAttr{ - Ino: repository.GetRepositoryID() + repositoryBaseInode, - Mode: fuse.S_IFLNK, + Ino: repository.GetRepositoryID() + repositoryBaseInode, + } + if n.param.UseSymlinks { + attrs.Mode = fuse.S_IFLNK + } else { + attrs.Mode = fuse.S_IFDIR + } + repositoryNode, err := newRepositoryNodeFromSource(ctx, repository, n.param) + if err != nil { + panic(err) } - repositoryNode, _ := newRepositoryNodeFromSource(repository, n.param) return n.NewInode(ctx, repositoryNode, attrs), 0 } diff --git a/fstree/repository.go b/fstree/repository.go index acf70bf..025ee11 100644 --- a/fstree/repository.go +++ b/fstree/repository.go @@ -2,7 +2,11 @@ package fstree import ( "context" + "errors" + "fmt" + "os" "syscall" + "time" "github.com/hanwen/go-fuse/v2/fs" ) @@ -11,7 +15,7 @@ const ( repositoryBaseInode = 2_000_000_000 ) -type repositoryNode struct { +type repositorySymlinkNode struct { fs.Inode param *FSParam @@ -26,24 +30,53 @@ type RepositorySource interface { } // Ensure we are implementing the NodeReaddirer interface -var _ = (fs.NodeReadlinker)((*repositoryNode)(nil)) +var _ = (fs.NodeReadlinker)((*repositorySymlinkNode)(nil)) -func newRepositoryNodeFromSource(source RepositorySource, param *FSParam) (*repositoryNode, error) { - node := &repositoryNode{ - param: param, - source: source, +func newRepositoryNodeFromSource(ctx context.Context, source RepositorySource, param *FSParam) (fs.InodeEmbedder, error) { + if param.UseSymlinks { + return &repositorySymlinkNode{ + param: param, + source: source, + }, nil + } else { + localRepositoryPath, err := param.GitClient.FetchLocalRepositoryPath(ctx, source) + if err != nil { + return nil, fmt.Errorf("failed to fetch local repository path: %w", err) + } + // The path must exist to successfully create a loopback. We poll the filesystem until its created. + // This of course add latency, maybe we should think of a way of mitigating it in the future. + // We do not care in the case of a symlink. A symlink pointing on nothing is still a valid symlink. + for ctx.Err() == nil { + _, err := os.Stat(localRepositoryPath) + if err == nil { + return fs.NewLoopbackRoot(localRepositoryPath) + } else if errors.Is(err, os.ErrNotExist) { + // wait for the file to be created + // TODO: think of a more efficient way of archiving this + time.Sleep(100 * time.Millisecond) + } else { + // error, filesystem + return nil, fmt.Errorf("error while waiting for the local repository to be created: %w", err) + } + } + // error, context cancelled + return nil, fmt.Errorf("context cancelled while waiting for the local repository to be created: %w", ctx.Err()) } - // Passthrough the error if there is one, nothing to add here - // Errors on clone/pull are non-fatal - return node, nil } -func (n *repositoryNode) Readlink(ctx context.Context) ([]byte, syscall.Errno) { +func (n *repositorySymlinkNode) Readlink(ctx context.Context) ([]byte, syscall.Errno) { // Create the local copy of the repo // TODO: cleanup - localRepositoryPath, err := n.param.GitClient.FetchLocalRepositoryPath(n.source) + localRepositoryPath, err := n.param.GitClient.FetchLocalRepositoryPath(ctx, n.source) if err != nil { n.param.logger.Error(err.Error()) } return []byte(localRepositoryPath), 0 } + +type repositoryLoopbackNode struct { + fs.LoopbackNode + param *FSParam + + source RepositorySource +} diff --git a/fstree/root.go b/fstree/root.go index d45e907..43d942f 100644 --- a/fstree/root.go +++ b/fstree/root.go @@ -19,7 +19,7 @@ type staticNode interface { } type GitClient interface { - FetchLocalRepositoryPath(source RepositorySource) (string, error) + FetchLocalRepositoryPath(ctx context.Context, source RepositorySource) (string, error) } type GitForge interface { @@ -28,6 +28,8 @@ type GitForge interface { } type FSParam struct { + UseSymlinks bool + GitClient GitClient GitForge GitForge @@ -75,7 +77,7 @@ func (n *rootNode) OnAdd(ctx context.Context) { } for groupName, group := range rootGroups { - groupNode, _ := newGroupNodeFromSource(group, n.param) + groupNode, _ := newGroupNodeFromSource(ctx, group, n.param) persistentInode := n.NewPersistentInode( ctx, groupNode, diff --git a/git/client.go b/git/client.go index cf25c08..387bb47 100644 --- a/git/client.go +++ b/git/client.go @@ -84,7 +84,7 @@ func NewClient(logger *slog.Logger, p config.GitClientConfig) (*gitClient, error return c, nil } -func (c *gitClient) FetchLocalRepositoryPath(source fstree.RepositorySource) (localRepoLoc string, err error) { +func (c *gitClient) FetchLocalRepositoryPath(ctx context.Context, source fstree.RepositorySource) (localRepoLoc string, err error) { rid := source.GetRepositoryID() cloneUrl := source.GetCloneURL() defaultBranch := source.GetDefaultBranch() @@ -98,12 +98,12 @@ func (c *gitClient) FetchLocalRepositoryPath(source fstree.RepositorySource) (lo localRepoLoc = filepath.Join(c.CloneLocation, hostname, strconv.Itoa(int(rid))) if _, err := os.Stat(localRepoLoc); os.IsNotExist(err) { // Dispatch clone msg - msg := c.cloneTask.WithArgs(context.Background(), cloneUrl, defaultBranch, localRepoLoc) + msg := c.cloneTask.WithArgs(ctx, cloneUrl, defaultBranch, localRepoLoc) msg.OnceInPeriod(time.Second, rid) c.queue.Add(msg) } else if c.AutoPull { // Dispatch pull msg - msg := c.pullTask.WithArgs(context.Background(), localRepoLoc, defaultBranch) + msg := c.pullTask.WithArgs(ctx, localRepoLoc, defaultBranch) msg.OnceInPeriod(time.Second, rid) c.queue.Add(msg) } diff --git a/go.mod b/go.mod index 36e7fed..378c4ac 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.21 require ( code.gitea.io/sdk/gitea v0.19.0 github.com/google/go-github/v63 v63.0.0 - github.com/hanwen/go-fuse/v2 v2.5.1 + github.com/hanwen/go-fuse/v2 v2.7.2 github.com/vmihailenco/taskq/v3 v3.2.9 github.com/xanzy/go-gitlab v0.107.0 gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index 5d2b0d8..35b7174 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hanwen/go-fuse/v2 v2.5.1 h1:OQBE8zVemSocRxA4OaFJbjJ5hlpCmIWbGr7r0M4uoQQ= github.com/hanwen/go-fuse/v2 v2.5.1/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1rlMJc+8IJs= +github.com/hanwen/go-fuse/v2 v2.7.2 h1:SbJP1sUP+n1UF8NXBA14BuojmTez+mDgOk0bC057HQw= +github.com/hanwen/go-fuse/v2 v2.7.2/go.mod h1:ugNaD/iv5JYyS1Rcvi57Wz7/vrLQJo10mmketmoef48= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= diff --git a/main.go b/main.go index d833972..ec56463 100644 --- a/main.go +++ b/main.go @@ -98,7 +98,11 @@ func main() { logger, mountpoint, parsedMountoptions, - &fstree.FSParam{GitClient: gitClient, GitForge: gitForgeClient}, + &fstree.FSParam{ + UseSymlinks: loadedConfig.FS.UseSymlinks, + GitClient: gitClient, + GitForge: gitForgeClient, + }, *debug, ) if err != nil {