From 2542edc3b76733b2fa4635652726c231c5da729b Mon Sep 17 00:00:00 2001 From: Massaki Archambault Date: Sun, 27 Dec 2020 02:23:00 -0500 Subject: [PATCH] start of fuse filesystem implementation --- Makefile | 9 +++-- fs/filesystem.go | 27 +++++++++++++++ fs/group.go | 89 ++++++++++++++++++++++++++++++++++++++++++++++++ fs/repository.go | 30 ++++++++++++++++ fs/utils.go | 10 ++++++ gitlab/client.go | 89 ++++++++++++++++++++++++++++++------------------ go.mod | 6 +++- go.sum | 7 ++++ main.go | 40 +++++++++++----------- 9 files changed, 249 insertions(+), 58 deletions(-) create mode 100644 fs/filesystem.go create mode 100644 fs/group.go create mode 100644 fs/repository.go create mode 100644 fs/utils.go diff --git a/Makefile b/Makefile index 8428783..ac29b4a 100644 --- a/Makefile +++ b/Makefile @@ -54,10 +54,15 @@ lint: $(GOFMT) ./... # run: run without building with debug flags - +ifeq (run,$(firstword $(MAKECMDGOALS))) + # use the rest as arguments for "run" + RUN_ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) + # ...and turn them into do-nothing targets + $(eval $(RUN_ARGS):;@:) +endif .PHONY: run run: lint - $(GORUN) main.go + $(GORUN) main.go $(RUN_ARGS) # prepare: prepare environment for build diff --git a/fs/filesystem.go b/fs/filesystem.go new file mode 100644 index 0000000..cdf3570 --- /dev/null +++ b/fs/filesystem.go @@ -0,0 +1,27 @@ +package fs + +import ( + "fmt" + + "github.com/badjware/gitlabfs/gitlab" + + "github.com/hanwen/go-fuse/v2/fs" +) + +func Start(gf gitlab.GroupFetcher, mountpoint string, rootGrouptID int) error { + fmt.Printf("Mounting in %v\n", mountpoint) + + opts := &fs.Options{} + opts.Debug = true + root, err := newRootGroupNode(gf, rootGrouptID) + if err != nil { + return fmt.Errorf("root group fetch fail: %w", err) + } + server, err := fs.Mount(mountpoint, root, opts) + if err != nil { + return fmt.Errorf("mount failed: %w", err) + } + server.Wait() + + return nil +} diff --git a/fs/group.go b/fs/group.go new file mode 100644 index 0000000..6c1247b --- /dev/null +++ b/fs/group.go @@ -0,0 +1,89 @@ +package fs + +import ( + "context" + "syscall" + + "github.com/badjware/gitlabfs/gitlab" + "github.com/hanwen/go-fuse/v2/fs" + "github.com/hanwen/go-fuse/v2/fuse" +) + +type groupNode struct { + fs.Inode + group *gitlab.Group + gf gitlab.GroupFetcher +} + +// Ensure we are implementing the NodeReaddirer interface +var _ = (fs.NodeReaddirer)((*groupNode)(nil)) + +// Ensure we are implementing the NodeLookuper interface +var _ = (fs.NodeLookuper)((*groupNode)(nil)) + +func newRootGroupNode(gf gitlab.GroupFetcher, gid int) (*groupNode, error) { + group, err := gf.FetchGroup(gid) + if err != nil { + return nil, err + } + node := &groupNode{ + group: group, + gf: gf, + } + return node, nil +} + +func newGroupNode(gf gitlab.GroupFetcher, group *gitlab.Group) (*groupNode, error) { + node := &groupNode{ + group: group, + gf: gf, + } + return node, nil +} + +func (n *groupNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) { + groupContent, _ := n.gf.FetchGroupContent(n.group) + entries := make([]fuse.DirEntry, 0, len(groupContent.Groups)+len(groupContent.Repositories)) + for _, group := range groupContent.Groups { + entries = append(entries, fuse.DirEntry{ + Name: group.Path, + Ino: uint64(group.ID), + Mode: fuse.S_IFDIR, + }) + } + for _, repository := range groupContent.Repositories { + entries = append(entries, fuse.DirEntry{ + Name: repository.Path, + Ino: uint64(repository.ID), + Mode: fuse.S_IFLNK, + }) + } + return fs.NewListDirStream(entries), 0 +} + +func (n *groupNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) { + groupContent, _ := n.gf.FetchGroupContent(n.group) + + // Check if the map of groups contains it + group, ok := groupContent.Groups[name] + if ok { + attrs := fs.StableAttr{ + Ino: uint64(group.ID), + Mode: fuse.S_IFDIR, + } + groupNode, _ := newGroupNode(n.gf, group) + return n.NewInode(ctx, groupNode, attrs), 0 + } + + // Check if the map of repositories contains it + repository, ok := groupContent.Repositories[name] + if ok { + attrs := fs.StableAttr{ + Ino: uint64(repository.ID), + Mode: fuse.S_IFLNK, + } + repositoryNode, _ := newRepositoryNode(repository) // TODO + return n.NewInode(ctx, repositoryNode, attrs), 0 + } + return nil, syscall.ENOENT +} diff --git a/fs/repository.go b/fs/repository.go new file mode 100644 index 0000000..aeae69a --- /dev/null +++ b/fs/repository.go @@ -0,0 +1,30 @@ +package fs + +import ( + "context" + "strconv" + "syscall" + + "github.com/badjware/gitlabfs/gitlab" + "github.com/hanwen/go-fuse/v2/fs" +) + +type RepositoryNode struct { + fs.Inode + repository *gitlab.Repository +} + +// Ensure we are implementing the NodeReaddirer interface +var _ = (fs.NodeReadlinker)((*RepositoryNode)(nil)) + +func newRepositoryNode(repository *gitlab.Repository) (*RepositoryNode, error) { + node := &RepositoryNode{ + repository: repository, + } + return node, nil +} + +func (n *RepositoryNode) Readlink(ctx context.Context) ([]byte, syscall.Errno) { + // TODO + return []byte(strconv.Itoa(n.repository.ID)), 0 +} diff --git a/fs/utils.go b/fs/utils.go new file mode 100644 index 0000000..eaafefc --- /dev/null +++ b/fs/utils.go @@ -0,0 +1,10 @@ +package fs + +import ( + "path/filepath" + "strings" +) + +func stripSlash(fn string) string { + return strings.TrimRight(fn, string(filepath.Separator)) +} diff --git a/gitlab/client.go b/gitlab/client.go index 286eda2..6220254 100644 --- a/gitlab/client.go +++ b/gitlab/client.go @@ -6,13 +6,21 @@ import ( "github.com/xanzy/go-gitlab" ) -type GroupContentFetcher interface { - FetchGroupContent(path string) (GroupContent, error) +type GroupFetcher interface { + FetchGroup(gid int) (*Group, error) + FetchGroupContent(group *Group) (*GroupContent, error) } type GroupContent struct { - Repositories []Repository - Groups []Group + Groups map[string]*Group + Repositories map[string]*Repository +} + +type Group struct { + ID int + Name string + Path string + Content *GroupContent } type Repository struct { @@ -22,12 +30,6 @@ type Repository struct { CloneURL string } -type Group struct { - ID int - Name string - Path string -} - type GitlabClient struct { Client *gitlab.Client } @@ -47,7 +49,7 @@ func NewClient(gitlabUrl string, gitlabToken string) (*GitlabClient, error) { return gitlabClient, nil } -func NewRepositryFromGitlabProject(project *gitlab.Project) Repository { +func NewRepositoryFromGitlabProject(project *gitlab.Project) Repository { // https://godoc.org/github.com/xanzy/go-gitlab#Project return Repository{ ID: project.ID, @@ -67,27 +69,23 @@ func NewGroupFromGitlabGroup(group *gitlab.Group) Group { } } -func (g GitlabClient) FetchGroupContent(group *Group) (*GroupContent, error) { - content := &GroupContent{} +func (c GitlabClient) FetchGroup(gid int) (*Group, error) { + gitlabGroup, _, err := c.Client.Groups.GetGroup(gid) + if err != nil { + return nil, fmt.Errorf("failed to fetch root group with id %v: %w\n", gid, err) + } + group := NewGroupFromGitlabGroup(gitlabGroup) + return &group, nil +} - // List repositories in path - listProjectOpt := &gitlab.ListGroupProjectsOptions{ - ListOptions: gitlab.ListOptions{ - Page: 1, - }} - for { - projects, response, err := g.Client.Groups.ListGroupProjects(group.ID, listProjectOpt) - if err != nil { - return nil, fmt.Errorf("failed to fetch projects in gitlab: %w", err) - } - for _, project := range projects { - content.Repositories = append(content.Repositories, NewRepositryFromGitlabProject(project)) - } - if response.CurrentPage >= response.TotalPages { - break - } - // Get the next page - listProjectOpt.Page = response.NextPage +func (c GitlabClient) FetchGroupContent(group *Group) (*GroupContent, error) { + if group.Content != nil { + return group.Content, nil + } + + content := &GroupContent{ + Groups: map[string]*Group{}, + Repositories: map[string]*Repository{}, } // List subgroups in path @@ -96,12 +94,13 @@ func (g GitlabClient) FetchGroupContent(group *Group) (*GroupContent, error) { Page: 1, }} for { - groups, response, err := g.Client.Groups.ListSubgroups(group.ID, ListGroupsOpt) + gitlabGroups, response, err := c.Client.Groups.ListSubgroups(group.ID, ListGroupsOpt) if err != nil { return nil, fmt.Errorf("failed to fetch groups in gitlab: %w", err) } - for _, group := range groups { - content.Groups = append(content.Groups, NewGroupFromGitlabGroup(group)) + for _, gitlabGroup := range gitlabGroups { + group := NewGroupFromGitlabGroup(gitlabGroup) + content.Groups[group.Path] = &group } if response.CurrentPage >= response.TotalPages { break @@ -110,5 +109,27 @@ func (g GitlabClient) FetchGroupContent(group *Group) (*GroupContent, error) { ListGroupsOpt.Page = response.NextPage } + // List repositories in path + listProjectOpt := &gitlab.ListGroupProjectsOptions{ + ListOptions: gitlab.ListOptions{ + Page: 1, + }} + for { + gitlabProjects, response, err := c.Client.Groups.ListGroupProjects(group.ID, listProjectOpt) + if err != nil { + return nil, fmt.Errorf("failed to fetch projects in gitlab: %w", err) + } + for _, gitlabProject := range gitlabProjects { + repository := NewRepositoryFromGitlabProject(gitlabProject) + content.Repositories[repository.Path] = &repository + } + if response.CurrentPage >= response.TotalPages { + break + } + // Get the next page + listProjectOpt.Page = response.NextPage + } + + group.Content = content return content, nil } diff --git a/go.mod b/go.mod index 0ad6c06..904bca7 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,8 @@ module github.com/badjware/gitlabfs go 1.15 -require github.com/xanzy/go-gitlab v0.40.2 +require ( + github.com/hanwen/go-fuse v1.0.0 + github.com/hanwen/go-fuse/v2 v2.0.3 + github.com/xanzy/go-gitlab v0.40.2 +) diff --git a/go.sum b/go.sum index 465f491..db8975b 100644 --- a/go.sum +++ b/go.sum @@ -3,11 +3,16 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/hanwen/go-fuse v1.0.0 h1:GxS9Zrn6c35/BnfiVsZVWmsG803xwE7eVRDvcf/BEVc= +github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok= +github.com/hanwen/go-fuse/v2 v2.0.3 h1:kpV28BKeSyVgZREItBLnaVBvOEwv2PuhNdKetwnvNHo= +github.com/hanwen/go-fuse/v2 v2.0.3/go.mod h1:0EQM6aH2ctVpvZ6a+onrQ/vaykxh2GH7hy3e13vzTUY= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-retryablehttp v0.6.4 h1:BbgctKO892xEyOXnGiaAwIoSq1QZ/SS4AhjoAh9DnfY= github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -20,6 +25,8 @@ golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288 h1:JIqe8uIcRBHXDQVvZtHwp80ai3Lw3IJAeJEs55Dc1W0= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522 h1:Ve1ORMCxvRmSXBwJK+t3Oy+V2vRW2OetUQBq4rJIkZE= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/main.go b/main.go index 56b5e23..4a0a053 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,7 @@ import ( "fmt" "os" + "github.com/badjware/gitlabfs/fs" "github.com/badjware/gitlabfs/gitlab" ) @@ -13,31 +14,28 @@ func main() { gitlabToken := flag.String("gitlab-token", "", "the gitlab authentication token") gitlabRootGroupID := flag.Int("gitlab-group-id", 9970, "the group id of the groups at the root of the filesystem") // gitlabNamespace := flag.String() - flag.Parse() + if flag.NArg() != 1 { + fmt.Printf("usage: %s MOUNTPOINT\n", os.Args[0]) + os.Exit(2) + } + mountpoint := flag.Arg(0) gitlabClient, _ := gitlab.NewClient(*gitlabURL, *gitlabToken) - // TODO: move this - group, _, err := gitlabClient.Client.Groups.GetGroup(*gitlabRootGroupID) - if err != nil { - fmt.Printf("failed to fetch root group with id %v: %w\n", *gitlabRootGroupID, err) - os.Exit(1) - } - rootGroup := gitlab.NewGroupFromGitlabGroup(group) - fmt.Printf("Root group: %v\n", rootGroup.Name) + fs.Start(gitlabClient, mountpoint, *gitlabRootGroupID) - content, err := gitlabClient.FetchGroupContent(&rootGroup) - if err != nil { - fmt.Println(err) - } + // content, err := gitlabClient.FetchGroupContent(&rootGroup) + // if err != nil { + // fmt.Println(err) + // } - fmt.Println("Projects") - for _, r := range content.Repositories { - fmt.Println(r.Name, r.Path, r.CloneURL) - } - fmt.Println("Groups") - for _, g := range content.Groups { - fmt.Println(g.Name, g.Path) - } + // fmt.Println("Projects") + // for _, r := range content.Repositories { + // fmt.Println(r.Name, r.Path, r.CloneURL) + // } + // fmt.Println("Groups") + // for _, g := range content.Groups { + // fmt.Println(g.Name, g.Path) + // } }