From 5ed64d523e0d8319edf06b5048d75fea25363418 Mon Sep 17 00:00:00 2001 From: Massaki Archambault Date: Sun, 5 May 2024 16:09:03 -0400 Subject: [PATCH] refactor to decouple fs package from gitlab package --- fs/group.go | 70 ++++++++++---------- fs/groups.go | 47 -------------- fs/refresh.go | 14 ++-- fs/repository.go | 24 ++++--- fs/root.go | 74 +++++++++------------ fs/users.go | 159 ---------------------------------------------- gitlab/client.go | 96 +++++++++++++++++++++++----- gitlab/group.go | 157 +++++++++++++++++++++++---------------------- gitlab/project.go | 12 ++++ gitlab/user.go | 124 +++++++++++++++++++----------------- go.mod | 26 ++++++-- go.sum | 3 - main.go | 33 ++++------ 13 files changed, 357 insertions(+), 482 deletions(-) delete mode 100644 fs/groups.go delete mode 100644 fs/users.go diff --git a/fs/group.go b/fs/group.go index 0485e5f..9c0b59e 100644 --- a/fs/group.go +++ b/fs/group.go @@ -2,9 +2,9 @@ package fs import ( "context" + "fmt" "syscall" - "github.com/badjware/gitlabfs/gitlab" "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" ) @@ -13,56 +13,50 @@ type groupNode struct { fs.Inode param *FSParam - group *gitlab.Group + source GroupSource staticNodes map[string]staticNode } +type GroupSource interface { + GetGroupID() uint64 + InvalidateCache() +} + // Ensure we are implementing the NodeReaddirer interface var _ = (fs.NodeReaddirer)((*groupNode)(nil)) // Ensure we are implementing the NodeLookuper interface var _ = (fs.NodeLookuper)((*groupNode)(nil)) -func newGroupNodeByID(gid int, param *FSParam) (*groupNode, error) { - group, err := param.Gitlab.FetchGroup(gid) - if err != nil { - return nil, err - } +func newGroupNodeFromSource(source GroupSource, param *FSParam) (*groupNode, error) { node := &groupNode{ - param: param, - group: group, + param: param, + source: source, staticNodes: map[string]staticNode{ - ".refresh": newRefreshNode(group, param), - }, - } - return node, nil -} - -func newGroupNode(group *gitlab.Group, param *FSParam) (*groupNode, error) { - node := &groupNode{ - param: param, - group: group, - staticNodes: map[string]staticNode{ - ".refresh": newRefreshNode(group, param), + ".refresh": newRefreshNode(source, param), }, } return node, nil } func (n *groupNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) { - groupContent, _ := n.param.Gitlab.FetchGroupContent(n.group) - entries := make([]fuse.DirEntry, 0, len(groupContent.Groups)+len(groupContent.Projects)+len(n.staticNodes)) - for _, group := range groupContent.Groups { + groups, repositories, err := n.param.GitPlatform.FetchGroupContent(n.source.GetGroupID()) + if err != nil { + fmt.Errorf("%v", err) + } + + entries := make([]fuse.DirEntry, 0, len(groups)+len(repositories)+len(n.staticNodes)) + for groupName, group := range groups { entries = append(entries, fuse.DirEntry{ - Name: group.Name, - Ino: uint64(group.ID), + Name: groupName, + Ino: group.GetGroupID(), Mode: fuse.S_IFDIR, }) } - for _, project := range groupContent.Projects { + for repositoryName, repository := range repositories { entries = append(entries, fuse.DirEntry{ - Name: project.Name, - Ino: uint64(project.ID), + Name: repositoryName, + Ino: repository.GetRepositoryID(), Mode: fuse.S_IFLNK, }) } @@ -77,27 +71,27 @@ func (n *groupNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) { } func (n *groupNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) { - groupContent, _ := n.param.Gitlab.FetchGroupContent(n.group) + groups, repositories, _ := n.param.GitPlatform.FetchGroupContent(n.source.GetGroupID()) // Check if the map of groups contains it - group, ok := groupContent.Groups[name] - if ok { + group, found := groups[name] + if found { attrs := fs.StableAttr{ - Ino: uint64(group.ID), + Ino: group.GetGroupID(), Mode: fuse.S_IFDIR, } - groupNode, _ := newGroupNode(group, n.param) + groupNode, _ := newGroupNodeFromSource(group, n.param) return n.NewInode(ctx, groupNode, attrs), 0 } // Check if the map of projects contains it - project, ok := groupContent.Projects[name] - if ok { + repository, found := repositories[name] + if found { attrs := fs.StableAttr{ - Ino: uint64(project.ID), + Ino: repository.GetRepositoryID(), Mode: fuse.S_IFLNK, } - repositoryNode, _ := newRepositoryNode(project, n.param) + repositoryNode, _ := newRepositoryNodeFromSource(repository, n.param) return n.NewInode(ctx, repositoryNode, attrs), 0 } diff --git a/fs/groups.go b/fs/groups.go deleted file mode 100644 index 2635407..0000000 --- a/fs/groups.go +++ /dev/null @@ -1,47 +0,0 @@ -package fs - -import ( - "context" - "fmt" - - "github.com/hanwen/go-fuse/v2/fs" - "github.com/hanwen/go-fuse/v2/fuse" -) - -type groupsNode struct { - fs.Inode - param *FSParam - - rootGroupIds []int -} - -// Ensure we are implementing the NodeOnAdder interface -var _ = (fs.NodeOnAdder)((*groupsNode)(nil)) - -func newGroupsNode(rootGroupIds []int, param *FSParam) *groupsNode { - return &groupsNode{ - param: param, - rootGroupIds: rootGroupIds, - } -} - -func (n *groupsNode) OnAdd(ctx context.Context) { - for _, groupID := range n.rootGroupIds { - groupNode, err := newGroupNodeByID(groupID, n.param) - if err != nil { - fmt.Printf("root group fetch fail: %v\n", err) - fmt.Printf("Please verify the group exists, is public or a token with sufficient permissions is set in the config files.\n") - fmt.Printf("Skipping group %v\n", groupID) - return - } - inode := n.NewPersistentInode( - ctx, - groupNode, - fs.StableAttr{ - Ino: <-n.param.staticInoChan, - Mode: fuse.S_IFDIR, - }, - ) - n.AddChild(groupNode.group.Name, inode, false) - } -} diff --git a/fs/refresh.go b/fs/refresh.go index 30f4974..02bdae6 100644 --- a/fs/refresh.go +++ b/fs/refresh.go @@ -4,15 +4,15 @@ import ( "context" "syscall" - "github.com/badjware/gitlabfs/gitlab" "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" ) type refreshNode struct { fs.Inode - ino uint64 - refresher gitlab.Refresher + ino uint64 + + source GroupSource } // Ensure we are implementing the NodeSetattrer interface @@ -21,10 +21,10 @@ var _ = (fs.NodeSetattrer)((*refreshNode)(nil)) // Ensure we are implementing the NodeOpener interface var _ = (fs.NodeOpener)((*refreshNode)(nil)) -func newRefreshNode(refresher gitlab.Refresher, param *FSParam) *refreshNode { +func newRefreshNode(source GroupSource, param *FSParam) *refreshNode { return &refreshNode{ - ino: <-param.staticInoChan, - refresher: refresher, + ino: <-param.staticInoChan, + source: source, } } @@ -41,6 +41,6 @@ func (n *refreshNode) Setattr(ctx context.Context, fh fs.FileHandle, in *fuse.Se } func (n *refreshNode) Open(ctx context.Context, flags uint32) (fh fs.FileHandle, fuseFlags uint32, errno syscall.Errno) { - n.refresher.InvalidateCache() + n.source.InvalidateCache() return nil, 0, 0 } diff --git a/fs/repository.go b/fs/repository.go index 5370473..046183a 100644 --- a/fs/repository.go +++ b/fs/repository.go @@ -4,23 +4,30 @@ import ( "context" "syscall" - "github.com/badjware/gitlabfs/gitlab" "github.com/hanwen/go-fuse/v2/fs" ) type RepositoryNode struct { fs.Inode - param *FSParam - project *gitlab.Project + param *FSParam + + source RepositorySource +} + +type RepositorySource interface { + // GetName() string + GetRepositoryID() uint64 + GetCloneURL() string + GetDefaultBranch() string } // Ensure we are implementing the NodeReaddirer interface var _ = (fs.NodeReadlinker)((*RepositoryNode)(nil)) -func newRepositoryNode(project *gitlab.Project, param *FSParam) (*RepositoryNode, error) { +func newRepositoryNodeFromSource(source RepositorySource, param *FSParam) (*RepositoryNode, error) { node := &RepositoryNode{ - param: param, - project: project, + param: param, + source: source, } // Passthrough the error if there is one, nothing to add here // Errors on clone/pull are non-fatal @@ -29,7 +36,8 @@ func newRepositoryNode(project *gitlab.Project, param *FSParam) (*RepositoryNode func (n *RepositoryNode) Readlink(ctx context.Context) ([]byte, syscall.Errno) { // Create the local copy of the repo - localRepoLoc, _ := n.param.Git.CloneOrPull(n.project.CloneURL, n.project.ID, n.project.DefaultBranch) + // TODO: cleanup + localRepositoryPath, _ := n.param.GitImplementation.CloneOrPull(n.source.GetCloneURL(), int(n.source.GetRepositoryID()), n.source.GetDefaultBranch()) - return []byte(localRepoLoc), 0 + return []byte(localRepositoryPath), 0 } diff --git a/fs/root.go b/fs/root.go index 9bcfc86..2155bee 100644 --- a/fs/root.go +++ b/fs/root.go @@ -8,8 +8,6 @@ import ( "syscall" "github.com/badjware/gitlabfs/git" - "github.com/badjware/gitlabfs/gitlab" - "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" ) @@ -24,55 +22,25 @@ type staticNode interface { Mode() uint32 } -type FSParam struct { - Git git.GitClonerPuller - Gitlab gitlab.GitlabFetcher +type GitPlatform interface { + FetchRootGroupContent() (map[string]GroupSource, error) + FetchGroupContent(gid uint64) (map[string]GroupSource, map[string]RepositorySource, error) +} - RootGroupIds []int - UserIds []int +type FSParam struct { + GitImplementation git.GitClonerPuller + GitPlatform GitPlatform staticInoChan chan uint64 } type rootNode struct { fs.Inode - param *FSParam - rootGroupIds []int - userIds []int + param *FSParam } var _ = (fs.NodeOnAdder)((*rootNode)(nil)) -func (n *rootNode) OnAdd(ctx context.Context) { - groupsInode := n.NewPersistentInode( - ctx, - newGroupsNode( - n.rootGroupIds, - n.param, - ), - fs.StableAttr{ - Ino: <-n.param.staticInoChan, - Mode: fuse.S_IFDIR, - }, - ) - n.AddChild("groups", groupsInode, false) - - usersInode := n.NewPersistentInode( - ctx, - newUsersNode( - n.userIds, - n.param, - ), - fs.StableAttr{ - Ino: <-n.param.staticInoChan, - Mode: fuse.S_IFDIR, - }, - ) - n.AddChild("users", usersInode, false) - - fmt.Println("Mounted and ready to use") -} - func Start(mountpoint string, mountoptions []string, param *FSParam, debug bool) error { fmt.Printf("Mounting in %v\n", mountpoint) @@ -82,9 +50,7 @@ func Start(mountpoint string, mountoptions []string, param *FSParam, debug bool) param.staticInoChan = make(chan uint64) root := &rootNode{ - param: param, - rootGroupIds: param.RootGroupIds, - userIds: param.UserIds, + param: param, } go staticInoGenerator(root.param.staticInoChan) @@ -104,6 +70,28 @@ func Start(mountpoint string, mountoptions []string, param *FSParam, debug bool) return nil } +func (n *rootNode) OnAdd(ctx context.Context) { + rootGroups, err := n.param.GitPlatform.FetchRootGroupContent() + if err != nil { + panic(err) + } + + for groupName, group := range rootGroups { + groupNode, _ := newGroupNodeFromSource(group, n.param) + persistentInode := n.NewPersistentInode( + ctx, + groupNode, + fs.StableAttr{ + Ino: <-n.param.staticInoChan, + Mode: fuse.S_IFDIR, + }, + ) + n.AddChild(groupName, persistentInode, false) + } + + fmt.Println("Mounted and ready to use") +} + func staticInoGenerator(staticInoChan chan<- uint64) { i := staticInodeStart for { diff --git a/fs/users.go b/fs/users.go deleted file mode 100644 index 6094c68..0000000 --- a/fs/users.go +++ /dev/null @@ -1,159 +0,0 @@ -package fs - -import ( - "context" - "fmt" - "syscall" - - "github.com/badjware/gitlabfs/gitlab" - "github.com/hanwen/go-fuse/v2/fs" - "github.com/hanwen/go-fuse/v2/fuse" -) - -type usersNode struct { - fs.Inode - param *FSParam - - userIds []int -} - -// Ensure we are implementing the NodeOnAdder interface -var _ = (fs.NodeOnAdder)((*usersNode)(nil)) - -func newUsersNode(userIds []int, param *FSParam) *usersNode { - return &usersNode{ - param: param, - userIds: userIds, - } -} - -func (n *usersNode) OnAdd(ctx context.Context) { - // Fetch the current logged user - currentUser, err := n.param.Gitlab.FetchCurrentUser() - // Skip if we are anonymous (or the call fails for some reason...) - if err != nil { - fmt.Println(err) - } else { - currentUserNode, _ := newUserNode(currentUser, n.param) - inode := n.NewPersistentInode( - ctx, - currentUserNode, - fs.StableAttr{ - Ino: <-n.param.staticInoChan, - Mode: fuse.S_IFDIR, - }, - ) - n.AddChild(currentUserNode.user.Name, inode, false) - } - - for _, userID := range n.userIds { - if currentUser != nil && currentUser.ID == userID { - // We already added the current user, we can skip it - continue - } - - userNode, err := newUserNodeByID(userID, n.param) - if err != nil { - fmt.Printf("user fetch fail: %v\n", err) - fmt.Printf("Please verify the user exists and token with sufficient permissions is set in the config files.\n") - fmt.Printf("Skipping user %v\n", userID) - return - } - inode := n.NewPersistentInode( - ctx, - userNode, - fs.StableAttr{ - Ino: <-n.param.staticInoChan, - Mode: fuse.S_IFDIR, - }, - ) - n.AddChild(userNode.user.Name, inode, false) - } -} - -type userNode struct { - fs.Inode - param *FSParam - - user *gitlab.User - staticNodes map[string]staticNode -} - -// Ensure we are implementing the NodeReaddirer interface -var _ = (fs.NodeReaddirer)((*userNode)(nil)) - -// Ensure we are implementing the NodeLookuper interface -var _ = (fs.NodeLookuper)((*userNode)(nil)) - -func newUserNodeByID(uid int, param *FSParam) (*userNode, error) { - user, err := param.Gitlab.FetchUser(uid) - if err != nil { - return nil, err - } - node := &userNode{ - param: param, - user: user, - staticNodes: map[string]staticNode{ - ".refresh": newRefreshNode(user, param), - }, - } - return node, nil -} - -func newUserNode(user *gitlab.User, param *FSParam) (*userNode, error) { - node := &userNode{ - param: param, - user: user, - staticNodes: map[string]staticNode{ - ".refresh": newRefreshNode(user, param), - }, - } - return node, nil -} - -func (n *userNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) { - userContent, _ := n.param.Gitlab.FetchUserContent(n.user) - entries := make([]fuse.DirEntry, 0, len(userContent.Projects)+len(n.staticNodes)) - for _, project := range userContent.Projects { - entries = append(entries, fuse.DirEntry{ - Name: project.Name, - Ino: uint64(project.ID), - Mode: fuse.S_IFLNK, - }) - } - for name, staticNode := range n.staticNodes { - entries = append(entries, fuse.DirEntry{ - Name: name, - Ino: staticNode.Ino(), - Mode: staticNode.Mode(), - }) - } - return fs.NewListDirStream(entries), 0 -} - -func (n *userNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) { - userContent, _ := n.param.Gitlab.FetchUserContent(n.user) - - // Check if the map of projects contains it - project, ok := userContent.Projects[name] - if ok { - attrs := fs.StableAttr{ - Ino: uint64(project.ID), - Mode: fuse.S_IFLNK, - } - repositoryNode, _ := newRepositoryNode(project, n.param) - return n.NewInode(ctx, repositoryNode, attrs), 0 - } - - // Check if the map of static nodes contains it - staticNode, ok := n.staticNodes[name] - if ok { - attrs := fs.StableAttr{ - Ino: staticNode.Ino(), - Mode: staticNode.Mode(), - } - return n.NewInode(ctx, staticNode, attrs), 0 - } - - return nil, syscall.ENOENT -} diff --git a/gitlab/client.go b/gitlab/client.go index 3c73f44..716e76f 100644 --- a/gitlab/client.go +++ b/gitlab/client.go @@ -2,7 +2,9 @@ package gitlab import ( "fmt" + "slices" + "github.com/badjware/gitlabfs/fs" "github.com/xanzy/go-gitlab" ) @@ -11,26 +13,29 @@ const ( PullMethodSSH = "ssh" ) -type GitlabFetcher interface { - GroupFetcher - UserFetcher -} - -type Refresher interface { - InvalidateCache() -} - -type GitlabClientParam struct { - PullMethod string - IncludeCurrentUser bool +type GitlabClientConfig struct { + URL string `yaml:"url,omitempty"` + Token string `yaml:"token,omitempty"` + GroupIDs []int `yaml:"group_ids,omitempty"` + UserIDs []int `yaml:"user_ids,omitempty"` + IncludeCurrentUser bool `yaml:"include_current_user,omitempty"` + PullMethod string `yaml:"pull_method,omitempty"` } type gitlabClient struct { - GitlabClientParam + GitlabClientConfig client *gitlab.Client + + // root group cache + rootGroupCache map[string]fs.GroupSource + currentUserCache *User + + // API response cache + groupCache map[int]*Group + userCache map[int]*User } -func NewClient(gitlabUrl string, gitlabToken string, p GitlabClientParam) (*gitlabClient, error) { +func NewClient(gitlabUrl string, gitlabToken string, p GitlabClientConfig) (*gitlabClient, error) { client, err := gitlab.NewClient( gitlabToken, gitlab.WithBaseURL(gitlabUrl), @@ -40,8 +45,67 @@ func NewClient(gitlabUrl string, gitlabToken string, p GitlabClientParam) (*gitl } gitlabClient := &gitlabClient{ - GitlabClientParam: p, - client: client, + GitlabClientConfig: p, + client: client, + + rootGroupCache: nil, + currentUserCache: nil, + + groupCache: map[int]*Group{}, + userCache: map[int]*User{}, } return gitlabClient, nil } + +func (c *gitlabClient) FetchRootGroupContent() (map[string]fs.GroupSource, error) { + // use cached values if available + if c.rootGroupCache == nil { + rootGroupCache := make(map[string]fs.GroupSource) + + // fetch root groups + for _, gid := range c.GroupIDs { + group, err := c.fetchGroup(gid) + if err != nil { + return nil, err + } + rootGroupCache[group.Name] = group + } + // fetch users + for _, uid := range c.UserIDs { + user, err := c.fetchUser(uid) + if err != nil { + return nil, err + } + rootGroupCache[user.Name] = user + } + // fetch current user if configured + if c.IncludeCurrentUser { + currentUser, err := c.fetchCurrentUser() + if err != nil { + return nil, err + } + rootGroupCache[currentUser.Name] = currentUser + } + + c.rootGroupCache = rootGroupCache + } + return c.rootGroupCache, nil +} + +func (c *gitlabClient) FetchGroupContent(gid uint64) (map[string]fs.GroupSource, map[string]fs.RepositorySource, error) { + if slices.Contains[[]int, int](c.UserIDs, int(gid)) || (c.currentUserCache != nil && c.currentUserCache.ID == int(gid)) { + // gid is a user + user, err := c.fetchUser(int(gid)) + if err != nil { + return nil, nil, err + } + return c.fetchUserContent(user) + } else { + // gid is a group + group, err := c.fetchGroup(int(gid)) + if err != nil { + return nil, nil, err + } + return c.fetchGroupContent(group) + } +} diff --git a/gitlab/group.go b/gitlab/group.go index 86a4c90..90fae26 100644 --- a/gitlab/group.go +++ b/gitlab/group.go @@ -4,111 +4,118 @@ import ( "fmt" "sync" + "github.com/badjware/gitlabfs/fs" "github.com/xanzy/go-gitlab" ) -type GroupFetcher interface { - FetchGroup(gid int) (*Group, error) - FetchGroupContent(group *Group) (*GroupContent, error) -} - -type GroupContent struct { - Groups map[string]*Group - Projects map[string]*Project -} - type Group struct { ID int Name string - mux sync.Mutex - content *GroupContent + mux sync.Mutex + + // group content cache + groupCache map[string]fs.GroupSource + projectCache map[string]fs.RepositorySource } -func NewGroupFromGitlabGroup(group *gitlab.Group) Group { - // https://godoc.org/github.com/xanzy/go-gitlab#Group - return Group{ - ID: group.ID, - Name: group.Path, - } +func (g *Group) GetGroupID() uint64 { + return uint64(g.ID) } func (g *Group) InvalidateCache() { g.mux.Lock() defer g.mux.Unlock() - g.content = nil + g.groupCache = nil + g.projectCache = nil } -func (c *gitlabClient) FetchGroup(gid int) (*Group, error) { +func (c *gitlabClient) fetchGroup(gid int) (*Group, error) { + // start by searching the cache + // TODO: cache invalidation? + group, found := c.groupCache[gid] + if found { + return group, nil + } + + // If not in cache, fetch group infos from API gitlabGroup, _, err := c.client.Groups.GetGroup(gid) if err != nil { return nil, fmt.Errorf("failed to fetch group with id %v: %v", gid, err) } - group := NewGroupFromGitlabGroup(gitlabGroup) - return &group, nil + newGroup := Group{ + ID: gitlabGroup.ID, + Name: gitlabGroup.Path, + + groupCache: nil, + projectCache: nil, + } + + // save in cache + c.groupCache[gid] = &newGroup + + return &newGroup, nil } -func (c *gitlabClient) FetchGroupContent(group *Group) (*GroupContent, error) { +func (c *gitlabClient) fetchGroupContent(group *Group) (map[string]fs.GroupSource, map[string]fs.RepositorySource, error) { group.mux.Lock() defer group.mux.Unlock() // Get cached data if available - if group.content != nil { - return group.content, nil - } + // TODO: cache cache invalidation? + if group.groupCache == nil || group.projectCache == nil { + groupCache := make(map[string]fs.GroupSource) + projectCache := make(map[string]fs.RepositorySource) - content := &GroupContent{ - Groups: map[string]*Group{}, - Projects: map[string]*Project{}, - } + // List subgroups in path + ListGroupsOpt := &gitlab.ListSubgroupsOptions{ + ListOptions: gitlab.ListOptions{ + Page: 1, + PerPage: 100, + }, + AllAvailable: gitlab.Bool(true), + } + for { + gitlabGroups, response, err := c.client.Groups.ListSubgroups(group.ID, ListGroupsOpt) + if err != nil { + return nil, nil, fmt.Errorf("failed to fetch groups in gitlab: %v", err) + } + for _, gitlabGroup := range gitlabGroups { + group, _ := c.fetchGroup(gitlabGroup.ID) + groupCache[group.Name] = group + } + if response.CurrentPage >= response.TotalPages { + break + } + // Get the next page + ListGroupsOpt.Page = response.NextPage + } - // List subgroups in path - ListGroupsOpt := &gitlab.ListSubgroupsOptions{ - ListOptions: gitlab.ListOptions{ - Page: 1, - PerPage: 100, - }, - AllAvailable: gitlab.Bool(true), - } - for { - gitlabGroups, response, err := c.client.Groups.ListSubgroups(group.ID, ListGroupsOpt) - if err != nil { - return nil, fmt.Errorf("failed to fetch groups in gitlab: %v", err) + // List projects in path + listProjectOpt := &gitlab.ListGroupProjectsOptions{ + ListOptions: gitlab.ListOptions{ + Page: 1, + PerPage: 100, + }} + for { + gitlabProjects, response, err := c.client.Groups.ListGroupProjects(group.ID, listProjectOpt) + if err != nil { + return nil, nil, fmt.Errorf("failed to fetch projects in gitlab: %v", err) + } + for _, gitlabProject := range gitlabProjects { + project := c.newProjectFromGitlabProject(gitlabProject) + projectCache[project.Name] = &project + } + if response.CurrentPage >= response.TotalPages { + break + } + // Get the next page + listProjectOpt.Page = response.NextPage } - for _, gitlabGroup := range gitlabGroups { - group := NewGroupFromGitlabGroup(gitlabGroup) - content.Groups[group.Name] = &group - } - if response.CurrentPage >= response.TotalPages { - break - } - // Get the next page - ListGroupsOpt.Page = response.NextPage - } - // List projects in path - listProjectOpt := &gitlab.ListGroupProjectsOptions{ - ListOptions: gitlab.ListOptions{ - Page: 1, - PerPage: 100, - }} - for { - gitlabProjects, response, err := c.client.Groups.ListGroupProjects(group.ID, listProjectOpt) - if err != nil { - return nil, fmt.Errorf("failed to fetch projects in gitlab: %v", err) - } - for _, gitlabProject := range gitlabProjects { - project := c.newProjectFromGitlabProject(gitlabProject) - content.Projects[project.Name] = &project - } - if response.CurrentPage >= response.TotalPages { - break - } - // Get the next page - listProjectOpt.Page = response.NextPage + group.groupCache = groupCache + group.projectCache = projectCache } - - group.content = content - return content, nil + return group.groupCache, group.projectCache, nil } diff --git a/gitlab/project.go b/gitlab/project.go index 6f789f5..8a5416c 100644 --- a/gitlab/project.go +++ b/gitlab/project.go @@ -11,6 +11,18 @@ type Project struct { DefaultBranch string } +func (p *Project) GetRepositoryID() uint64 { + return uint64(p.ID) +} + +func (p *Project) GetCloneURL() string { + return p.CloneURL +} + +func (p *Project) GetDefaultBranch() string { + return p.DefaultBranch +} + func (c *gitlabClient) newProjectFromGitlabProject(project *gitlab.Project) Project { // https://godoc.org/github.com/xanzy/go-gitlab#Project p := Project{ diff --git a/gitlab/user.go b/gitlab/user.go index a54af3a..7b6de4e 100644 --- a/gitlab/user.go +++ b/gitlab/user.go @@ -1,103 +1,109 @@ package gitlab import ( - "errors" "fmt" "sync" + "github.com/badjware/gitlabfs/fs" "github.com/xanzy/go-gitlab" ) -type UserFetcher interface { - FetchUser(uid int) (*User, error) - FetchCurrentUser() (*User, error) - FetchUserContent(user *User) (*UserContent, error) -} - -type UserContent struct { - Projects map[string]*Project -} - type User struct { ID int Name string - mux sync.Mutex - content *UserContent + mux sync.Mutex + + // user content cache + projectCache map[string]fs.RepositorySource } -func NewUserFromGitlabUser(user *gitlab.User) User { - // https://godoc.org/github.com/xanzy/go-gitlab#User - return User{ - ID: user.ID, - Name: user.Username, - } +func (u *User) GetGroupID() uint64 { + return uint64(u.ID) } func (u *User) InvalidateCache() { u.mux.Lock() defer u.mux.Unlock() - u.content = nil + u.projectCache = nil } -func (c *gitlabClient) FetchUser(uid int) (*User, error) { +func (c *gitlabClient) fetchUser(uid int) (*User, error) { + // start by searching the cache + // TODO: cache invalidation? + user, found := c.userCache[uid] + if found { + return user, nil + } + + // If not in cache, fetch group infos from API gitlabUser, _, err := c.client.Users.GetUser(uid) if err != nil { return nil, fmt.Errorf("failed to fetch user with id %v: %v", uid, err) } - user := NewUserFromGitlabUser(gitlabUser) - return &user, nil + newUser := User{ + ID: gitlabUser.ID, + Name: gitlabUser.Username, + + projectCache: nil, + } + + // save in cache + c.userCache[uid] = &newUser + + return &newUser, nil } -func (c *gitlabClient) FetchCurrentUser() (*User, error) { - if c.IncludeCurrentUser { +func (c *gitlabClient) fetchCurrentUser() (*User, error) { + if c.currentUserCache == nil { gitlabUser, _, err := c.client.Users.CurrentUser() if err != nil { return nil, fmt.Errorf("failed to fetch current user: %v", err) } - user := NewUserFromGitlabUser(gitlabUser) - return &user, nil + newUser := User{ + ID: gitlabUser.ID, + Name: gitlabUser.Username, + + projectCache: nil, + } + c.currentUserCache = &newUser } - // no current user to fetch, return nil - return nil, errors.New("current user fetch is disabled") + return c.currentUserCache, nil } -func (c *gitlabClient) FetchUserContent(user *User) (*UserContent, error) { +func (c *gitlabClient) fetchUserContent(user *User) (map[string]fs.GroupSource, map[string]fs.RepositorySource, error) { user.mux.Lock() defer user.mux.Unlock() // Get cached data if available - if user.content != nil { - return user.content, nil - } + // TODO: cache cache invalidation? + if user.projectCache == nil { + projectCache := make(map[string]fs.RepositorySource) - content := &UserContent{ - Projects: map[string]*Project{}, - } + // Fetch the user repositories + listProjectOpt := &gitlab.ListProjectsOptions{ + ListOptions: gitlab.ListOptions{ + Page: 1, + PerPage: 100, + }} + for { + gitlabProjects, response, err := c.client.Projects.ListUserProjects(user.ID, listProjectOpt) + if err != nil { + return nil, nil, fmt.Errorf("failed to fetch projects in gitlab: %v", err) + } + for _, gitlabProject := range gitlabProjects { + project := c.newProjectFromGitlabProject(gitlabProject) + projectCache[project.Name] = &project + } + if response.CurrentPage >= response.TotalPages { + break + } + // Get the next page + listProjectOpt.Page = response.NextPage + } - // Fetch the user repositories - listProjectOpt := &gitlab.ListProjectsOptions{ - ListOptions: gitlab.ListOptions{ - Page: 1, - PerPage: 100, - }} - for { - gitlabProjects, response, err := c.client.Projects.ListUserProjects(user.ID, listProjectOpt) - if err != nil { - return nil, fmt.Errorf("failed to fetch projects in gitlab: %v", err) - } - for _, gitlabProject := range gitlabProjects { - project := c.newProjectFromGitlabProject(gitlabProject) - content.Projects[project.Name] = &project - } - if response.CurrentPage >= response.TotalPages { - break - } - // Get the next page - listProjectOpt.Page = response.NextPage + user.projectCache = projectCache } - - user.content = content - return content, nil + return make(map[string]fs.GroupSource), user.projectCache, nil } diff --git a/go.mod b/go.mod index e61a2d5..100b39d 100644 --- a/go.mod +++ b/go.mod @@ -1,18 +1,34 @@ module github.com/badjware/gitlabfs -go 1.15 +go 1.21 + +require ( + github.com/hanwen/go-fuse/v2 v2.1.0 + github.com/vmihailenco/taskq/v3 v3.2.9-0.20211122085105-720ffc56ac4d + github.com/xanzy/go-gitlab v0.47.0 + gopkg.in/yaml.v2 v2.4.0 +) require ( github.com/bsm/redislock v0.7.2 // indirect + github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/go-redis/redis/v8 v8.11.4 // indirect + github.com/go-redis/redis_rate/v9 v9.1.2 // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/hanwen/go-fuse/v2 v2.1.0 github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.6.8 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/klauspost/compress v1.14.4 // indirect - github.com/vmihailenco/taskq/v3 v3.2.9-0.20211122085105-720ffc56ac4d - github.com/xanzy/go-gitlab v0.47.0 + github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558 // indirect + golang.org/x/sys v0.0.0-20210423082822-04245dca01da // indirect golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect google.golang.org/appengine v1.6.7 // indirect - gopkg.in/yaml.v2 v2.4.0 + google.golang.org/protobuf v1.26.0 // indirect ) diff --git a/go.sum b/go.sum index 51a4346..6dccd6e 100644 --- a/go.sum +++ b/go.sum @@ -131,7 +131,6 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/hanwen/go-fuse v1.0.0 h1:GxS9Zrn6c35/BnfiVsZVWmsG803xwE7eVRDvcf/BEVc= github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok= github.com/hanwen/go-fuse/v2 v2.1.0 h1:+32ffteETaLYClUj0a3aHjZ1hOPxxaNEHiZiujuDaek= github.com/hanwen/go-fuse/v2 v2.1.0/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc= @@ -151,11 +150,9 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac h1:w5wltlINIIqRTqQ64dASrCo0fM7k9nosPbKCZnkL0W0= github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac/go.mod h1:gyMTRVO+ZkEy7wQDyD++okPsBN2q127EpuShhHMWG54= -github.com/jeffh/go.bdd v0.0.0-20120717032931-88f798ee0c74 h1:gyfyP8SEIZHs1u2ivTdIbWRtfaKbg5K79d06vnqroJo= github.com/jeffh/go.bdd v0.0.0-20120717032931-88f798ee0c74/go.mod h1:qNa9FlAfO0U/qNkzYBMH1JKYRMzC+sP9IcyV4U18l98= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= diff --git a/main.go b/main.go index cbe3585..8983393 100644 --- a/main.go +++ b/main.go @@ -16,25 +16,17 @@ import ( type ( Config struct { - FS FSConfig `yaml:"fs,omitempty"` - Gitlab GitlabConfig `yaml:"gitlab,omitempty"` - Git GitConfig `yaml:"git,omitempty"` + FS FSConfig `yaml:"fs,omitempty"` + Gitlab gitlab.GitlabClientConfig `yaml:"gitlab,omitempty"` + Git GitConfig `yaml:"git,omitempty"` } FSConfig struct { Mountpoint string `yaml:"mountpoint,omitempty"` MountOptions string `yaml:"mountoptions,omitempty"` } - GitlabConfig struct { - URL string `yaml:"url,omitempty"` - Token string `yaml:"token,omitempty"` - GroupIDs []int `yaml:"group_ids,omitempty"` - UserIDs []int `yaml:"user_ids,omitempty"` - IncludeCurrentUser bool `yaml:"include_current_user,omitempty"` - } GitConfig struct { CloneLocation string `yaml:"clone_location,omitempty"` Remote string `yaml:"remote,omitempty"` - PullMethod string `yaml:"pull_method,omitempty"` OnClone string `yaml:"on_clone,omitempty"` AutoPull bool `yaml:"auto_pull,omitempty"` Depth int `yaml:"depth,omitempty"` @@ -56,17 +48,17 @@ func loadConfig(configPath string) (*Config, error) { Mountpoint: "", MountOptions: "nodev,nosuid", }, - Gitlab: GitlabConfig{ + Gitlab: gitlab.GitlabClientConfig{ URL: "https://gitlab.com", Token: "", GroupIDs: []int{9970}, UserIDs: []int{}, IncludeCurrentUser: true, + PullMethod: "http", }, Git: GitConfig{ CloneLocation: defaultCloneLocation, Remote: "origin", - PullMethod: "http", OnClone: "init", AutoPull: false, Depth: 0, @@ -91,16 +83,13 @@ func loadConfig(configPath string) (*Config, error) { return config, nil } -func makeGitlabConfig(config *Config) (*gitlab.GitlabClientParam, error) { +func makeGitlabConfig(config *Config) (*gitlab.GitlabClientConfig, error) { // parse pull_method - if config.Git.PullMethod != gitlab.PullMethodHTTP && config.Git.PullMethod != gitlab.PullMethodSSH { + if config.Gitlab.PullMethod != gitlab.PullMethodHTTP && config.Gitlab.PullMethod != gitlab.PullMethodSSH { return nil, fmt.Errorf("pull_method must be either \"%v\" or \"%v\"", gitlab.PullMethodHTTP, gitlab.PullMethodSSH) } - return &gitlab.GitlabClientParam{ - PullMethod: config.Git.PullMethod, - IncludeCurrentUser: config.Gitlab.IncludeCurrentUser && config.Gitlab.Token != "", - }, nil + return &config.Gitlab, nil } func makeGitConfig(config *Config) (*git.GitClientParam, error) { @@ -181,18 +170,18 @@ func main() { gitClient, _ := git.NewClient(*gitClientParam) // Create the gitlab client - gitlabClientParam, err := makeGitlabConfig(config) + GitlabClientConfig, err := makeGitlabConfig(config) if err != nil { fmt.Println(err) os.Exit(1) } - gitlabClient, _ := gitlab.NewClient(config.Gitlab.URL, config.Gitlab.Token, *gitlabClientParam) + gitlabClient, _ := gitlab.NewClient(config.Gitlab.URL, config.Gitlab.Token, *GitlabClientConfig) // Start the filesystem err = fs.Start( mountpoint, parsedMountoptions, - &fs.FSParam{Git: gitClient, Gitlab: gitlabClient, RootGroupIds: config.Gitlab.GroupIDs, UserIds: config.Gitlab.UserIDs}, + &fs.FSParam{GitImplementation: gitClient, GitPlatform: gitlabClient}, *debug, ) if err != nil {