badjware
/
gitforgefs
mirror of https://github.com/badjware/gitlabfs.git
1
0
Fork 0
Code Releases Activity

refactor to decouple fs package from gitlab package

This commit is contained in:
Massaki Archambault 2024-05-05 16:09:03 -04:00
parent 026089d786
commit 5ed64d523e
13 changed files with 357 additions and 482 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
fs/group.go
View File

@ -2,9 +2,9 @@ package fs
import (
"context"
"fmt"
"syscall"
"github.com/badjware/gitlabfs/gitlab"
"github.com/hanwen/go-fuse/v2/fs"
"github.com/hanwen/go-fuse/v2/fuse"
)
@ -13,56 +13,50 @@ type groupNode struct {
fs.Inode
param *FSParam
group *gitlab.Group
source GroupSource
staticNodes map[string]staticNode
}
type GroupSource interface {
GetGroupID() uint64
InvalidateCache()
}
// Ensure we are implementing the NodeReaddirer interface
var _ = (fs.NodeReaddirer)((*groupNode)(nil))
// Ensure we are implementing the NodeLookuper interface
var _ = (fs.NodeLookuper)((*groupNode)(nil))
func newGroupNodeByID(gid int, param *FSParam) (*groupNode, error) {
group, err := param.Gitlab.FetchGroup(gid)
if err != nil {
return nil, err
}
func newGroupNodeFromSource(source GroupSource, param *FSParam) (*groupNode, error) {
node := &groupNode{
param: param,
group: group,
param: param,
source: source,
staticNodes: map[string]staticNode{
".refresh": newRefreshNode(group, param),
},
}
return node, nil
}
func newGroupNode(group *gitlab.Group, param *FSParam) (*groupNode, error) {
node := &groupNode{
param: param,
group: group,
staticNodes: map[string]staticNode{
".refresh": newRefreshNode(group, param),
".refresh": newRefreshNode(source, param),
},
}
return node, nil
}
func (n *groupNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) {
groupContent, _ := n.param.Gitlab.FetchGroupContent(n.group)
entries := make([]fuse.DirEntry, 0, len(groupContent.Groups)+len(groupContent.Projects)+len(n.staticNodes))
for _, group := range groupContent.Groups {
groups, repositories, err := n.param.GitPlatform.FetchGroupContent(n.source.GetGroupID())
if err != nil {
fmt.Errorf("%v", err)
}
entries := make([]fuse.DirEntry, 0, len(groups)+len(repositories)+len(n.staticNodes))
for groupName, group := range groups {
entries = append(entries, fuse.DirEntry{
Name: group.Name,
Ino: uint64(group.ID),
Name: groupName,
Ino: group.GetGroupID(),
Mode: fuse.S_IFDIR,
})
}
for _, project := range groupContent.Projects {
for repositoryName, repository := range repositories {
entries = append(entries, fuse.DirEntry{
Name: project.Name,
Ino: uint64(project.ID),
Name: repositoryName,
Ino: repository.GetRepositoryID(),
Mode: fuse.S_IFLNK,
})
}
@ -77,27 +71,27 @@ func (n *groupNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) {
}
func (n *groupNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) {
groupContent, _ := n.param.Gitlab.FetchGroupContent(n.group)
groups, repositories, _ := n.param.GitPlatform.FetchGroupContent(n.source.GetGroupID())
// Check if the map of groups contains it
group, ok := groupContent.Groups[name]
if ok {
group, found := groups[name]
if found {
attrs := fs.StableAttr{
Ino: uint64(group.ID),
Ino: group.GetGroupID(),
Mode: fuse.S_IFDIR,
}
groupNode, _ := newGroupNode(group, n.param)
groupNode, _ := newGroupNodeFromSource(group, n.param)
return n.NewInode(ctx, groupNode, attrs), 0
}
// Check if the map of projects contains it
project, ok := groupContent.Projects[name]
if ok {
repository, found := repositories[name]
if found {
attrs := fs.StableAttr{
Ino: uint64(project.ID),
Ino: repository.GetRepositoryID(),
Mode: fuse.S_IFLNK,
}
repositoryNode, _ := newRepositoryNode(project, n.param)
repositoryNode, _ := newRepositoryNodeFromSource(repository, n.param)
return n.NewInode(ctx, repositoryNode, attrs), 0
}

47
fs/groups.go
View File

@ -1,47 +0,0 @@
package fs
import (
"context"
"fmt"
"github.com/hanwen/go-fuse/v2/fs"
"github.com/hanwen/go-fuse/v2/fuse"
)
type groupsNode struct {
fs.Inode
param *FSParam
rootGroupIds []int
}
// Ensure we are implementing the NodeOnAdder interface
var _ = (fs.NodeOnAdder)((*groupsNode)(nil))
func newGroupsNode(rootGroupIds []int, param *FSParam) *groupsNode {
return &groupsNode{
param: param,
rootGroupIds: rootGroupIds,
}
}
func (n *groupsNode) OnAdd(ctx context.Context) {
for _, groupID := range n.rootGroupIds {
groupNode, err := newGroupNodeByID(groupID, n.param)
if err != nil {
fmt.Printf("root group fetch fail: %v\n", err)
fmt.Printf("Please verify the group exists, is public or a token with sufficient permissions is set in the config files.\n")
fmt.Printf("Skipping group %v\n", groupID)
return
}
inode := n.NewPersistentInode(
ctx,
groupNode,
fs.StableAttr{
Ino: <-n.param.staticInoChan,
Mode: fuse.S_IFDIR,
},
)
n.AddChild(groupNode.group.Name, inode, false)
}
}

14
fs/refresh.go
View File

@ -4,15 +4,15 @@ import (
"context"
"syscall"
"github.com/badjware/gitlabfs/gitlab"
"github.com/hanwen/go-fuse/v2/fs"
"github.com/hanwen/go-fuse/v2/fuse"
)
type refreshNode struct {
fs.Inode
ino uint64
refresher gitlab.Refresher
ino uint64
source GroupSource
}
// Ensure we are implementing the NodeSetattrer interface
@ -21,10 +21,10 @@ var _ = (fs.NodeSetattrer)((*refreshNode)(nil))
// Ensure we are implementing the NodeOpener interface
var _ = (fs.NodeOpener)((*refreshNode)(nil))
func newRefreshNode(refresher gitlab.Refresher, param *FSParam) *refreshNode {
func newRefreshNode(source GroupSource, param *FSParam) *refreshNode {
return &refreshNode{
ino: <-param.staticInoChan,
refresher: refresher,
ino: <-param.staticInoChan,
source: source,
}
}
@ -41,6 +41,6 @@ func (n *refreshNode) Setattr(ctx context.Context, fh fs.FileHandle, in *fuse.Se
}
func (n *refreshNode) Open(ctx context.Context, flags uint32) (fh fs.FileHandle, fuseFlags uint32, errno syscall.Errno) {
n.refresher.InvalidateCache()
n.source.InvalidateCache()
return nil, 0, 0
}

24
fs/repository.go
View File

@ -4,23 +4,30 @@ import (
"context"
"syscall"
"github.com/badjware/gitlabfs/gitlab"
"github.com/hanwen/go-fuse/v2/fs"
)
type RepositoryNode struct {
fs.Inode
param *FSParam
project *gitlab.Project
param *FSParam
source RepositorySource
}
type RepositorySource interface {
// GetName() string
GetRepositoryID() uint64
GetCloneURL() string
GetDefaultBranch() string
}
// Ensure we are implementing the NodeReaddirer interface
var _ = (fs.NodeReadlinker)((*RepositoryNode)(nil))
func newRepositoryNode(project *gitlab.Project, param *FSParam) (*RepositoryNode, error) {
func newRepositoryNodeFromSource(source RepositorySource, param *FSParam) (*RepositoryNode, error) {
node := &RepositoryNode{
param: param,
project: project,
param: param,
source: source,
}
// Passthrough the error if there is one, nothing to add here
// Errors on clone/pull are non-fatal
@ -29,7 +36,8 @@ func newRepositoryNode(project *gitlab.Project, param *FSParam) (*RepositoryNode
func (n *RepositoryNode) Readlink(ctx context.Context) ([]byte, syscall.Errno) {
// Create the local copy of the repo
localRepoLoc, _ := n.param.Git.CloneOrPull(n.project.CloneURL, n.project.ID, n.project.DefaultBranch)
// TODO: cleanup
localRepositoryPath, _ := n.param.GitImplementation.CloneOrPull(n.source.GetCloneURL(), int(n.source.GetRepositoryID()), n.source.GetDefaultBranch())
return []byte(localRepoLoc), 0
return []byte(localRepositoryPath), 0
}

74
fs/root.go
View File

@ -8,8 +8,6 @@ import (
"syscall"
"github.com/badjware/gitlabfs/git"
"github.com/badjware/gitlabfs/gitlab"
"github.com/hanwen/go-fuse/v2/fs"
"github.com/hanwen/go-fuse/v2/fuse"
)
@ -24,55 +22,25 @@ type staticNode interface {
Mode() uint32
}
type FSParam struct {
Git git.GitClonerPuller
Gitlab gitlab.GitlabFetcher
type GitPlatform interface {
FetchRootGroupContent() (map[string]GroupSource, error)
FetchGroupContent(gid uint64) (map[string]GroupSource, map[string]RepositorySource, error)
}
RootGroupIds []int
UserIds []int
type FSParam struct {
GitImplementation git.GitClonerPuller
GitPlatform GitPlatform
staticInoChan chan uint64
}
type rootNode struct {
fs.Inode
param *FSParam
rootGroupIds []int
userIds []int
param *FSParam
}
var _ = (fs.NodeOnAdder)((*rootNode)(nil))
func (n *rootNode) OnAdd(ctx context.Context) {
groupsInode := n.NewPersistentInode(
ctx,
newGroupsNode(
n.rootGroupIds,
n.param,
),
fs.StableAttr{
Ino: <-n.param.staticInoChan,
Mode: fuse.S_IFDIR,
},
)
n.AddChild("groups", groupsInode, false)
usersInode := n.NewPersistentInode(
ctx,
newUsersNode(
n.userIds,
n.param,
),
fs.StableAttr{
Ino: <-n.param.staticInoChan,
Mode: fuse.S_IFDIR,
},
)
n.AddChild("users", usersInode, false)
fmt.Println("Mounted and ready to use")
}
func Start(mountpoint string, mountoptions []string, param *FSParam, debug bool) error {
fmt.Printf("Mounting in %v\n", mountpoint)
@ -82,9 +50,7 @@ func Start(mountpoint string, mountoptions []string, param *FSParam, debug bool)
param.staticInoChan = make(chan uint64)
root := &rootNode{
param: param,
rootGroupIds: param.RootGroupIds,
userIds: param.UserIds,
param: param,
}
go staticInoGenerator(root.param.staticInoChan)
@ -104,6 +70,28 @@ func Start(mountpoint string, mountoptions []string, param *FSParam, debug bool)
return nil
}
func (n *rootNode) OnAdd(ctx context.Context) {
rootGroups, err := n.param.GitPlatform.FetchRootGroupContent()
if err != nil {
panic(err)
}
for groupName, group := range rootGroups {
groupNode, _ := newGroupNodeFromSource(group, n.param)
persistentInode := n.NewPersistentInode(
ctx,
groupNode,
fs.StableAttr{
Ino: <-n.param.staticInoChan,
Mode: fuse.S_IFDIR,
},
)
n.AddChild(groupName, persistentInode, false)
}
fmt.Println("Mounted and ready to use")
}
func staticInoGenerator(staticInoChan chan<- uint64) {
i := staticInodeStart
for {

159
fs/users.go
View File

@ -1,159 +0,0 @@
package fs
import (
"context"
"fmt"
"syscall"
"github.com/badjware/gitlabfs/gitlab"
"github.com/hanwen/go-fuse/v2/fs"
"github.com/hanwen/go-fuse/v2/fuse"
)
type usersNode struct {
fs.Inode
param *FSParam
userIds []int
}
// Ensure we are implementing the NodeOnAdder interface
var _ = (fs.NodeOnAdder)((*usersNode)(nil))
func newUsersNode(userIds []int, param *FSParam) *usersNode {
return &usersNode{
param: param,
userIds: userIds,
}
}
func (n *usersNode) OnAdd(ctx context.Context) {
// Fetch the current logged user
currentUser, err := n.param.Gitlab.FetchCurrentUser()
// Skip if we are anonymous (or the call fails for some reason...)
if err != nil {
fmt.Println(err)
} else {
currentUserNode, _ := newUserNode(currentUser, n.param)
inode := n.NewPersistentInode(
ctx,
currentUserNode,
fs.StableAttr{
Ino: <-n.param.staticInoChan,
Mode: fuse.S_IFDIR,
},
)
n.AddChild(currentUserNode.user.Name, inode, false)
}
for _, userID := range n.userIds {
if currentUser != nil && currentUser.ID == userID {
// We already added the current user, we can skip it
continue
}
userNode, err := newUserNodeByID(userID, n.param)
if err != nil {
fmt.Printf("user fetch fail: %v\n", err)
fmt.Printf("Please verify the user exists and token with sufficient permissions is set in the config files.\n")
fmt.Printf("Skipping user %v\n", userID)
return
}
inode := n.NewPersistentInode(
ctx,
userNode,
fs.StableAttr{
Ino: <-n.param.staticInoChan,
Mode: fuse.S_IFDIR,
},
)
n.AddChild(userNode.user.Name, inode, false)
}
}
type userNode struct {
fs.Inode
param *FSParam
user *gitlab.User
staticNodes map[string]staticNode
}
// Ensure we are implementing the NodeReaddirer interface
var _ = (fs.NodeReaddirer)((*userNode)(nil))
// Ensure we are implementing the NodeLookuper interface
var _ = (fs.NodeLookuper)((*userNode)(nil))
func newUserNodeByID(uid int, param *FSParam) (*userNode, error) {
user, err := param.Gitlab.FetchUser(uid)
if err != nil {
return nil, err
}
node := &userNode{
param: param,
user: user,
staticNodes: map[string]staticNode{
".refresh": newRefreshNode(user, param),
},
}
return node, nil
}
func newUserNode(user *gitlab.User, param *FSParam) (*userNode, error) {
node := &userNode{
param: param,
user: user,
staticNodes: map[string]staticNode{
".refresh": newRefreshNode(user, param),
},
}
return node, nil
}
func (n *userNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) {
userContent, _ := n.param.Gitlab.FetchUserContent(n.user)
entries := make([]fuse.DirEntry, 0, len(userContent.Projects)+len(n.staticNodes))
for _, project := range userContent.Projects {
entries = append(entries, fuse.DirEntry{
Name: project.Name,
Ino: uint64(project.ID),
Mode: fuse.S_IFLNK,
})
}
for name, staticNode := range n.staticNodes {
entries = append(entries, fuse.DirEntry{
Name: name,
Ino: staticNode.Ino(),
Mode: staticNode.Mode(),
})
}
return fs.NewListDirStream(entries), 0
}
func (n *userNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) {
userContent, _ := n.param.Gitlab.FetchUserContent(n.user)
// Check if the map of projects contains it
project, ok := userContent.Projects[name]
if ok {
attrs := fs.StableAttr{
Ino: uint64(project.ID),
Mode: fuse.S_IFLNK,
}
repositoryNode, _ := newRepositoryNode(project, n.param)
return n.NewInode(ctx, repositoryNode, attrs), 0
}
// Check if the map of static nodes contains it
staticNode, ok := n.staticNodes[name]
if ok {
attrs := fs.StableAttr{
Ino: staticNode.Ino(),
Mode: staticNode.Mode(),
}
return n.NewInode(ctx, staticNode, attrs), 0
}
return nil, syscall.ENOENT
}

96
gitlab/client.go
View File

@ -2,7 +2,9 @@ package gitlab
import (
"fmt"
"slices"
"github.com/badjware/gitlabfs/fs"
"github.com/xanzy/go-gitlab"
)
@ -11,26 +13,29 @@ const (
PullMethodSSH = "ssh"
)
type GitlabFetcher interface {
GroupFetcher
UserFetcher
}
type Refresher interface {
InvalidateCache()
}
type GitlabClientParam struct {
PullMethod string
IncludeCurrentUser bool
type GitlabClientConfig struct {
URL string `yaml:"url,omitempty"`
Token string `yaml:"token,omitempty"`
GroupIDs []int `yaml:"group_ids,omitempty"`
UserIDs []int `yaml:"user_ids,omitempty"`
IncludeCurrentUser bool `yaml:"include_current_user,omitempty"`
PullMethod string `yaml:"pull_method,omitempty"`
}
type gitlabClient struct {
GitlabClientParam
GitlabClientConfig
client *gitlab.Client
// root group cache
rootGroupCache map[string]fs.GroupSource
currentUserCache *User
// API response cache
groupCache map[int]*Group
userCache map[int]*User
}
func NewClient(gitlabUrl string, gitlabToken string, p GitlabClientParam) (*gitlabClient, error) {
func NewClient(gitlabUrl string, gitlabToken string, p GitlabClientConfig) (*gitlabClient, error) {
client, err := gitlab.NewClient(
gitlabToken,
gitlab.WithBaseURL(gitlabUrl),
@ -40,8 +45,67 @@ func NewClient(gitlabUrl string, gitlabToken string, p GitlabClientParam) (*gitl
}
gitlabClient := &gitlabClient{
GitlabClientParam: p,
client: client,
GitlabClientConfig: p,
client: client,
rootGroupCache: nil,
currentUserCache: nil,
groupCache: map[int]*Group{},
userCache: map[int]*User{},
}
return gitlabClient, nil
}
func (c *gitlabClient) FetchRootGroupContent() (map[string]fs.GroupSource, error) {
// use cached values if available
if c.rootGroupCache == nil {
rootGroupCache := make(map[string]fs.GroupSource)
// fetch root groups
for _, gid := range c.GroupIDs {
group, err := c.fetchGroup(gid)
if err != nil {
return nil, err
}
rootGroupCache[group.Name] = group
}
// fetch users
for _, uid := range c.UserIDs {
user, err := c.fetchUser(uid)
if err != nil {
return nil, err
}
rootGroupCache[user.Name] = user
}
// fetch current user if configured
if c.IncludeCurrentUser {
currentUser, err := c.fetchCurrentUser()
if err != nil {
return nil, err
}
rootGroupCache[currentUser.Name] = currentUser
}
c.rootGroupCache = rootGroupCache
}
return c.rootGroupCache, nil
}
func (c *gitlabClient) FetchGroupContent(gid uint64) (map[string]fs.GroupSource, map[string]fs.RepositorySource, error) {
if slices.Contains[[]int, int](c.UserIDs, int(gid)) || (c.currentUserCache != nil && c.currentUserCache.ID == int(gid)) {
// gid is a user
user, err := c.fetchUser(int(gid))
if err != nil {
return nil, nil, err
}
return c.fetchUserContent(user)
} else {
// gid is a group
group, err := c.fetchGroup(int(gid))
if err != nil {
return nil, nil, err
}
return c.fetchGroupContent(group)
}
}

157
gitlab/group.go
View File

@ -4,111 +4,118 @@ import (
"fmt"
"sync"
"github.com/badjware/gitlabfs/fs"
"github.com/xanzy/go-gitlab"
)
type GroupFetcher interface {
FetchGroup(gid int) (*Group, error)
FetchGroupContent(group *Group) (*GroupContent, error)
}
type GroupContent struct {
Groups map[string]*Group
Projects map[string]*Project
}
type Group struct {
ID int
Name string
mux sync.Mutex
content *GroupContent
mux sync.Mutex
// group content cache
groupCache map[string]fs.GroupSource
projectCache map[string]fs.RepositorySource
}
func NewGroupFromGitlabGroup(group *gitlab.Group) Group {
// https://godoc.org/github.com/xanzy/go-gitlab#Group
return Group{
ID: group.ID,
Name: group.Path,
}
func (g *Group) GetGroupID() uint64 {
return uint64(g.ID)
}
func (g *Group) InvalidateCache() {
g.mux.Lock()
defer g.mux.Unlock()
g.content = nil
g.groupCache = nil
g.projectCache = nil
}
func (c *gitlabClient) FetchGroup(gid int) (*Group, error) {
func (c *gitlabClient) fetchGroup(gid int) (*Group, error) {
// start by searching the cache
// TODO: cache invalidation?
group, found := c.groupCache[gid]
if found {
return group, nil
}
// If not in cache, fetch group infos from API
gitlabGroup, _, err := c.client.Groups.GetGroup(gid)
if err != nil {
return nil, fmt.Errorf("failed to fetch group with id %v: %v", gid, err)
}
group := NewGroupFromGitlabGroup(gitlabGroup)
return &group, nil
newGroup := Group{
ID: gitlabGroup.ID,
Name: gitlabGroup.Path,
groupCache: nil,
projectCache: nil,
}
// save in cache
c.groupCache[gid] = &newGroup
return &newGroup, nil
}
func (c *gitlabClient) FetchGroupContent(group *Group) (*GroupContent, error) {
func (c *gitlabClient) fetchGroupContent(group *Group) (map[string]fs.GroupSource, map[string]fs.RepositorySource, error) {
group.mux.Lock()
defer group.mux.Unlock()
// Get cached data if available
if group.content != nil {
return group.content, nil
}
// TODO: cache cache invalidation?
if group.groupCache == nil || group.projectCache == nil {
groupCache := make(map[string]fs.GroupSource)
projectCache := make(map[string]fs.RepositorySource)
content := &GroupContent{
Groups: map[string]*Group{},
Projects: map[string]*Project{},
}
// List subgroups in path
ListGroupsOpt := &gitlab.ListSubgroupsOptions{
ListOptions: gitlab.ListOptions{
Page: 1,
PerPage: 100,
},
AllAvailable: gitlab.Bool(true),
}
for {
gitlabGroups, response, err := c.client.Groups.ListSubgroups(group.ID, ListGroupsOpt)
if err != nil {
return nil, nil, fmt.Errorf("failed to fetch groups in gitlab: %v", err)
}
for _, gitlabGroup := range gitlabGroups {
group, _ := c.fetchGroup(gitlabGroup.ID)
groupCache[group.Name] = group
}
if response.CurrentPage >= response.TotalPages {
break
}
// Get the next page
ListGroupsOpt.Page = response.NextPage
}
// List subgroups in path
ListGroupsOpt := &gitlab.ListSubgroupsOptions{
ListOptions: gitlab.ListOptions{
Page: 1,
PerPage: 100,
},
AllAvailable: gitlab.Bool(true),
}
for {
gitlabGroups, response, err := c.client.Groups.ListSubgroups(group.ID, ListGroupsOpt)
if err != nil {
return nil, fmt.Errorf("failed to fetch groups in gitlab: %v", err)
// List projects in path
listProjectOpt := &gitlab.ListGroupProjectsOptions{
ListOptions: gitlab.ListOptions{
Page: 1,
PerPage: 100,
}}
for {
gitlabProjects, response, err := c.client.Groups.ListGroupProjects(group.ID, listProjectOpt)
if err != nil {
return nil, nil, fmt.Errorf("failed to fetch projects in gitlab: %v", err)
}
for _, gitlabProject := range gitlabProjects {
project := c.newProjectFromGitlabProject(gitlabProject)
projectCache[project.Name] = &project
}
if response.CurrentPage >= response.TotalPages {
break
}
// Get the next page
listProjectOpt.Page = response.NextPage
}
for _, gitlabGroup := range gitlabGroups {
group := NewGroupFromGitlabGroup(gitlabGroup)
content.Groups[group.Name] = &group
}
if response.CurrentPage >= response.TotalPages {
break
}
// Get the next page
ListGroupsOpt.Page = response.NextPage
}
// List projects in path
listProjectOpt := &gitlab.ListGroupProjectsOptions{
ListOptions: gitlab.ListOptions{
Page: 1,
PerPage: 100,
}}
for {
gitlabProjects, response, err := c.client.Groups.ListGroupProjects(group.ID, listProjectOpt)
if err != nil {
return nil, fmt.Errorf("failed to fetch projects in gitlab: %v", err)
}
for _, gitlabProject := range gitlabProjects {
project := c.newProjectFromGitlabProject(gitlabProject)
content.Projects[project.Name] = &project
}
if response.CurrentPage >= response.TotalPages {
break
}
// Get the next page
listProjectOpt.Page = response.NextPage
group.groupCache = groupCache
group.projectCache = projectCache
}
group.content = content
return content, nil
return group.groupCache, group.projectCache, nil
}

12
gitlab/project.go
View File

@ -11,6 +11,18 @@ type Project struct {
DefaultBranch string
}
func (p *Project) GetRepositoryID() uint64 {
return uint64(p.ID)
}
func (p *Project) GetCloneURL() string {
return p.CloneURL
}
func (p *Project) GetDefaultBranch() string {
return p.DefaultBranch
}
func (c *gitlabClient) newProjectFromGitlabProject(project *gitlab.Project) Project {
// https://godoc.org/github.com/xanzy/go-gitlab#Project
p := Project{

124
gitlab/user.go
View File

@ -1,103 +1,109 @@
package gitlab
import (
"errors"
"fmt"
"sync"
"github.com/badjware/gitlabfs/fs"
"github.com/xanzy/go-gitlab"
)
type UserFetcher interface {
FetchUser(uid int) (*User, error)
FetchCurrentUser() (*User, error)
FetchUserContent(user *User) (*UserContent, error)
}
type UserContent struct {
Projects map[string]*Project
}
type User struct {
ID int
Name string
mux sync.Mutex
content *UserContent
mux sync.Mutex
// user content cache
projectCache map[string]fs.RepositorySource
}
func NewUserFromGitlabUser(user *gitlab.User) User {
// https://godoc.org/github.com/xanzy/go-gitlab#User
return User{
ID: user.ID,
Name: user.Username,
}
func (u *User) GetGroupID() uint64 {
return uint64(u.ID)
}
func (u *User) InvalidateCache() {
u.mux.Lock()
defer u.mux.Unlock()
u.content = nil
u.projectCache = nil
}
func (c *gitlabClient) FetchUser(uid int) (*User, error) {
func (c *gitlabClient) fetchUser(uid int) (*User, error) {
// start by searching the cache
// TODO: cache invalidation?
user, found := c.userCache[uid]
if found {
return user, nil
}
// If not in cache, fetch group infos from API
gitlabUser, _, err := c.client.Users.GetUser(uid)
if err != nil {
return nil, fmt.Errorf("failed to fetch user with id %v: %v", uid, err)
}
user := NewUserFromGitlabUser(gitlabUser)
return &user, nil
newUser := User{
ID: gitlabUser.ID,
Name: gitlabUser.Username,
projectCache: nil,
}
// save in cache
c.userCache[uid] = &newUser
return &newUser, nil
}
func (c *gitlabClient) FetchCurrentUser() (*User, error) {
if c.IncludeCurrentUser {
func (c *gitlabClient) fetchCurrentUser() (*User, error) {
if c.currentUserCache == nil {
gitlabUser, _, err := c.client.Users.CurrentUser()
if err != nil {
return nil, fmt.Errorf("failed to fetch current user: %v", err)
}
user := NewUserFromGitlabUser(gitlabUser)
return &user, nil
newUser := User{
ID: gitlabUser.ID,
Name: gitlabUser.Username,
projectCache: nil,
}
c.currentUserCache = &newUser
}
// no current user to fetch, return nil
return nil, errors.New("current user fetch is disabled")
return c.currentUserCache, nil
}
func (c *gitlabClient) FetchUserContent(user *User) (*UserContent, error) {
func (c *gitlabClient) fetchUserContent(user *User) (map[string]fs.GroupSource, map[string]fs.RepositorySource, error) {
user.mux.Lock()
defer user.mux.Unlock()
// Get cached data if available
if user.content != nil {
return user.content, nil
}
// TODO: cache cache invalidation?
if user.projectCache == nil {
projectCache := make(map[string]fs.RepositorySource)
content := &UserContent{
Projects: map[string]*Project{},
}
// Fetch the user repositories
listProjectOpt := &gitlab.ListProjectsOptions{
ListOptions: gitlab.ListOptions{
Page: 1,
PerPage: 100,
}}
for {
gitlabProjects, response, err := c.client.Projects.ListUserProjects(user.ID, listProjectOpt)
if err != nil {
return nil, nil, fmt.Errorf("failed to fetch projects in gitlab: %v", err)
}
for _, gitlabProject := range gitlabProjects {
project := c.newProjectFromGitlabProject(gitlabProject)
projectCache[project.Name] = &project
}
if response.CurrentPage >= response.TotalPages {
break
}
// Get the next page
listProjectOpt.Page = response.NextPage
}
// Fetch the user repositories
listProjectOpt := &gitlab.ListProjectsOptions{
ListOptions: gitlab.ListOptions{
Page: 1,
PerPage: 100,
}}
for {
gitlabProjects, response, err := c.client.Projects.ListUserProjects(user.ID, listProjectOpt)
if err != nil {
return nil, fmt.Errorf("failed to fetch projects in gitlab: %v", err)
}
for _, gitlabProject := range gitlabProjects {
project := c.newProjectFromGitlabProject(gitlabProject)
content.Projects[project.Name] = &project
}
if response.CurrentPage >= response.TotalPages {
break
}
// Get the next page
listProjectOpt.Page = response.NextPage
user.projectCache = projectCache
}
user.content = content
return content, nil
return make(map[string]fs.GroupSource), user.projectCache, nil
}

26
go.mod
View File

@ -1,18 +1,34 @@
module github.com/badjware/gitlabfs
go 1.15
go 1.21
require (
github.com/hanwen/go-fuse/v2 v2.1.0
github.com/vmihailenco/taskq/v3 v3.2.9-0.20211122085105-720ffc56ac4d
github.com/xanzy/go-gitlab v0.47.0
gopkg.in/yaml.v2 v2.4.0
)
require (
github.com/bsm/redislock v0.7.2 // indirect
github.com/capnm/sysinfo v0.0.0-20130621111458-5909a53897f3 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/go-redis/redis/v8 v8.11.4 // indirect
github.com/go-redis/redis_rate/v9 v9.1.2 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/hanwen/go-fuse/v2 v2.1.0
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.6.8 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/klauspost/compress v1.14.4 // indirect
github.com/vmihailenco/taskq/v3 v3.2.9-0.20211122085105-720ffc56ac4d
github.com/xanzy/go-gitlab v0.47.0
github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558 // indirect
golang.org/x/sys v0.0.0-20210423082822-04245dca01da // indirect
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
google.golang.org/appengine v1.6.7 // indirect
gopkg.in/yaml.v2 v2.4.0
google.golang.org/protobuf v1.26.0 // indirect
)

3
go.sum
View File

@ -131,7 +131,6 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/hanwen/go-fuse v1.0.0 h1:GxS9Zrn6c35/BnfiVsZVWmsG803xwE7eVRDvcf/BEVc=
github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok=
github.com/hanwen/go-fuse/v2 v2.1.0 h1:+32ffteETaLYClUj0a3aHjZ1hOPxxaNEHiZiujuDaek=
github.com/hanwen/go-fuse/v2 v2.1.0/go.mod h1:oRyA5eK+pvJyv5otpO/DgccS8y/RvYMaO00GgRLGryc=
@ -151,11 +150,9 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac h1:w5wltlINIIqRTqQ64dASrCo0fM7k9nosPbKCZnkL0W0=
github.com/iron-io/iron_go3 v0.0.0-20190916120531-a4a7f74b73ac/go.mod h1:gyMTRVO+ZkEy7wQDyD++okPsBN2q127EpuShhHMWG54=
github.com/jeffh/go.bdd v0.0.0-20120717032931-88f798ee0c74 h1:gyfyP8SEIZHs1u2ivTdIbWRtfaKbg5K79d06vnqroJo=
github.com/jeffh/go.bdd v0.0.0-20120717032931-88f798ee0c74/go.mod h1:qNa9FlAfO0U/qNkzYBMH1JKYRMzC+sP9IcyV4U18l98=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=

33
main.go
View File

@ -16,25 +16,17 @@ import (
type (
Config struct {
FS FSConfig `yaml:"fs,omitempty"`
Gitlab GitlabConfig `yaml:"gitlab,omitempty"`
Git GitConfig `yaml:"git,omitempty"`
FS FSConfig `yaml:"fs,omitempty"`
Gitlab gitlab.GitlabClientConfig `yaml:"gitlab,omitempty"`
Git GitConfig `yaml:"git,omitempty"`
}
FSConfig struct {
Mountpoint string `yaml:"mountpoint,omitempty"`
MountOptions string `yaml:"mountoptions,omitempty"`
}
GitlabConfig struct {
URL string `yaml:"url,omitempty"`
Token string `yaml:"token,omitempty"`
GroupIDs []int `yaml:"group_ids,omitempty"`
UserIDs []int `yaml:"user_ids,omitempty"`
IncludeCurrentUser bool `yaml:"include_current_user,omitempty"`
}
GitConfig struct {
CloneLocation string `yaml:"clone_location,omitempty"`
Remote string `yaml:"remote,omitempty"`
PullMethod string `yaml:"pull_method,omitempty"`
OnClone string `yaml:"on_clone,omitempty"`
AutoPull bool `yaml:"auto_pull,omitempty"`
Depth int `yaml:"depth,omitempty"`
@ -56,17 +48,17 @@ func loadConfig(configPath string) (*Config, error) {
Mountpoint: "",
MountOptions: "nodev,nosuid",
},
Gitlab: GitlabConfig{
Gitlab: gitlab.GitlabClientConfig{
URL: "https://gitlab.com",
Token: "",
GroupIDs: []int{9970},
UserIDs: []int{},
IncludeCurrentUser: true,
PullMethod: "http",
},
Git: GitConfig{
CloneLocation: defaultCloneLocation,
Remote: "origin",
PullMethod: "http",
OnClone: "init",
AutoPull: false,
Depth: 0,
@ -91,16 +83,13 @@ func loadConfig(configPath string) (*Config, error) {
return config, nil
}
func makeGitlabConfig(config *Config) (*gitlab.GitlabClientParam, error) {
func makeGitlabConfig(config *Config) (*gitlab.GitlabClientConfig, error) {
// parse pull_method
if config.Git.PullMethod != gitlab.PullMethodHTTP && config.Git.PullMethod != gitlab.PullMethodSSH {
if config.Gitlab.PullMethod != gitlab.PullMethodHTTP && config.Gitlab.PullMethod != gitlab.PullMethodSSH {
return nil, fmt.Errorf("pull_method must be either \"%v\" or \"%v\"", gitlab.PullMethodHTTP, gitlab.PullMethodSSH)
}
return &gitlab.GitlabClientParam{
PullMethod: config.Git.PullMethod,
IncludeCurrentUser: config.Gitlab.IncludeCurrentUser && config.Gitlab.Token != "",
}, nil
return &config.Gitlab, nil
}
func makeGitConfig(config *Config) (*git.GitClientParam, error) {
@ -181,18 +170,18 @@ func main() {
gitClient, _ := git.NewClient(*gitClientParam)
// Create the gitlab client
gitlabClientParam, err := makeGitlabConfig(config)
GitlabClientConfig, err := makeGitlabConfig(config)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
gitlabClient, _ := gitlab.NewClient(config.Gitlab.URL, config.Gitlab.Token, *gitlabClientParam)
gitlabClient, _ := gitlab.NewClient(config.Gitlab.URL, config.Gitlab.Token, *GitlabClientConfig)
// Start the filesystem
err = fs.Start(
mountpoint,
parsedMountoptions,
&fs.FSParam{Git: gitClient, Gitlab: gitlabClient, RootGroupIds: config.Gitlab.GroupIDs, UserIds: config.Gitlab.UserIDs},
&fs.FSParam{GitImplementation: gitClient, GitPlatform: gitlabClient},
*debug,
)
if err != nil {