diff --git a/fstree/group.go b/fstree/group.go index a60337c..9f0ade0 100644 --- a/fstree/group.go +++ b/fstree/group.go @@ -70,38 +70,41 @@ func (n *groupNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) { } func (n *groupNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fs.Inode, syscall.Errno) { - groups, repositories, _ := n.param.GitPlatform.FetchGroupContent(n.source.GetGroupID()) - - // Check if the map of groups contains it - group, found := groups[name] - if found { - attrs := fs.StableAttr{ - Ino: group.GetGroupID(), - Mode: fuse.S_IFDIR, + groups, repositories, err := n.param.GitPlatform.FetchGroupContent(n.source.GetGroupID()) + if err != nil { + n.param.logger.Error(err.Error()) + } else { + // Check if the map of groups contains it + group, found := groups[name] + if found { + attrs := fs.StableAttr{ + Ino: group.GetGroupID(), + Mode: fuse.S_IFDIR, + } + groupNode, _ := newGroupNodeFromSource(group, n.param) + return n.NewInode(ctx, groupNode, attrs), 0 } - groupNode, _ := newGroupNodeFromSource(group, n.param) - return n.NewInode(ctx, groupNode, attrs), 0 - } - // Check if the map of projects contains it - repository, found := repositories[name] - if found { - attrs := fs.StableAttr{ - Ino: repository.GetRepositoryID(), - Mode: fuse.S_IFLNK, + // Check if the map of projects contains it + repository, found := repositories[name] + if found { + attrs := fs.StableAttr{ + Ino: repository.GetRepositoryID(), + Mode: fuse.S_IFLNK, + } + repositoryNode, _ := newRepositoryNodeFromSource(repository, n.param) + return n.NewInode(ctx, repositoryNode, attrs), 0 } - repositoryNode, _ := newRepositoryNodeFromSource(repository, n.param) - return n.NewInode(ctx, repositoryNode, attrs), 0 - } - // Check if the map of static nodes contains it - staticNode, ok := n.staticNodes[name] - if ok { - attrs := fs.StableAttr{ - Ino: staticNode.Ino(), - Mode: staticNode.Mode(), + // Check if the map of static nodes contains it + staticNode, ok := n.staticNodes[name] + if ok { + attrs := fs.StableAttr{ + Ino: staticNode.Ino(), + Mode: staticNode.Mode(), + } + return n.NewInode(ctx, staticNode, attrs), 0 } - return n.NewInode(ctx, staticNode, attrs), 0 } return nil, syscall.ENOENT diff --git a/main.go b/main.go index 3a0d421..0af7f52 100644 --- a/main.go +++ b/main.go @@ -73,7 +73,7 @@ func main() { fmt.Println(err) os.Exit(1) } - gitPlatformClient, _ = gitlab.NewClient(logger, loadedConfig.Gitlab.URL, loadedConfig.Gitlab.Token, *GitlabClientConfig) + gitPlatformClient, _ = gitlab.NewClient(logger, *GitlabClientConfig) } else if loadedConfig.FS.Platform == config.PlatformGithub { // Create the github client GithubClientConfig, err := config.MakeGithubConfig(loadedConfig) diff --git a/platforms/github/client.go b/platforms/github/client.go index 7a3624c..8381519 100644 --- a/platforms/github/client.go +++ b/platforms/github/client.go @@ -1,7 +1,9 @@ package github import ( + "fmt" "log/slog" + "sync" "github.com/badjware/gitlabfs/config" "github.com/badjware/gitlabfs/fstree" @@ -15,6 +17,11 @@ type githubClient struct { logger *slog.Logger rootContent map[string]fstree.GroupSource + + // API response cache + organizationCacheMux sync.RWMutex + organizationNameToIDMap map[string]int64 + organizationCache map[int64]*Organization } func NewClient(logger *slog.Logger, config config.GithubClientConfig) (*githubClient, error) { @@ -28,15 +35,38 @@ func NewClient(logger *slog.Logger, config config.GithubClientConfig) (*githubCl client: client, logger: logger, + + rootContent: nil, + + organizationNameToIDMap: map[string]int64{}, + organizationCache: map[int64]*Organization{}, } return gitHubClient, nil } func (c *githubClient) FetchRootGroupContent() (map[string]fstree.GroupSource, error) { - return nil, nil + if c.rootContent == nil { + rootContent := make(map[string]fstree.GroupSource) + + for _, org_name := range c.GithubClientConfig.OrgNames { + org, err := c.fetchOrganization(org_name) + if err != nil { + c.logger.Warn(err.Error()) + } else { + rootContent[org.Name] = org + } + } + // TODO: user + current user + + c.rootContent = rootContent + } + return c.rootContent, nil } func (c *githubClient) FetchGroupContent(gid uint64) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { - return nil, nil, nil + if org, found := c.organizationCache[int64(gid)]; found { + return c.fetchOrganizationContent(org) + } + return nil, nil, fmt.Errorf("invalid gid: %v", gid) } diff --git a/platforms/github/organization.go b/platforms/github/organization.go index d2e73c2..ffee699 100644 --- a/platforms/github/organization.go +++ b/platforms/github/organization.go @@ -1 +1,104 @@ package github + +import ( + "context" + "fmt" + "sync" + + "github.com/badjware/gitlabfs/fstree" + "github.com/google/go-github/v63/github" +) + +type Organization struct { + ID int64 + Name string + + mux sync.Mutex + + // hold org content + childRepositories map[string]fstree.RepositorySource +} + +func (o *Organization) GetGroupID() uint64 { + return uint64(o.ID) +} + +func (o *Organization) InvalidateContentCache() { + o.mux.Lock() + defer o.mux.Unlock() + + // clear child repositories from cache + o.childRepositories = nil +} + +func (c *githubClient) fetchOrganization(org_name string) (*Organization, error) { + c.organizationCacheMux.RLock() + cachedId, found := c.organizationNameToIDMap[org_name] + if found { + cachedOrg := c.organizationCache[cachedId] + c.organizationCacheMux.RUnlock() + + // if found in cache, return the cached reference + c.logger.Debug("Organization cache hit", "org_name", org_name) + return cachedOrg, nil + } else { + c.organizationCacheMux.RUnlock() + + c.logger.Debug("Organization cache miss", "org_name", org_name) + } + + // If not found in cache, fetch organization infos from API + githubOrg, _, err := c.client.Organizations.Get(context.Background(), org_name) + if err != nil { + return nil, fmt.Errorf("failed to fetch organization with name %v: %v", org_name, err) + } + newOrg := Organization{ + ID: *githubOrg.ID, + Name: *githubOrg.Login, + + childRepositories: nil, + } + + // save in cache + c.organizationCacheMux.Lock() + c.organizationCache[newOrg.ID] = &newOrg + c.organizationCacheMux.Unlock() + + return &newOrg, nil +} + +func (c *githubClient) fetchOrganizationContent(org *Organization) (map[string]fstree.GroupSource, map[string]fstree.RepositorySource, error) { + org.mux.Lock() + defer org.mux.Unlock() + + // Get cached data if available + // TODO: cache cache invalidation? + if org.childRepositories == nil { + childRepositories := make(map[string]fstree.RepositorySource) + + // Fetch the organization repositories + repositoryListOpt := &github.RepositoryListByOrgOptions{ + ListOptions: github.ListOptions{PerPage: 100}, + } + for { + githubRepositories, response, err := c.client.Repositories.ListByOrg(context.Background(), org.Name, repositoryListOpt) + if err != nil { + return nil, nil, fmt.Errorf("failed to fetch repository in github: %v", err) + } + for _, githubRepository := range githubRepositories { + repository := c.newRepositoryFromGithubRepository(githubRepository) + if repository != nil { + childRepositories[repository.Path] = repository + } + } + if response.NextPage == 0 { + break + } + // Get the next page + repositoryListOpt.Page = response.NextPage + } + + org.childRepositories = childRepositories + } + return make(map[string]fstree.GroupSource), org.childRepositories, nil +} diff --git a/platforms/github/repository.go b/platforms/github/repository.go new file mode 100644 index 0000000..abbf482 --- /dev/null +++ b/platforms/github/repository.go @@ -0,0 +1,50 @@ +package github + +import ( + "path" + + "github.com/badjware/gitlabfs/config" + "github.com/google/go-github/v63/github" +) + +type Repository struct { + ID int64 + Path string + CloneURL string + DefaultBranch string +} + +func (r *Repository) GetRepositoryID() uint64 { + return uint64(r.ID) +} + +func (r *Repository) GetCloneURL() string { + return r.CloneURL +} + +func (r *Repository) GetDefaultBranch() string { + return r.DefaultBranch +} + +func (c *githubClient) newRepositoryFromGithubRepository(repository *github.Repository) *Repository { + if c.ArchivedRepoHandling == config.ArchivedProjectIgnore && *repository.Archived { + return nil + } + r := Repository{ + ID: *repository.ID, + Path: *repository.Name, + DefaultBranch: *repository.DefaultBranch, + } + if r.DefaultBranch == "" { + r.DefaultBranch = "master" + } + if c.PullMethod == config.PullMethodSSH { + r.CloneURL = *repository.SSHURL + } else { + r.CloneURL = *repository.CloneURL + } + if c.ArchivedRepoHandling == config.ArchivedProjectHide && *repository.Archived { + r.Path = path.Join(path.Dir(r.Path), "."+path.Base(r.Path)) + } + return &r +} diff --git a/platforms/gitlab/client.go b/platforms/gitlab/client.go index 44e2000..080744a 100644 --- a/platforms/gitlab/client.go +++ b/platforms/gitlab/client.go @@ -28,17 +28,17 @@ type gitlabClient struct { userCache map[int]*User } -func NewClient(logger *slog.Logger, gitlabUrl string, gitlabToken string, p config.GitlabClientConfig) (*gitlabClient, error) { +func NewClient(logger *slog.Logger, config config.GitlabClientConfig) (*gitlabClient, error) { client, err := gitlab.NewClient( - gitlabToken, - gitlab.WithBaseURL(gitlabUrl), + config.Token, + gitlab.WithBaseURL(config.URL), ) if err != nil { return nil, fmt.Errorf("failed to create gitlab client: %v", err) } gitlabClient := &gitlabClient{ - GitlabClientConfig: p, + GitlabClientConfig: config, client: client, logger: logger,