From 735a803cdbd06543b9cbb9c6633377fa329f0811 Mon Sep 17 00:00:00 2001 From: Massaki Archambault Date: Sun, 5 May 2024 19:52:57 -0400 Subject: [PATCH] refactor to decouple fstree package from git package --- fstree/group.go | 2 +- fstree/repository.go | 7 ++++-- fstree/root.go | 9 ++++--- git/client.go | 56 +++++++++++++++++++++++--------------------- git/clone.go | 12 +++++----- git/pull.go | 6 ++--- main.go | 40 ++++--------------------------- 7 files changed, 55 insertions(+), 77 deletions(-) diff --git a/fstree/group.go b/fstree/group.go index de0978f..76782dd 100644 --- a/fstree/group.go +++ b/fstree/group.go @@ -42,7 +42,7 @@ func newGroupNodeFromSource(source GroupSource, param *FSParam) (*groupNode, err func (n *groupNode) Readdir(ctx context.Context) (fs.DirStream, syscall.Errno) { groups, repositories, err := n.param.GitPlatform.FetchGroupContent(n.source.GetGroupID()) if err != nil { - fmt.Errorf("%v", err) + fmt.Println(err) } entries := make([]fuse.DirEntry, 0, len(groups)+len(repositories)+len(n.staticNodes)) diff --git a/fstree/repository.go b/fstree/repository.go index d37fd3d..d8fbbfe 100644 --- a/fstree/repository.go +++ b/fstree/repository.go @@ -2,6 +2,7 @@ package fstree import ( "context" + "fmt" "syscall" "github.com/hanwen/go-fuse/v2/fs" @@ -37,7 +38,9 @@ func newRepositoryNodeFromSource(source RepositorySource, param *FSParam) (*repo func (n *repositoryNode) Readlink(ctx context.Context) ([]byte, syscall.Errno) { // Create the local copy of the repo // TODO: cleanup - localRepositoryPath, _ := n.param.GitImplementation.CloneOrPull(n.source.GetCloneURL(), int(n.source.GetRepositoryID()), n.source.GetDefaultBranch()) - + localRepositoryPath, err := n.param.GitClient.FetchLocalRepositoryPath(n.source) + if err != nil { + fmt.Println(err) + } return []byte(localRepositoryPath), 0 } diff --git a/fstree/root.go b/fstree/root.go index 5159ca9..0ba9b4f 100644 --- a/fstree/root.go +++ b/fstree/root.go @@ -7,7 +7,6 @@ import ( "os/signal" "syscall" - "github.com/badjware/gitlabfs/git" "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" ) @@ -22,14 +21,18 @@ type staticNode interface { Mode() uint32 } +type GitClient interface { + FetchLocalRepositoryPath(source RepositorySource) (string, error) +} + type GitPlatform interface { FetchRootGroupContent() (map[string]GroupSource, error) FetchGroupContent(gid uint64) (map[string]GroupSource, map[string]RepositorySource, error) } type FSParam struct { - GitImplementation git.GitClonerPuller - GitPlatform GitPlatform + GitClient GitClient + GitPlatform GitPlatform staticInoChan chan uint64 } diff --git a/git/client.go b/git/client.go index 868e255..fa73f7d 100644 --- a/git/client.go +++ b/git/client.go @@ -2,39 +2,33 @@ package git import ( "context" - "net/url" + "fmt" "os" "path/filepath" + "regexp" "strconv" "time" + "github.com/badjware/gitlabfs/fstree" "github.com/vmihailenco/taskq/v3" "github.com/vmihailenco/taskq/v3/memqueue" ) -const ( - CloneInit = iota - CloneClone = iota -) - -type GitClonerPuller interface { - CloneOrPull(url string, pid int, defaultBranch string) (localRepoLoc string, err error) -} - type GitClientParam struct { - CloneLocation string - RemoteName string - RemoteURL *url.URL - CloneMethod int - PullDepth int - AutoPull bool - - QueueSize int - QueueWorkerCount int + CloneLocation string `yaml:"clone_location,omitempty"` + Remote string `yaml:"remote,omitempty"` + OnClone string `yaml:"on_clone,omitempty"` + AutoPull bool `yaml:"auto_pull,omitempty"` + Depth int `yaml:"depth,omitempty"` + QueueSize int `yaml:"queue_size,omitempty"` + QueueWorkerCount int `yaml:"worker_count,omitempty"` } type gitClient struct { GitClientParam + + hostnameProg *regexp.Regexp + queue taskq.Queue cloneTask *taskq.Task pullTask *taskq.Task @@ -46,6 +40,8 @@ func NewClient(p GitClientParam) (*gitClient, error) { c := &gitClient{ GitClientParam: p, + hostnameProg: regexp.MustCompile(`([a-z0-1]+\.)+[a-z0-1]+`), + queue: queueFactory.RegisterQueue(&taskq.QueueOptions{ Name: "git-queue", MaxNumWorker: int32(p.QueueWorkerCount), @@ -68,21 +64,27 @@ func NewClient(p GitClientParam) (*gitClient, error) { return c, nil } -func (c *gitClient) getLocalRepoLoc(pid int) string { - return filepath.Join(c.CloneLocation, c.RemoteURL.Hostname(), strconv.Itoa(pid)) -} +func (c *gitClient) FetchLocalRepositoryPath(source fstree.RepositorySource) (localRepoLoc string, err error) { + rid := source.GetRepositoryID() + cloneUrl := source.GetCloneURL() + defaultBranch := source.GetDefaultBranch() -func (c *gitClient) CloneOrPull(url string, pid int, defaultBranch string) (localRepoLoc string, err error) { - localRepoLoc = c.getLocalRepoLoc(pid) + // Parse the url + hostname := c.hostnameProg.FindString(cloneUrl) + if hostname == "" { + return "", fmt.Errorf("failed to match a valid hostname from \"%v\"", cloneUrl) + } + + localRepoLoc = filepath.Join(c.CloneLocation, hostname, strconv.Itoa(int(rid))) if _, err := os.Stat(localRepoLoc); os.IsNotExist(err) { // Dispatch clone msg - msg := c.cloneTask.WithArgs(context.Background(), url, defaultBranch, localRepoLoc) - msg.OnceInPeriod(time.Second, pid) + msg := c.cloneTask.WithArgs(context.Background(), cloneUrl, defaultBranch, localRepoLoc) + msg.OnceInPeriod(time.Second, rid) c.queue.Add(msg) } else if c.AutoPull { // Dispatch pull msg msg := c.pullTask.WithArgs(context.Background(), localRepoLoc, defaultBranch) - msg.OnceInPeriod(time.Second, pid) + msg.OnceInPeriod(time.Second, rid) c.queue.Add(msg) } return localRepoLoc, nil diff --git a/git/clone.go b/git/clone.go index 8a92920..10b9f3b 100644 --- a/git/clone.go +++ b/git/clone.go @@ -8,7 +8,7 @@ import ( ) func (c *gitClient) clone(url string, defaultBranch string, dst string) error { - if c.CloneMethod == CloneInit { + if c.GitClientParam.OnClone == "init" { // "Fake" cloning the repo by never actually talking to the git server // This skip a fetch operation that we would do if we where to do a proper clone // We can save a lot of time and network i/o doing it this way, at the cost of @@ -32,8 +32,8 @@ func (c *gitClient) clone(url string, defaultBranch string, dst string) error { "git", "remote", "add", "-m", defaultBranch, "--", - c.RemoteName, // name - url, // url + c.GitClientParam.Remote, // name + url, // url ) if err != nil { return fmt.Errorf("failed to setup remote %v in git repo %v: %v", url, dst, err) @@ -45,7 +45,7 @@ func (c *gitClient) clone(url string, defaultBranch string, dst string) error { "git", "config", "--local", "--", fmt.Sprintf("branch.%s.remote", defaultBranch), // key - c.RemoteName, // value + c.GitClientParam.Remote, // value ) if err != nil { @@ -66,8 +66,8 @@ func (c *gitClient) clone(url string, defaultBranch string, dst string) error { // Clone the repo _, err := utils.ExecProcess( "git", "clone", - "--origin", c.RemoteName, - "--depth", strconv.Itoa(c.PullDepth), + "--origin", c.GitClientParam.Remote, + "--depth", strconv.Itoa(c.GitClientParam.Depth), "--", url, // repository dst, // directory diff --git a/git/pull.go b/git/pull.go index 31096a4..b53d908 100644 --- a/git/pull.go +++ b/git/pull.go @@ -23,10 +23,10 @@ func (c *gitClient) pull(repoPath string, defaultBranch string) error { _, err = utils.ExecProcessInDir( repoPath, // workdir "git", "pull", - "--depth", strconv.Itoa(c.PullDepth), + "--depth", strconv.Itoa(c.GitClientParam.Depth), "--", - c.RemoteName, // repository - defaultBranch, // refspec + c.GitClientParam.Remote, // repository + defaultBranch, // refspec ) if err != nil { return fmt.Errorf("failed to pull git repo %v: %v", repoPath, err) diff --git a/main.go b/main.go index 2ef9960..e10355e 100644 --- a/main.go +++ b/main.go @@ -3,7 +3,6 @@ package main import ( "flag" "fmt" - "net/url" "os" "path/filepath" "strings" @@ -18,21 +17,12 @@ type ( Config struct { FS FSConfig `yaml:"fs,omitempty"` Gitlab gitlab.GitlabClientConfig `yaml:"gitlab,omitempty"` - Git GitConfig `yaml:"git,omitempty"` + Git git.GitClientParam `yaml:"git,omitempty"` } FSConfig struct { Mountpoint string `yaml:"mountpoint,omitempty"` MountOptions string `yaml:"mountoptions,omitempty"` } - GitConfig struct { - CloneLocation string `yaml:"clone_location,omitempty"` - Remote string `yaml:"remote,omitempty"` - OnClone string `yaml:"on_clone,omitempty"` - AutoPull bool `yaml:"auto_pull,omitempty"` - Depth int `yaml:"depth,omitempty"` - QueueSize int `yaml:"queue_size,omitempty"` - QueueWorkerCount int `yaml:"worker_count,omitempty"` - } ) func loadConfig(configPath string) (*Config, error) { @@ -56,7 +46,7 @@ func loadConfig(configPath string) (*Config, error) { IncludeCurrentUser: true, PullMethod: "http", }, - Git: GitConfig{ + Git: git.GitClientParam{ CloneLocation: defaultCloneLocation, Remote: "origin", OnClone: "init", @@ -93,32 +83,12 @@ func makeGitlabConfig(config *Config) (*gitlab.GitlabClientConfig, error) { } func makeGitConfig(config *Config) (*git.GitClientParam, error) { - // Parse the gilab url - parsedGitlabURL, err := url.Parse(config.Gitlab.URL) - if err != nil { - return nil, err - } - // parse on_clone - cloneMethod := 0 - if config.Git.OnClone == "init" { - cloneMethod = git.CloneInit - } else if config.Git.OnClone == "clone" { - cloneMethod = git.CloneClone - } else { + if config.Git.OnClone != "init" && config.Git.OnClone != "clone" { return nil, fmt.Errorf("on_clone must be either \"init\" or \"clone\"") } - return &git.GitClientParam{ - CloneLocation: config.Git.CloneLocation, - RemoteName: config.Git.Remote, - RemoteURL: parsedGitlabURL, - CloneMethod: cloneMethod, - AutoPull: config.Git.AutoPull, - PullDepth: config.Git.Depth, - QueueSize: config.Git.QueueSize, - QueueWorkerCount: config.Git.QueueWorkerCount, - }, nil + return &config.Git, nil } func main() { @@ -181,7 +151,7 @@ func main() { err = fstree.Start( mountpoint, parsedMountoptions, - &fstree.FSParam{GitImplementation: gitClient, GitPlatform: gitlabClient}, + &fstree.FSParam{GitClient: gitClient, GitPlatform: gitlabClient}, *debug, ) if err != nil {