From 66fbf713d1adde3c20a5aba941bd8b882ccbb4a3 Mon Sep 17 00:00:00 2001 From: Massaki Archambault Date: Wed, 30 Dec 2020 18:00:37 -0500 Subject: [PATCH] add config file --- .gitignore | 2 + config.example.yaml | 49 ++++++++++++ fs/root.go | 8 +- git/client.go | 21 ----- git/pullclone.go | 109 +++++++++++++------------- gitlab/client.go | 9 ++- gitlab/group.go | 2 +- gitlab/project.go | 16 ++-- gitlab/user.go | 2 +- go.mod | 1 + go.sum | 1 + main.go | 184 ++++++++++++++++++++++++++++++++++++++------ 12 files changed, 291 insertions(+), 113 deletions(-) create mode 100644 config.example.yaml diff --git a/.gitignore b/.gitignore index 53e2045..25ca447 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,5 @@ tags [._]*.un~ # End of https://www.toptal.com/developers/gitignore/api/go,vim,code + +config.yaml \ No newline at end of file diff --git a/config.example.yaml b/config.example.yaml new file mode 100644 index 0000000..53bc87a --- /dev/null +++ b/config.example.yaml @@ -0,0 +1,49 @@ +fs: + # The mountpoint. Can be overwritten via the command line. + #mountpoint: /mnt + +gitlab: + # The gitlab url. + url: https://gitlab.com + + # The gitlab api token. + # Default to anonymous (only public projects will be visible). + #token: + + # A list of the group ids to expose their projects in the filesystem. + group_ids: + - 9970 # gitlab-org + + # A list of the user ids to expose their personal projects in the filesystem. + user_ids: [] + + # If set to true, the user the api token belongs to will automatically be added to the list of users exposed by the filesystem. + include_current_user: true + +git: + # Path to the local repository cache. Repositories in the filesystem will symlink to a folder in this path. + # Default to $XDG_DATA_HOME/gitlabfs, or $HOME/.local/share/gitlabfs if the environment variable $XDG_DATA_HOME is unset. + #clone_location: + + # The name of the remote in the local clone. + remote: origin + + # Must be set to either "http" or "ssh". + # The protocol to configure the git remote on. + pull_method: http + + # Must be set to either "init", "no-checkout" or "checkout". + # If set to "init", the local clone will be initialized with `git init` and set to track the default branch. (fastest) + # If set to "no-checkout", the local clone will be initialized with `git clone --no-checkout`. (slower) + # If set to "checkout", the local clone will be initialized with `git clone`. (slowest) + # NOTE: If set to "init" or "no-checkout", the local clone will appear empty. Running `git pull` will download the files from the git server. + # It's highly recommended to leave this setting on "init". + on_clone: init + + # If set to true, the local clone will automatically run `git pull` in the local clone if it's on the default branch and the worktree is clean. + # Pulls are asynchronous so it can take a few minutes for all repositories to sync up. + # It's highly recommended to leave this setting turned off. + auto_pull: false + + # The depth of the git history to pull. Set to 0 to pull the full history. + depth: 0 \ No newline at end of file diff --git a/fs/root.go b/fs/root.go index 8d2f94c..95009f8 100644 --- a/fs/root.go +++ b/fs/root.go @@ -16,8 +16,8 @@ const ( ) type FSParam struct { - Gitlab gitlab.GitlabFetcher Git git.GitClonerPuller + Gitlab gitlab.GitlabFetcher staticInoChan chan uint64 } @@ -57,13 +57,15 @@ func (n *rootNode) OnAdd(ctx context.Context) { }, ) n.AddChild("users", usersInode, false) + + fmt.Println("Mounted and ready to use") } -func Start(mountpoint string, rootGroupIds []int, userIds []int, param *FSParam) error { +func Start(mountpoint string, rootGroupIds []int, userIds []int, param *FSParam, debug bool) error { fmt.Printf("Mounting in %v\n", mountpoint) opts := &fs.Options{} - opts.Debug = true + opts.Debug = debug param.staticInoChan = make(chan uint64) root := &rootNode{ diff --git a/git/client.go b/git/client.go index be1937e..79aa0e9 100644 --- a/git/client.go +++ b/git/client.go @@ -1,10 +1,7 @@ package git import ( - "errors" "net/url" - "os" - "path/filepath" ) type GitClientParam struct { @@ -13,9 +10,7 @@ type GitClientParam struct { RemoteURL *url.URL Fetch bool Checkout bool - SingleBranch bool PullDepth int - AutoClone bool AutoPull bool ChanBuffSize int @@ -28,22 +23,6 @@ type gitClient struct { } func NewClient(p GitClientParam) (*gitClient, error) { - // Some validations - if p.RemoteURL == nil { - return nil, errors.New("required param RemoteURL is nil") - } - - // Setup defaults - if p.CloneLocation == "" { - dataHome := os.Getenv("XDG_DATA_HOME") - if dataHome == "" { - dataHome = filepath.Join(os.Getenv("HOME"), ".local/share") - } - p.CloneLocation = filepath.Join(dataHome, "gitlabfs") - } - if p.RemoteName == "" { - p.RemoteName = "origin" - } if p.ChanBuffSize == 0 { p.ChanBuffSize = 500 } diff --git a/git/pullclone.go b/git/pullclone.go index ae27f25..cfae533 100644 --- a/git/pullclone.go +++ b/git/pullclone.go @@ -62,63 +62,60 @@ func (c *gitClient) clonePullWorker() { func (c *gitClient) clone(gpp *gitClonePullParam) error { branchRef := plumbing.NewBranchReferenceName(gpp.defaultBranch) - if c.AutoClone { - if c.Fetch { - // Clone the repo - // TODO: figure out why this operation is so memory intensive... - fmt.Printf("Cloning %v into %v\n", gpp.url, gpp.dst) - fs := osfs.New(gpp.dst) - storer := filesystem.NewStorage(fs, cache.NewObjectLRU(0)) - _, err := git.Clone(storer, fs, &git.CloneOptions{ - URL: gpp.url, - RemoteName: c.RemoteName, - ReferenceName: branchRef, - NoCheckout: !c.Checkout, - SingleBranch: c.SingleBranch, - Depth: c.PullDepth, - }) - if err != nil { - return fmt.Errorf("failed to clone git repo %v to %v: %v", gpp.url, gpp.dst, err) - } - } else { - // "Fake" cloning the repo by never actually talking to the git server - // This skip a fetch operation that we would do if we where to do a proper clone - // We can save a lot of time and network i/o doing it this way, at the cost of - // resulting in a very barebone local copy - fmt.Printf("Initializing %v into %v\n", gpp.url, gpp.dst) - r, err := git.PlainInit(gpp.dst, false) - if err != nil { - return fmt.Errorf("failed to clone git repo %v to %v: %v", gpp.url, gpp.dst, err) - } - - // Configure the remote - _, err = r.CreateRemote(&config.RemoteConfig{ - Name: c.RemoteName, - URLs: []string{gpp.url}, - }) - if err != nil { - return fmt.Errorf("failed to setup remote %v in git repo %v: %v", gpp.url, gpp.dst, err) - } - - // Configure a local branch to track the remote branch - err = r.CreateBranch(&config.Branch{ - Name: gpp.defaultBranch, - Remote: c.RemoteName, - Merge: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", gpp.defaultBranch)), - }) - if err != nil { - return fmt.Errorf("failed to create branch %v of git repo %v: %v", gpp.defaultBranch, gpp.dst, err) - } - - // Checkout the default branch - w, err := r.Worktree() - if err != nil { - return fmt.Errorf("failed to retrieve worktree of git repo %v: %v", gpp.dst, err) - } - w.Checkout(&git.CheckoutOptions{ - Branch: branchRef, - }) + if c.Fetch { + // Clone the repo + // TODO: figure out why this operation is so memory intensive... + fmt.Printf("Cloning %v into %v\n", gpp.url, gpp.dst) + fs := osfs.New(gpp.dst) + storer := filesystem.NewStorage(fs, cache.NewObjectLRU(0)) + _, err := git.Clone(storer, fs, &git.CloneOptions{ + URL: gpp.url, + RemoteName: c.RemoteName, + ReferenceName: branchRef, + NoCheckout: !c.Checkout, + Depth: c.PullDepth, + }) + if err != nil { + return fmt.Errorf("failed to clone git repo %v to %v: %v", gpp.url, gpp.dst, err) } + } else { + // "Fake" cloning the repo by never actually talking to the git server + // This skip a fetch operation that we would do if we where to do a proper clone + // We can save a lot of time and network i/o doing it this way, at the cost of + // resulting in a very barebone local copy + fmt.Printf("Initializing %v into %v\n", gpp.url, gpp.dst) + r, err := git.PlainInit(gpp.dst, false) + if err != nil { + return fmt.Errorf("failed to clone git repo %v to %v: %v", gpp.url, gpp.dst, err) + } + + // Configure the remote + _, err = r.CreateRemote(&config.RemoteConfig{ + Name: c.RemoteName, + URLs: []string{gpp.url}, + }) + if err != nil { + return fmt.Errorf("failed to setup remote %v in git repo %v: %v", gpp.url, gpp.dst, err) + } + + // Configure a local branch to track the remote branch + err = r.CreateBranch(&config.Branch{ + Name: gpp.defaultBranch, + Remote: c.RemoteName, + Merge: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", gpp.defaultBranch)), + }) + if err != nil { + return fmt.Errorf("failed to create branch %v of git repo %v: %v", gpp.defaultBranch, gpp.dst, err) + } + + // Checkout the default branch + w, err := r.Worktree() + if err != nil { + return fmt.Errorf("failed to retrieve worktree of git repo %v: %v", gpp.dst, err) + } + w.Checkout(&git.CheckoutOptions{ + Branch: branchRef, + }) } return nil } diff --git a/gitlab/client.go b/gitlab/client.go index 11c8f3e..0776f61 100644 --- a/gitlab/client.go +++ b/gitlab/client.go @@ -6,12 +6,18 @@ import ( "github.com/xanzy/go-gitlab" ) +const ( + PullMethodHTTP = "http" + PullMethodSSH = "ssh" +) + type GitlabFetcher interface { GroupFetcher UserFetcher } type GitlabClientParam struct { + PullMethod string } type gitlabClient struct { @@ -29,7 +35,8 @@ func NewClient(gitlabUrl string, gitlabToken string, p GitlabClientParam) (*gitl } gitlabClient := &gitlabClient{ - client: client, + GitlabClientParam: p, + client: client, } return gitlabClient, nil } diff --git a/gitlab/group.go b/gitlab/group.go index a20c558..f8b3a3c 100644 --- a/gitlab/group.go +++ b/gitlab/group.go @@ -84,7 +84,7 @@ func (c *gitlabClient) FetchGroupContent(group *Group) (*GroupContent, error) { return nil, fmt.Errorf("failed to fetch projects in gitlab: %v", err) } for _, gitlabProject := range gitlabProjects { - project := NewProjectFromGitlabProject(gitlabProject) + project := c.newProjectFromGitlabProject(gitlabProject) content.Projects[project.Name] = &project } if response.CurrentPage >= response.TotalPages { diff --git a/gitlab/project.go b/gitlab/project.go index 695f409..7432bab 100644 --- a/gitlab/project.go +++ b/gitlab/project.go @@ -1,6 +1,8 @@ package gitlab -import "github.com/xanzy/go-gitlab" +import ( + "github.com/xanzy/go-gitlab" +) type Project struct { ID int @@ -8,12 +10,16 @@ type Project struct { CloneURL string } -func NewProjectFromGitlabProject(project *gitlab.Project) Project { +func (c *gitlabClient) newProjectFromGitlabProject(project *gitlab.Project) Project { // https://godoc.org/github.com/xanzy/go-gitlab#Project - return Project{ + p := Project{ ID: project.ID, Name: project.Path, - // CloneURL: project.HTTPURLToRepo, - CloneURL: project.SSHURLToRepo, } + if c.PullMethod == PullMethodSSH { + p.CloneURL = project.SSHURLToRepo + } else { + p.CloneURL = project.HTTPURLToRepo + } + return p } diff --git a/gitlab/user.go b/gitlab/user.go index dd8566c..206e766 100644 --- a/gitlab/user.go +++ b/gitlab/user.go @@ -70,7 +70,7 @@ func (c *gitlabClient) FetchUserContent(user *User) (*UserContent, error) { return nil, fmt.Errorf("failed to fetch projects in gitlab: %v", err) } for _, gitlabProject := range gitlabProjects { - project := NewProjectFromGitlabProject(gitlabProject) + project := c.newProjectFromGitlabProject(gitlabProject) content.Projects[project.Name] = &project } if response.CurrentPage >= response.TotalPages { diff --git a/go.mod b/go.mod index c4d8ec1..0676d05 100644 --- a/go.mod +++ b/go.mod @@ -10,4 +10,5 @@ require ( github.com/hanwen/go-fuse/v2 v2.0.3 github.com/xanzy/go-gitlab v0.40.2 gopkg.in/src-d/go-git.v4 v4.13.1 // indirect + gopkg.in/yaml.v2 v2.2.4 ) diff --git a/go.sum b/go.sum index 90c071a..d0bb9cf 100644 --- a/go.sum +++ b/go.sum @@ -109,4 +109,5 @@ gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQb gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index 14a2cc5..9a427d3 100644 --- a/main.go +++ b/main.go @@ -5,45 +5,179 @@ import ( "fmt" "net/url" "os" + "path/filepath" "github.com/badjware/gitlabfs/fs" "github.com/badjware/gitlabfs/git" "github.com/badjware/gitlabfs/gitlab" + "gopkg.in/yaml.v2" ) -func main() { - gitlabURL := flag.String("gitlab-url", "https://gitlab.com", "the gitlab url") - gitlabToken := flag.String("gitlab-token", "", "the gitlab authentication token") - gitlabRootGroupID := flag.Int("gitlab-group-id", 9970, "the group id of the groups at the root of the filesystem") - // gitlabNamespace := flag.String() - flag.Parse() - if flag.NArg() != 1 { - fmt.Printf("usage: %s MOUNTPOINT\n", os.Args[0]) - os.Exit(2) +const ( + OnCloneInit = "init" + OnCloneNoCheckout = "no-checkout" + OnCloneCheckout = "checkout" +) + +type ( + Config struct { + FS FSConfig `yaml:"fs,omitempty"` + Gitlab GitlabConfig `yaml:"gitlab,omitempty"` + Git GitConfig `yaml:"git,omitempty"` } - mountpoint := flag.Arg(0) - parsedGitlabURL, err := url.Parse(*gitlabURL) + FSConfig struct { + Mountpoint string `yaml:"mountpoint,omitempty"` + } + GitlabConfig struct { + URL string `yaml:"url,omitempty"` + Token string `yaml:"token,omitempty"` + GroupIDs []int `yaml:"group_ids,omitempty"` + UserIDs []int `yaml:"user_ids,omitempty"` + IncludeCurrentUser bool `yaml:"include_current_user,omitempty"` + } + GitConfig struct { + CloneLocation string `yaml:"clone_location,omitempty"` + Remote string `yaml:"remote,omitempty"` + PullMethod string `yaml:"pull_method,omitempty"` + OnClone string `yaml:"on_clone,omitempty"` + AutoPull bool `yaml:"auto_pull,omitempty"` + Depth int `yaml:"depth,omitempty"` + } +) + +func loadConfig(configPath string) (*Config, error) { + // defaults + dataHome := os.Getenv("XDG_DATA_HOME") + if dataHome == "" { + dataHome = filepath.Join(os.Getenv("HOME"), ".local/share") + } + defaultCloneLocation := filepath.Join(dataHome, "gitlabfs") + + config := &Config{ + FS: FSConfig{ + Mountpoint: "", + }, + Gitlab: GitlabConfig{ + URL: "https://gitlab.com", + Token: "", + GroupIDs: []int{9970}, + UserIDs: []int{}, + IncludeCurrentUser: true, + }, + Git: GitConfig{ + CloneLocation: defaultCloneLocation, + Remote: "origin", + PullMethod: "http", + OnClone: "init", + AutoPull: false, + Depth: 0, + }, + } + + if configPath != "" { + f, err := os.Open(configPath) + if err != nil { + return nil, fmt.Errorf("failed to open config file: %v", err) + } + defer f.Close() + + d := yaml.NewDecoder(f) + if err := d.Decode(config); err != nil { + return nil, fmt.Errorf("failed to parse config file: %v", err) + } + } + + return config, nil +} + +func makeGitlabConfig(config *Config) (*gitlab.GitlabClientParam, error) { + // parse pull_method + if config.Git.PullMethod != gitlab.PullMethodHTTP && config.Git.PullMethod != gitlab.PullMethodSSH { + return nil, fmt.Errorf("pull_method must be either \"%v\" or \"%v\"", gitlab.PullMethodHTTP, gitlab.PullMethodSSH) + } + + return &gitlab.GitlabClientParam{ + PullMethod: config.Git.PullMethod, + }, nil +} + +func makeGitConfig(config *Config) (*git.GitClientParam, error) { + // Parse the gilab url + parsedGitlabURL, err := url.Parse(config.Gitlab.URL) if err != nil { - fmt.Printf("%v is not a valid url: %v\n", *gitlabURL, err) + return nil, err + } + + // parse on_clone + fetch := false + checkout := false + if config.Git.OnClone == OnCloneInit { + fetch = false + checkout = false + } else if config.Git.OnClone == OnCloneNoCheckout { + fetch = true + checkout = false + } else if config.Git.OnClone == OnCloneCheckout { + fetch = true + checkout = true + } else { + return nil, fmt.Errorf("on_clone must be either \"%v\", \"%v\" or \"%V\"", OnCloneInit, OnCloneNoCheckout, OnCloneCheckout) + } + + return &git.GitClientParam{ + CloneLocation: config.Git.CloneLocation, + RemoteName: config.Git.Remote, + RemoteURL: parsedGitlabURL, + Fetch: fetch, + Checkout: checkout, + AutoPull: config.Git.AutoPull, + PullDepth: config.Git.Depth, + }, nil +} + +func main() { + configPath := flag.String("config", "", "the config file") + debug := flag.Bool("debug", false, "enable debug logging") + flag.Parse() + + config, err := loadConfig(*configPath) + if err != nil { + fmt.Println(err) os.Exit(1) } - // Create the gitlab client - gitlabClientParam := gitlab.GitlabClientParam{} - gitlabClient, _ := gitlab.NewClient(*gitlabURL, *gitlabToken, gitlabClientParam) + // Configure mountpoint + mountpoint := config.FS.Mountpoint + if flag.NArg() == 1 { + mountpoint = flag.Arg(0) + } + if mountpoint == "" { + fmt.Printf("usage: %s MOUNTPOINT\n", os.Args[0]) + os.Exit(2) + } // Create the git client - gitClientParam := git.GitClientParam{ - RemoteURL: parsedGitlabURL, - AutoClone: true, - AutoPull: false, - Fetch: false, - Checkout: false, - SingleBranch: true, - PullDepth: 0, + gitClientParam, err := makeGitConfig(config) + if err != nil { + fmt.Println(err) + os.Exit(1) } - gitClient, _ := git.NewClient(gitClientParam) + gitClient, _ := git.NewClient(*gitClientParam) + + // Create the gitlab client + gitlabClientParam, err := makeGitlabConfig(config) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + gitlabClient, _ := gitlab.NewClient(config.Gitlab.URL, config.Gitlab.Token, *gitlabClientParam) // Start the filesystem - fs.Start(mountpoint, []int{*gitlabRootGroupID}, []int{}, &fs.FSParam{Gitlab: gitlabClient, Git: gitClient}) + fs.Start( + mountpoint, + config.Gitlab.GroupIDs, + config.Gitlab.UserIDs, + &fs.FSParam{Git: gitClient, Gitlab: gitlabClient}, + *debug, + ) }