badjware
/
gitforgefs
mirror of https://github.com/badjware/gitlabfs.git
1
0
Fork 0
Code Releases Activity

add config file

This commit is contained in:
Massaki Archambault 2020-12-30 18:00:37 -05:00
parent 9b09b6aef9
commit 66fbf713d1
12 changed files with 291 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -52,3 +52,5 @@ tags
[._]*.un~
# End of https://www.toptal.com/developers/gitignore/api/go,vim,code
config.yaml

49
config.example.yaml Normal file
View File

@ -0,0 +1,49 @@
fs:
# The mountpoint. Can be overwritten via the command line.
#mountpoint: /mnt
gitlab:
# The gitlab url.
url: https://gitlab.com
# The gitlab api token.
# Default to anonymous (only public projects will be visible).
#token:
# A list of the group ids to expose their projects in the filesystem.
group_ids:
- 9970 # gitlab-org
# A list of the user ids to expose their personal projects in the filesystem.
user_ids: []
# If set to true, the user the api token belongs to will automatically be added to the list of users exposed by the filesystem.
include_current_user: true
git:
# Path to the local repository cache. Repositories in the filesystem will symlink to a folder in this path.
# Default to $XDG_DATA_HOME/gitlabfs, or $HOME/.local/share/gitlabfs if the environment variable $XDG_DATA_HOME is unset.
#clone_location:
# The name of the remote in the local clone.
remote: origin
# Must be set to either "http" or "ssh".
# The protocol to configure the git remote on.
pull_method: http
# Must be set to either "init", "no-checkout" or "checkout".
# If set to "init", the local clone will be initialized with `git init` and set to track the default branch. (fastest)
# If set to "no-checkout", the local clone will be initialized with `git clone --no-checkout`. (slower)
# If set to "checkout", the local clone will be initialized with `git clone`. (slowest)
# NOTE: If set to "init" or "no-checkout", the local clone will appear empty. Running `git pull` will download the files from the git server.
# It's highly recommended to leave this setting on "init".
on_clone: init
# If set to true, the local clone will automatically run `git pull` in the local clone if it's on the default branch and the worktree is clean.
# Pulls are asynchronous so it can take a few minutes for all repositories to sync up.
# It's highly recommended to leave this setting turned off.
auto_pull: false
# The depth of the git history to pull. Set to 0 to pull the full history.
depth: 0

8
fs/root.go
View File

@ -16,8 +16,8 @@ const (
)
type FSParam struct {
Gitlab gitlab.GitlabFetcher
Git git.GitClonerPuller
Gitlab gitlab.GitlabFetcher
staticInoChan chan uint64
}
@ -57,13 +57,15 @@ func (n *rootNode) OnAdd(ctx context.Context) {
},
)
n.AddChild("users", usersInode, false)
fmt.Println("Mounted and ready to use")
}
func Start(mountpoint string, rootGroupIds []int, userIds []int, param *FSParam) error {
func Start(mountpoint string, rootGroupIds []int, userIds []int, param *FSParam, debug bool) error {
fmt.Printf("Mounting in %v\n", mountpoint)
opts := &fs.Options{}
opts.Debug = true
opts.Debug = debug
param.staticInoChan = make(chan uint64)
root := &rootNode{

21
git/client.go
View File

@ -1,10 +1,7 @@
package git
import (
"errors"
"net/url"
"os"
"path/filepath"
)
type GitClientParam struct {
@ -13,9 +10,7 @@ type GitClientParam struct {
RemoteURL *url.URL
Fetch bool
Checkout bool
SingleBranch bool
PullDepth int
AutoClone bool
AutoPull bool
ChanBuffSize int
@ -28,22 +23,6 @@ type gitClient struct {
}
func NewClient(p GitClientParam) (*gitClient, error) {
// Some validations
if p.RemoteURL == nil {
return nil, errors.New("required param RemoteURL is nil")
}
// Setup defaults
if p.CloneLocation == "" {
dataHome := os.Getenv("XDG_DATA_HOME")
if dataHome == "" {
dataHome = filepath.Join(os.Getenv("HOME"), ".local/share")
}
p.CloneLocation = filepath.Join(dataHome, "gitlabfs")
}
if p.RemoteName == "" {
p.RemoteName = "origin"
}
if p.ChanBuffSize == 0 {
p.ChanBuffSize = 500
}

109
git/pullclone.go
View File

@ -62,63 +62,60 @@ func (c *gitClient) clonePullWorker() {
func (c *gitClient) clone(gpp *gitClonePullParam) error {
branchRef := plumbing.NewBranchReferenceName(gpp.defaultBranch)
if c.AutoClone {
if c.Fetch {
// Clone the repo
// TODO: figure out why this operation is so memory intensive...
fmt.Printf("Cloning %v into %v\n", gpp.url, gpp.dst)
fs := osfs.New(gpp.dst)
storer := filesystem.NewStorage(fs, cache.NewObjectLRU(0))
_, err := git.Clone(storer, fs, &git.CloneOptions{
URL: gpp.url,
RemoteName: c.RemoteName,
ReferenceName: branchRef,
NoCheckout: !c.Checkout,
SingleBranch: c.SingleBranch,
Depth: c.PullDepth,
})
if err != nil {
return fmt.Errorf("failed to clone git repo %v to %v: %v", gpp.url, gpp.dst, err)
}
} else {
// "Fake" cloning the repo by never actually talking to the git server
// This skip a fetch operation that we would do if we where to do a proper clone
// We can save a lot of time and network i/o doing it this way, at the cost of
// resulting in a very barebone local copy
fmt.Printf("Initializing %v into %v\n", gpp.url, gpp.dst)
r, err := git.PlainInit(gpp.dst, false)
if err != nil {
return fmt.Errorf("failed to clone git repo %v to %v: %v", gpp.url, gpp.dst, err)
}
// Configure the remote
_, err = r.CreateRemote(&config.RemoteConfig{
Name: c.RemoteName,
URLs: []string{gpp.url},
})
if err != nil {
return fmt.Errorf("failed to setup remote %v in git repo %v: %v", gpp.url, gpp.dst, err)
}
// Configure a local branch to track the remote branch
err = r.CreateBranch(&config.Branch{
Name: gpp.defaultBranch,
Remote: c.RemoteName,
Merge: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", gpp.defaultBranch)),
})
if err != nil {
return fmt.Errorf("failed to create branch %v of git repo %v: %v", gpp.defaultBranch, gpp.dst, err)
}
// Checkout the default branch
w, err := r.Worktree()
if err != nil {
return fmt.Errorf("failed to retrieve worktree of git repo %v: %v", gpp.dst, err)
}
w.Checkout(&git.CheckoutOptions{
Branch: branchRef,
})
if c.Fetch {
// Clone the repo
// TODO: figure out why this operation is so memory intensive...
fmt.Printf("Cloning %v into %v\n", gpp.url, gpp.dst)
fs := osfs.New(gpp.dst)
storer := filesystem.NewStorage(fs, cache.NewObjectLRU(0))
_, err := git.Clone(storer, fs, &git.CloneOptions{
URL: gpp.url,
RemoteName: c.RemoteName,
ReferenceName: branchRef,
NoCheckout: !c.Checkout,
Depth: c.PullDepth,
})
if err != nil {
return fmt.Errorf("failed to clone git repo %v to %v: %v", gpp.url, gpp.dst, err)
}
} else {
// "Fake" cloning the repo by never actually talking to the git server
// This skip a fetch operation that we would do if we where to do a proper clone
// We can save a lot of time and network i/o doing it this way, at the cost of
// resulting in a very barebone local copy
fmt.Printf("Initializing %v into %v\n", gpp.url, gpp.dst)
r, err := git.PlainInit(gpp.dst, false)
if err != nil {
return fmt.Errorf("failed to clone git repo %v to %v: %v", gpp.url, gpp.dst, err)
}
// Configure the remote
_, err = r.CreateRemote(&config.RemoteConfig{
Name: c.RemoteName,
URLs: []string{gpp.url},
})
if err != nil {
return fmt.Errorf("failed to setup remote %v in git repo %v: %v", gpp.url, gpp.dst, err)
}
// Configure a local branch to track the remote branch
err = r.CreateBranch(&config.Branch{
Name: gpp.defaultBranch,
Remote: c.RemoteName,
Merge: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", gpp.defaultBranch)),
})
if err != nil {
return fmt.Errorf("failed to create branch %v of git repo %v: %v", gpp.defaultBranch, gpp.dst, err)
}
// Checkout the default branch
w, err := r.Worktree()
if err != nil {
return fmt.Errorf("failed to retrieve worktree of git repo %v: %v", gpp.dst, err)
}
w.Checkout(&git.CheckoutOptions{
Branch: branchRef,
})
}
return nil
}

9
gitlab/client.go
View File

@ -6,12 +6,18 @@ import (
"github.com/xanzy/go-gitlab"
)
const (
PullMethodHTTP = "http"
PullMethodSSH = "ssh"
)
type GitlabFetcher interface {
GroupFetcher
UserFetcher
}
type GitlabClientParam struct {
PullMethod string
}
type gitlabClient struct {
@ -29,7 +35,8 @@ func NewClient(gitlabUrl string, gitlabToken string, p GitlabClientParam) (*gitl
}
gitlabClient := &gitlabClient{
client: client,
GitlabClientParam: p,
client: client,
}
return gitlabClient, nil
}

2
gitlab/group.go
View File

@ -84,7 +84,7 @@ func (c *gitlabClient) FetchGroupContent(group *Group) (*GroupContent, error) {
return nil, fmt.Errorf("failed to fetch projects in gitlab: %v", err)
}
for _, gitlabProject := range gitlabProjects {
project := NewProjectFromGitlabProject(gitlabProject)
project := c.newProjectFromGitlabProject(gitlabProject)
content.Projects[project.Name] = &project
}
if response.CurrentPage >= response.TotalPages {

16
gitlab/project.go
View File

@ -1,6 +1,8 @@
package gitlab
import "github.com/xanzy/go-gitlab"
import (
"github.com/xanzy/go-gitlab"
)
type Project struct {
ID int
@ -8,12 +10,16 @@ type Project struct {
CloneURL string
}
func NewProjectFromGitlabProject(project *gitlab.Project) Project {
func (c *gitlabClient) newProjectFromGitlabProject(project *gitlab.Project) Project {
// https://godoc.org/github.com/xanzy/go-gitlab#Project
return Project{
p := Project{
ID: project.ID,
Name: project.Path,
// CloneURL: project.HTTPURLToRepo,
CloneURL: project.SSHURLToRepo,
}
if c.PullMethod == PullMethodSSH {
p.CloneURL = project.SSHURLToRepo
} else {
p.CloneURL = project.HTTPURLToRepo
}
return p
}

2
gitlab/user.go
View File

@ -70,7 +70,7 @@ func (c *gitlabClient) FetchUserContent(user *User) (*UserContent, error) {
return nil, fmt.Errorf("failed to fetch projects in gitlab: %v", err)
}
for _, gitlabProject := range gitlabProjects {
project := NewProjectFromGitlabProject(gitlabProject)
project := c.newProjectFromGitlabProject(gitlabProject)
content.Projects[project.Name] = &project
}
if response.CurrentPage >= response.TotalPages {

1
go.mod
View File

@ -10,4 +10,5 @@ require (
github.com/hanwen/go-fuse/v2 v2.0.3
github.com/xanzy/go-gitlab v0.40.2
gopkg.in/src-d/go-git.v4 v4.13.1 // indirect
gopkg.in/yaml.v2 v2.2.4
)

1
go.sum
View File

@ -109,4 +109,5 @@ gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQb
gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

184
main.go
View File

@ -5,45 +5,179 @@ import (
"fmt"
"net/url"
"os"
"path/filepath"
"github.com/badjware/gitlabfs/fs"
"github.com/badjware/gitlabfs/git"
"github.com/badjware/gitlabfs/gitlab"
"gopkg.in/yaml.v2"
)
func main() {
gitlabURL := flag.String("gitlab-url", "https://gitlab.com", "the gitlab url")
gitlabToken := flag.String("gitlab-token", "", "the gitlab authentication token")
gitlabRootGroupID := flag.Int("gitlab-group-id", 9970, "the group id of the groups at the root of the filesystem")
// gitlabNamespace := flag.String()
flag.Parse()
if flag.NArg() != 1 {
fmt.Printf("usage: %s MOUNTPOINT\n", os.Args[0])
os.Exit(2)
const (
OnCloneInit = "init"
OnCloneNoCheckout = "no-checkout"
OnCloneCheckout = "checkout"
)
type (
Config struct {
FS FSConfig `yaml:"fs,omitempty"`
Gitlab GitlabConfig `yaml:"gitlab,omitempty"`
Git GitConfig `yaml:"git,omitempty"`
}
mountpoint := flag.Arg(0)
parsedGitlabURL, err := url.Parse(*gitlabURL)
FSConfig struct {
Mountpoint string `yaml:"mountpoint,omitempty"`
}
GitlabConfig struct {
URL string `yaml:"url,omitempty"`
Token string `yaml:"token,omitempty"`
GroupIDs []int `yaml:"group_ids,omitempty"`
UserIDs []int `yaml:"user_ids,omitempty"`
IncludeCurrentUser bool `yaml:"include_current_user,omitempty"`
}
GitConfig struct {
CloneLocation string `yaml:"clone_location,omitempty"`
Remote string `yaml:"remote,omitempty"`
PullMethod string `yaml:"pull_method,omitempty"`
OnClone string `yaml:"on_clone,omitempty"`
AutoPull bool `yaml:"auto_pull,omitempty"`
Depth int `yaml:"depth,omitempty"`
}
)
func loadConfig(configPath string) (*Config, error) {
// defaults
dataHome := os.Getenv("XDG_DATA_HOME")
if dataHome == "" {
dataHome = filepath.Join(os.Getenv("HOME"), ".local/share")
}
defaultCloneLocation := filepath.Join(dataHome, "gitlabfs")
config := &Config{
FS: FSConfig{
Mountpoint: "",
},
Gitlab: GitlabConfig{
URL: "https://gitlab.com",
Token: "",
GroupIDs: []int{9970},
UserIDs: []int{},
IncludeCurrentUser: true,
},
Git: GitConfig{
CloneLocation: defaultCloneLocation,
Remote: "origin",
PullMethod: "http",
OnClone: "init",
AutoPull: false,
Depth: 0,
},
}
if configPath != "" {
f, err := os.Open(configPath)
if err != nil {
return nil, fmt.Errorf("failed to open config file: %v", err)
}
defer f.Close()
d := yaml.NewDecoder(f)
if err := d.Decode(config); err != nil {
return nil, fmt.Errorf("failed to parse config file: %v", err)
}
}
return config, nil
}
func makeGitlabConfig(config *Config) (*gitlab.GitlabClientParam, error) {
// parse pull_method
if config.Git.PullMethod != gitlab.PullMethodHTTP && config.Git.PullMethod != gitlab.PullMethodSSH {
return nil, fmt.Errorf("pull_method must be either \"%v\" or \"%v\"", gitlab.PullMethodHTTP, gitlab.PullMethodSSH)
}
return &gitlab.GitlabClientParam{
PullMethod: config.Git.PullMethod,
}, nil
}
func makeGitConfig(config *Config) (*git.GitClientParam, error) {
// Parse the gilab url
parsedGitlabURL, err := url.Parse(config.Gitlab.URL)
if err != nil {
fmt.Printf("%v is not a valid url: %v\n", *gitlabURL, err)
return nil, err
}
// parse on_clone
fetch := false
checkout := false
if config.Git.OnClone == OnCloneInit {
fetch = false
checkout = false
} else if config.Git.OnClone == OnCloneNoCheckout {
fetch = true
checkout = false
} else if config.Git.OnClone == OnCloneCheckout {
fetch = true
checkout = true
} else {
return nil, fmt.Errorf("on_clone must be either \"%v\", \"%v\" or \"%V\"", OnCloneInit, OnCloneNoCheckout, OnCloneCheckout)
}
return &git.GitClientParam{
CloneLocation: config.Git.CloneLocation,
RemoteName: config.Git.Remote,
RemoteURL: parsedGitlabURL,
Fetch: fetch,
Checkout: checkout,
AutoPull: config.Git.AutoPull,
PullDepth: config.Git.Depth,
}, nil
}
func main() {
configPath := flag.String("config", "", "the config file")
debug := flag.Bool("debug", false, "enable debug logging")
flag.Parse()
config, err := loadConfig(*configPath)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
// Create the gitlab client
gitlabClientParam := gitlab.GitlabClientParam{}
gitlabClient, _ := gitlab.NewClient(*gitlabURL, *gitlabToken, gitlabClientParam)
// Configure mountpoint
mountpoint := config.FS.Mountpoint
if flag.NArg() == 1 {
mountpoint = flag.Arg(0)
}
if mountpoint == "" {
fmt.Printf("usage: %s MOUNTPOINT\n", os.Args[0])
os.Exit(2)
}
// Create the git client
gitClientParam := git.GitClientParam{
RemoteURL: parsedGitlabURL,
AutoClone: true,
AutoPull: false,
Fetch: false,
Checkout: false,
SingleBranch: true,
PullDepth: 0,
gitClientParam, err := makeGitConfig(config)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
gitClient, _ := git.NewClient(gitClientParam)
gitClient, _ := git.NewClient(*gitClientParam)
// Create the gitlab client
gitlabClientParam, err := makeGitlabConfig(config)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
gitlabClient, _ := gitlab.NewClient(config.Gitlab.URL, config.Gitlab.Token, *gitlabClientParam)
// Start the filesystem
fs.Start(mountpoint, []int{*gitlabRootGroupID}, []int{}, &fs.FSParam{Gitlab: gitlabClient, Git: gitClient})
fs.Start(
mountpoint,
config.Gitlab.GroupIDs,
config.Gitlab.UserIDs,
&fs.FSParam{Git: gitClient, Gitlab: gitlabClient},
*debug,
)
}