From 9fac824dddd91329fd602fbdd267ddda5f93eaa9 Mon Sep 17 00:00:00 2001 From: Abdhesh Nayak Date: Fri, 2 Jun 2023 23:37:18 +0530 Subject: [PATCH 1/5] :construction: wip --- apps/nodectrl/internal/app/main.go | 7 +- apps/nodectrl/internal/domain/aws/main.go | 44 ++- .../internal/domain/common/interface.go | 26 +- apps/nodectrl/internal/domain/do/main.go | 3 - .../internal/domain/entities/agent-token.go | 34 ++ apps/nodectrl/internal/domain/main.go | 15 +- apps/nodectrl/internal/domain/utils/common.go | 307 ------------------ apps/nodectrl/internal/domain/utils/fs.go | 122 +++++++ apps/nodectrl/internal/domain/utils/main.go | 56 ++++ apps/nodectrl/internal/domain/utils/ssh.go | 78 +++++ apps/nodectrl/internal/domain/utils/tf.go | 120 +++++++ apps/nodectrl/internal/framework/main.go | 2 + apps/nodectrl/task.txt | 1 + go.mod | 6 +- 14 files changed, 492 insertions(+), 329 deletions(-) create mode 100644 apps/nodectrl/internal/domain/entities/agent-token.go delete mode 100644 apps/nodectrl/internal/domain/utils/common.go create mode 100644 apps/nodectrl/internal/domain/utils/fs.go create mode 100644 apps/nodectrl/internal/domain/utils/main.go create mode 100644 apps/nodectrl/internal/domain/utils/ssh.go create mode 100644 apps/nodectrl/internal/domain/utils/tf.go create mode 100644 apps/nodectrl/task.txt diff --git a/apps/nodectrl/internal/app/main.go b/apps/nodectrl/internal/app/main.go index aff934b6d..8f781dfc9 100644 --- a/apps/nodectrl/internal/app/main.go +++ b/apps/nodectrl/internal/app/main.go @@ -5,19 +5,22 @@ import ( "fmt" "go.uber.org/fx" + "kloudlite.io/apps/nodectrl/internal/domain" "kloudlite.io/apps/nodectrl/internal/domain/common" + "kloudlite.io/apps/nodectrl/internal/domain/entities" "kloudlite.io/apps/nodectrl/internal/domain/utils" "kloudlite.io/apps/nodectrl/internal/env" + "kloudlite.io/pkg/repos" ) var Module = fx.Module("app", + repos.NewFxMongoRepo[*entities.Token]("tokens", "tkn", entities.TokenIndexes), domain.Module, fx.Invoke( func(env *env.Env, pc common.ProviderClient, shutdowner fx.Shutdowner, lifecycle fx.Lifecycle) { lifecycle.Append(fx.Hook{ OnStart: func(context.Context) error { - go func() error { ctx := context.Background() if err := utils.SetupGetWorkDir(); err != nil { @@ -48,7 +51,6 @@ var Module = fx.Module("app", shutdowner.Shutdown() return nil }() - if err != nil { fmt.Println(utils.ColorText(fmt.Sprint("\n", "Error: ", err, "\n"), 1)) return err @@ -62,7 +64,6 @@ var Module = fx.Module("app", return nil }, }) - }, ), ) diff --git a/apps/nodectrl/internal/domain/aws/main.go b/apps/nodectrl/internal/domain/aws/main.go index 85e6dcac4..ac7d221ac 100644 --- a/apps/nodectrl/internal/domain/aws/main.go +++ b/apps/nodectrl/internal/domain/aws/main.go @@ -1,11 +1,11 @@ package aws import ( + "context" "fmt" "path" "time" - "golang.org/x/net/context" "kloudlite.io/apps/nodectrl/internal/domain/common" "kloudlite.io/apps/nodectrl/internal/domain/utils" mongogridfs "kloudlite.io/pkg/mongo-gridfs" @@ -31,15 +31,48 @@ type awsClient struct { accessKey string accessSecret string + accountId string SSHPath string - accountId string - providerDir string tfTemplates string labels map[string]string taints []string } +// CreateAndAttachNode implements common.ProviderClient +func (a awsClient) CreateAndAttachNode(ctx context.Context) error { + privateKeyBytes, publicKeyBytes, err := utils.GenerateKeys() + if err != nil { + return err + } + + if err := a.NewNode(ctx, privateKeyBytes); err != nil { + return err + } + + if err := a.AttachNode(ctx, publicKeyBytes); err != nil { + return err + } + + return nil +} + +// AttachNode implements common.ProviderClient +func (a awsClient) AttachNode(ctx context.Context, publicKeyBytes []byte) error { + /* + check readyness, wait if not ready + if ready install agent + to install fetch + */ + + panic("unimplemented") +} + +// CreateCluster implements common.ProviderClient +func (awsClient) CreateCluster() error { + panic("unimplemented") +} + func parseValues(a awsClient) map[string]string { values := map[string]string{} @@ -67,8 +100,7 @@ func (a awsClient) SaveToDbGuranteed(ctx context.Context) { } // NewNode implements ProviderClient -func (a awsClient) NewNode(ctx context.Context) error { - +func (a awsClient) NewNode(ctx context.Context, privateKeyBytes []byte) error { values := parseValues(a) /* @@ -108,7 +140,6 @@ func (a awsClient) NewNode(ctx context.Context) error { // DeleteNode implements ProviderClient func (a awsClient) DeleteNode(ctx context.Context) error { - values := parseValues(a) /* @@ -153,7 +184,6 @@ func NewAwsProviderClient(node AWSNode, cpd common.CommonProviderData, apc AwsPr accessSecret: apc.AccessSecret, accountId: apc.AccountId, - providerDir: "aws", tfTemplates: cpd.TfTemplates, labels: cpd.Labels, taints: cpd.Taints, diff --git a/apps/nodectrl/internal/domain/common/interface.go b/apps/nodectrl/internal/domain/common/interface.go index a23c9ba46..2f8140937 100644 --- a/apps/nodectrl/internal/domain/common/interface.go +++ b/apps/nodectrl/internal/domain/common/interface.go @@ -2,9 +2,33 @@ package common import "context" +/* +Tasks needs to be performed by this job + - create node + - attach node + - delete node + - craete cluster + - delete cluster +*/ + type ProviderClient interface { + CreateAndAttachNode(ctx context.Context) error + /* + ssh generation + create node + AttachNode + */ NewNode(ctx context.Context) error - DeleteNode(ctx context.Context) error + AttachNode(ctx context.Context) error + DeleteNode(ctx context.Context) error SaveToDbGuranteed(ctx context.Context) + + CreateCluster() error + /* + It will perform generation of ssh + create node + install master + fetch agent token and Master URL and save it to db + */ } diff --git a/apps/nodectrl/internal/domain/do/main.go b/apps/nodectrl/internal/domain/do/main.go index 96847bab5..b09106f02 100644 --- a/apps/nodectrl/internal/domain/do/main.go +++ b/apps/nodectrl/internal/domain/do/main.go @@ -31,7 +31,6 @@ type doClient struct { SSHPath string accountId string - providerDir string tfTemplates string labels map[string]string taints []string @@ -145,8 +144,6 @@ func NewDoProviderClient(node DoNode, cpd common.CommonProviderData, dpc DoProvi apiToken: dpc.ApiToken, accountId: dpc.AccountId, - providerDir: "do", - tfTemplates: cpd.TfTemplates, labels: cpd.Labels, taints: cpd.Taints, diff --git a/apps/nodectrl/internal/domain/entities/agent-token.go b/apps/nodectrl/internal/domain/entities/agent-token.go new file mode 100644 index 000000000..317320036 --- /dev/null +++ b/apps/nodectrl/internal/domain/entities/agent-token.go @@ -0,0 +1,34 @@ +package entities + +import ( + crdsv1 "github.com/kloudlite/operator/apis/crds/v1" + + "kloudlite.io/pkg/repos" +) + +type Token struct { + repos.BaseEntity `json:",inline"` + crdsv1.Secret `json:",inline"` + + Token string `json:"token"` + NodeId string `json:"nodeId"` + AccountName string `json:"accountName"` + ClusterName string `json:"clusterName"` +} + +var TokenIndexes = []repos.IndexField{ + { + Field: []repos.IndexKey{ + {Key: "id", Value: repos.IndexAsc}, + }, + Unique: true, + }, + { + Field: []repos.IndexKey{ + {Key: "nodeId", Value: repos.IndexAsc}, + {Key: "accountName", Value: repos.IndexAsc}, + {Key: "clusterName", Value: repos.IndexAsc}, + }, + Unique: true, + }, +} diff --git a/apps/nodectrl/internal/domain/main.go b/apps/nodectrl/internal/domain/main.go index afb8613fb..6e0bca9a0 100644 --- a/apps/nodectrl/internal/domain/main.go +++ b/apps/nodectrl/internal/domain/main.go @@ -2,21 +2,26 @@ package domain import ( "go.uber.org/fx" + + "kloudlite.io/apps/nodectrl/internal/domain/entities" "kloudlite.io/apps/nodectrl/internal/env" mongogridfs "kloudlite.io/pkg/mongo-gridfs" + "kloudlite.io/pkg/repos" ) type domain struct { - env *env.Env - gfs mongogridfs.GridFs + env *env.Env + gfs mongogridfs.GridFs + tokenRepo repos.DbRepo[*entities.Token] } var Module = fx.Module("domain", fx.Provide( - func(env *env.Env, gfs mongogridfs.GridFs) Domain { + func(env *env.Env, gfs mongogridfs.GridFs, tokenRepo repos.DbRepo[*entities.Token]) Domain { return domain{ - env: env, - gfs: gfs, + env: env, + gfs: gfs, + tokenRepo: tokenRepo, } }, ), diff --git a/apps/nodectrl/internal/domain/utils/common.go b/apps/nodectrl/internal/domain/utils/common.go deleted file mode 100644 index 3436df7f7..000000000 --- a/apps/nodectrl/internal/domain/utils/common.go +++ /dev/null @@ -1,307 +0,0 @@ -package utils - -import ( - "context" - "encoding/base64" - "encoding/csv" - "encoding/json" - "fmt" - "os" - "os/exec" - "path" - "strings" - - "github.com/containerd/continuity/fs" - "gopkg.in/yaml.v2" - mongogridfs "kloudlite.io/pkg/mongo-gridfs" -) - -const ( - Workdir string = "/tmp/tf-workdir" -) - -func Base64YamlDecode(in string, out interface{}) error { - rawDecodedText, err := base64.StdEncoding.DecodeString(in) - if err != nil { - return err - } - - fmt.Println(string(rawDecodedText)) - - return yaml.Unmarshal(rawDecodedText, out) -} - -func SaveToDb(ctx context.Context, nodeId string, gfs mongogridfs.GridFs) error { - /* - Steps: - - compress the workdir into zip - - check if file present. if yes, upsert file else upload file - */ - - dir := path.Join(Workdir, nodeId) - filename := fmt.Sprintf("%s.zip", nodeId) - - // compress the workdir and upsert to db - if err := func() error { - if _, err := os.Stat(dir); err != nil { - return err - } - - source := fmt.Sprintf("%s.zip", dir) - - // compress - if err := ZipSource(dir, source); err != nil { - return err - } - - if err := gfs.Upsert(ctx, filename, source); err != nil { - return err - } - - return nil - }(); err != nil { - fmt.Println(ColorText(fmt.Sprint("Error: ", err), 1)) - return err - } - - return nil -} - -const ( - enableClear bool = false -) - -func CreateNodeWorkDir(nodeId string) error { - dir := path.Join(Workdir, nodeId) - if _, err := os.Stat(dir); err != nil { - return os.Mkdir(dir, os.ModePerm) - } - - if enableClear { - if err := os.RemoveAll(dir); err != nil { - return err - } - - return os.Mkdir(dir, os.ModePerm) - } else { - return nil - } -} - -func SetupGetWorkDir() error { - if _, err := os.Stat(Workdir); err != nil { - return os.Mkdir(Workdir, os.ModePerm) - } - return nil -} - -func MakeTfWorkFileReady(ctx context.Context, nodeId, tfPath string, gfs mongogridfs.GridFs, createIfNotExists bool) error { - - filename := fmt.Sprintf("%s.zip", nodeId) - // check if file exists in db - gf, err := gfs.FetchFileRef(ctx, filename) - if err != nil { - return err - } - - // not found create new dir - if gf == nil { - if !createIfNotExists { - return fmt.Errorf("no state file found with the nodeId %s to operate", nodeId) - } - - if err := CreateNodeWorkDir(nodeId); err != nil { - return err - } - - // a.tfTemplates - if err := fs.CopyDir(path.Join(Workdir, nodeId), tfPath); err != nil { - return err - } - - return nil - } - - // found file in db, download and extract to the workdir - fmt.Println(gf.Name, "found, extract it by downloading") - - source := path.Join(Workdir, filename) - // Download from db - if err := gfs.Download(ctx, filename, source); err != nil { - return err - } - - if s, err := Unzip(source, path.Join(Workdir)); err != nil { - return err - } else { - for _, v := range s { - fmt.Print(v, " \n") - } - } - - return nil -} - -func ColorText(text interface{}, code int) string { - return fmt.Sprintf("\033[38;05;%dm%v\033[0m", code, text) -} - -func DownloadDir() error { - return nil -} - -func UploadDir() error { - return nil -} - -func ExecCmd(cmdString string, logStr string) error { - r := csv.NewReader(strings.NewReader(cmdString)) - r.Comma = ' ' - cmdArr, err := r.Read() - if err != nil { - return err - } - - if logStr != "" { - fmt.Printf("[#] %s\n", logStr) - } else { - fmt.Printf("[#] %s\n", strings.Join(cmdArr, " ")) - } - - cmd := exec.Command(cmdArr[0], cmdArr[1:]...) - cmd.Stderr = os.Stderr - // cmd.Stdout = os.Stdout - - if err := cmd.Run(); err != nil { - fmt.Printf("err occurred: %s\n", err.Error()) - return err - } - return nil -} - -const ( - CLUSTER_ID = "kl" -) - -// rmTFdir implements doProviderClient -// func rmdir(folder string) error { -// return execCmd(fmt.Sprintf("rm -rf %q", folder), "") -// } - -// makeTFdir implements doProviderClient -func Mkdir(folder string) error { - return ExecCmd(fmt.Sprintf("mkdir -p %q", folder), "mkdir ") -} - -func getOutput(folder, key string) ([]byte, error) { - vars := []string{"output", "-json"} - fmt.Printf("[#] terraform %s\n", strings.Join(vars, " ")) - cmd := exec.Command("terraform", vars...) - cmd.Dir = folder - - // cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - out, err := cmd.Output() - if err != nil { - return nil, err - - } - - // fmt.Println(string(out)) - - var resp map[string]struct { - Value string `json:"value"` - } - - err = json.Unmarshal(out, &resp) - if err != nil { - return nil, err - } - - return []byte(resp[key].Value), nil -} - -func InitTFdir(dir string) error { - cmd := exec.Command("terraform", "init") - cmd.Dir = dir - - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - return cmd.Run() -} - -// applyTF implements doProviderClient -func ApplyTF(folder string, values map[string]string) error { - - vars := []string{"apply", "-auto-approve"} - - fmt.Printf("[#] terraform %s", strings.Join(vars, " ")) - - for k, v := range values { - vars = append(vars, fmt.Sprintf("-var=%s=%s", k, v)) - } - - cmd := exec.Command("terraform", vars...) - cmd.Dir = folder - - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - cmd.Dir = folder - - return cmd.Run() -} - -// destroyNode implements doProviderClient -func DestroyNode(nodeId string, values map[string]string) error { - dest := path.Join(Workdir, nodeId) - vars := []string{"destroy", "-auto-approve"} - - for k, v := range values { - vars = append(vars, fmt.Sprintf("-var=%s=%s", k, v)) - } - - cmd := exec.Command("terraform", vars...) - cmd.Dir = dest - - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - err := cmd.Run() - if err != nil { - fmt.Println(err) - return err - } - return nil -} - -func GetOutput(folder, key string) ([]byte, error) { - vars := []string{"output", "-json"} - fmt.Printf("[#] terraform %s\n", strings.Join(vars, " ")) - cmd := exec.Command("terraform", vars...) - cmd.Dir = folder - - // cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - out, err := cmd.Output() - if err != nil { - return nil, err - - } - - // fmt.Println(string(out)) - - var resp map[string]struct { - Value string `json:"value"` - } - - err = json.Unmarshal(out, &resp) - if err != nil { - return nil, err - } - - return []byte(resp[key].Value), nil -} diff --git a/apps/nodectrl/internal/domain/utils/fs.go b/apps/nodectrl/internal/domain/utils/fs.go new file mode 100644 index 000000000..d4571d9a7 --- /dev/null +++ b/apps/nodectrl/internal/domain/utils/fs.go @@ -0,0 +1,122 @@ +package utils + +import ( + "context" + "fmt" + "os" + "path" + + "github.com/containerd/continuity/fs" + + mongogridfs "kloudlite.io/pkg/mongo-gridfs" +) + +func CreateNodeWorkDir(nodeId string) error { + dir := path.Join(Workdir, nodeId) + if _, err := os.Stat(dir); err != nil { + return os.Mkdir(dir, os.ModePerm) + } + + if enableClear { + if err := os.RemoveAll(dir); err != nil { + return err + } + + return os.Mkdir(dir, os.ModePerm) + } else { + return nil + } +} + +func SetupGetWorkDir() error { + if _, err := os.Stat(Workdir); err != nil { + return os.Mkdir(Workdir, os.ModePerm) + } + return nil +} + +func MakeTfWorkFileReady(ctx context.Context, nodeId, tfPath string, gfs mongogridfs.GridFs, createIfNotExists bool) error { + filename := fmt.Sprintf("%s.zip", nodeId) + // check if file exists in db + gf, err := gfs.FetchFileRef(ctx, filename) + if err != nil { + return err + } + + // not found create new dir + if gf == nil { + if !createIfNotExists { + return fmt.Errorf("no state file found with the nodeId %s to operate", nodeId) + } + + if err := CreateNodeWorkDir(nodeId); err != nil { + return err + } + + // a.tfTemplates + if err := fs.CopyDir(path.Join(Workdir, nodeId), tfPath); err != nil { + return err + } + + return nil + } + + // found file in db, download and extract to the workdir + fmt.Println(gf.Name, "found, extract it by downloading") + + source := path.Join(Workdir, filename) + // Download from db + if err := gfs.Download(ctx, filename, source); err != nil { + return err + } + + if s, err := Unzip(source, path.Join(Workdir)); err != nil { + return err + } else { + for _, v := range s { + fmt.Print(v, " \n") + } + } + + return nil +} + +func SaveToDb(ctx context.Context, nodeId string, gfs mongogridfs.GridFs) error { + /* + Steps: + - compress the workdir into zip + - check if file present. if yes, upsert file else upload file + */ + + dir := path.Join(Workdir, nodeId) + filename := fmt.Sprintf("%s.zip", nodeId) + + // compress the workdir and upsert to db + if err := func() error { + if _, err := os.Stat(dir); err != nil { + return err + } + + source := fmt.Sprintf("%s.zip", dir) + + // compress + if err := ZipSource(dir, source); err != nil { + return err + } + + if err := gfs.Upsert(ctx, filename, source); err != nil { + return err + } + + return nil + }(); err != nil { + fmt.Println(ColorText(fmt.Sprint("Error: ", err), 1)) + return err + } + + return nil +} + +const ( + enableClear bool = false +) diff --git a/apps/nodectrl/internal/domain/utils/main.go b/apps/nodectrl/internal/domain/utils/main.go new file mode 100644 index 000000000..5e2842757 --- /dev/null +++ b/apps/nodectrl/internal/domain/utils/main.go @@ -0,0 +1,56 @@ +package utils + +import ( + "encoding/base64" + "encoding/csv" + "fmt" + "os" + "os/exec" + "strings" + + "gopkg.in/yaml.v2" +) + +const ( + Workdir string = "/tmp/tf-workdir" +) + +func Base64YamlDecode(in string, out interface{}) error { + rawDecodedText, err := base64.StdEncoding.DecodeString(in) + if err != nil { + return err + } + + fmt.Println(string(rawDecodedText)) + + return yaml.Unmarshal(rawDecodedText, out) +} + +func ColorText(text interface{}, code int) string { + return fmt.Sprintf("\033[38;05;%dm%v\033[0m", code, text) +} + +func ExecCmd(cmdString string, logStr string) error { + r := csv.NewReader(strings.NewReader(cmdString)) + r.Comma = ' ' + cmdArr, err := r.Read() + if err != nil { + return err + } + + if logStr != "" { + fmt.Printf("[#] %s\n", logStr) + } else { + fmt.Printf("[#] %s\n", strings.Join(cmdArr, " ")) + } + + cmd := exec.Command(cmdArr[0], cmdArr[1:]...) + cmd.Stderr = os.Stderr + // cmd.Stdout = os.Stdout + + if err := cmd.Run(); err != nil { + fmt.Printf("err occurred: %s\n", err.Error()) + return err + } + return nil +} diff --git a/apps/nodectrl/internal/domain/utils/ssh.go b/apps/nodectrl/internal/domain/utils/ssh.go new file mode 100644 index 000000000..a0ee15311 --- /dev/null +++ b/apps/nodectrl/internal/domain/utils/ssh.go @@ -0,0 +1,78 @@ +package utils + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + + "golang.org/x/crypto/ssh" +) + +func GenerateKeys() (privateKeyBytes []byte, publicKeyBytes []byte, err error) { + bitSize := 4096 + + privateKey, err := generatePrivateKey(bitSize) + if err != nil { + return nil, nil, err + } + + publicKeyBytes, err = generatePublicKey(&privateKey.PublicKey) + if err != nil { + return nil, nil, err + } + + privateKeyBytes = encodePrivateKeyToPEM(privateKey) + + return privateKeyBytes, publicKeyBytes, nil +} + +// generatePrivateKey creates a RSA Private Key of specified byte size +func generatePrivateKey(bitSize int) (*rsa.PrivateKey, error) { + // Private Key generation + privateKey, err := rsa.GenerateKey(rand.Reader, bitSize) + if err != nil { + return nil, err + } + + // Validate Private Key + err = privateKey.Validate() + if err != nil { + return nil, err + } + + // log.Println("Private Key generated") + return privateKey, nil +} + +// encodePrivateKeyToPEM encodes Private Key from RSA to PEM format +func encodePrivateKeyToPEM(privateKey *rsa.PrivateKey) []byte { + // Get ASN.1 DER format + privDER := x509.MarshalPKCS1PrivateKey(privateKey) + + // pem.Block + privBlock := pem.Block{ + Type: "RSA PRIVATE KEY", + Headers: nil, + Bytes: privDER, + } + + // Private key in PEM format + privatePEM := pem.EncodeToMemory(&privBlock) + + return privatePEM +} + +// generatePublicKey take a rsa.PublicKey and return bytes suitable for writing to .pub file +// returns in the format "ssh-rsa ..." +func generatePublicKey(privatekey *rsa.PublicKey) ([]byte, error) { + publicRsaKey, err := ssh.NewPublicKey(privatekey) + if err != nil { + return nil, err + } + + pubKeyBytes := ssh.MarshalAuthorizedKey(publicRsaKey) + + // log.Println("Public key generated") + return pubKeyBytes, nil +} diff --git a/apps/nodectrl/internal/domain/utils/tf.go b/apps/nodectrl/internal/domain/utils/tf.go new file mode 100644 index 000000000..08cda37a9 --- /dev/null +++ b/apps/nodectrl/internal/domain/utils/tf.go @@ -0,0 +1,120 @@ +package utils + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path" + "strings" +) + +func GetOutput(folder, key string) ([]byte, error) { + vars := []string{"output", "-json"} + fmt.Printf("[#] terraform %s\n", strings.Join(vars, " ")) + cmd := exec.Command("terraform", vars...) + cmd.Dir = folder + + // cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + out, err := cmd.Output() + if err != nil { + return nil, err + } + + // fmt.Println(string(out)) + + var resp map[string]struct { + Value string `json:"value"` + } + + err = json.Unmarshal(out, &resp) + if err != nil { + return nil, err + } + + return []byte(resp[key].Value), nil +} + +// destroyNode implements doProviderClient +func DestroyNode(nodeId string, values map[string]string) error { + dest := path.Join(Workdir, nodeId) + vars := []string{"destroy", "-auto-approve"} + + for k, v := range values { + vars = append(vars, fmt.Sprintf("-var=%s=%s", k, v)) + } + + cmd := exec.Command("terraform", vars...) + cmd.Dir = dest + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err := cmd.Run() + if err != nil { + fmt.Println(err) + return err + } + return nil +} + +// applyTF implements doProviderClient +func ApplyTF(folder string, values map[string]string) error { + vars := []string{"apply", "-auto-approve"} + + fmt.Printf("[#] terraform %s", strings.Join(vars, " ")) + + for k, v := range values { + vars = append(vars, fmt.Sprintf("-var=%s=%s", k, v)) + } + + cmd := exec.Command("terraform", vars...) + cmd.Dir = folder + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + cmd.Dir = folder + + return cmd.Run() +} + +func getOutput(folder, key string) ([]byte, error) { + vars := []string{"output", "-json"} + fmt.Printf("[#] terraform %s\n", strings.Join(vars, " ")) + cmd := exec.Command("terraform", vars...) + cmd.Dir = folder + + // cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + out, err := cmd.Output() + if err != nil { + return nil, err + } + + // fmt.Println(string(out)) + + var resp map[string]struct { + Value string `json:"value"` + } + + err = json.Unmarshal(out, &resp) + if err != nil { + return nil, err + } + + return []byte(resp[key].Value), nil +} + +func InitTFdir(dir string) error { + cmd := exec.Command("terraform", "init") + cmd.Dir = dir + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + return cmd.Run() +} diff --git a/apps/nodectrl/internal/framework/main.go b/apps/nodectrl/internal/framework/main.go index 853d00a9f..1f1218b02 100644 --- a/apps/nodectrl/internal/framework/main.go +++ b/apps/nodectrl/internal/framework/main.go @@ -5,6 +5,7 @@ import ( "kloudlite.io/apps/nodectrl/internal/app" "kloudlite.io/apps/nodectrl/internal/env" mongogridfs "kloudlite.io/pkg/mongo-gridfs" + mongoDb "kloudlite.io/pkg/repos" ) type fm struct { @@ -21,5 +22,6 @@ var Module = fx.Module( return &fm{env} }), mongogridfs.NewMongoGridFsClientFx[*fm](), + mongoDb.NewMongoClientFx[*fm](), app.Module, ) diff --git a/apps/nodectrl/task.txt b/apps/nodectrl/task.txt new file mode 100644 index 000000000..dcc8876f6 --- /dev/null +++ b/apps/nodectrl/task.txt @@ -0,0 +1 @@ +1. diff --git a/go.mod b/go.mod index 9707a0f2c..88428ffcc 100644 --- a/go.mod +++ b/go.mod @@ -36,14 +36,12 @@ require ( go.mongodb.org/mongo-driver v1.9.1 go.uber.org/fx v1.17.1 go.uber.org/zap v1.24.0 - golang.org/x/net v0.8.0 golang.org/x/oauth2 v0.3.0 golang.org/x/sync v0.1.0 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20221104135756-97bc4ad4a1cb google.golang.org/grpc v1.51.0 google.golang.org/protobuf v1.28.1 gopkg.in/yaml.v2 v2.4.0 - gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.26.1 k8s.io/apiextensions-apiserver v0.26.0 k8s.io/apimachinery v0.26.1 @@ -57,6 +55,8 @@ require ( github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/google/gnostic v0.6.9 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + golang.org/x/net v0.8.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( @@ -155,7 +155,7 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/dig v1.14.0 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/crypto v0.5.0 // indirect + golang.org/x/crypto v0.5.0 golang.org/x/mod v0.8.0 // indirect golang.org/x/sys v0.6.0 // indirect golang.org/x/term v0.6.0 // indirect From 0c4fca06f4b112063a9c9958ef8619f94a736d12 Mon Sep 17 00:00:00 2001 From: Abdhesh Nayak Date: Mon, 5 Jun 2023 18:51:41 +0530 Subject: [PATCH 2/5] :construction: wip --- apps/nodectrl/Taskfile.yml | 1 + apps/nodectrl/internal/app/main.go | 5 +- apps/nodectrl/internal/domain/aws/main.go | 189 ++++++++++++++++-- .../nodectrl/internal/domain/common/common.go | 28 +++ .../internal/domain/common/interface.go | 2 +- apps/nodectrl/internal/domain/do/main.go | 19 +- .../internal/domain/entities/agent-token.go | 8 +- apps/nodectrl/internal/domain/main.go | 12 ++ apps/nodectrl/internal/domain/port.go | 11 + .../internal/domain/provider-client-fx.go | 39 +++- apps/nodectrl/internal/domain/utils/fs.go | 8 +- apps/nodectrl/internal/domain/utils/main.go | 21 ++ apps/nodectrl/internal/domain/utils/tf.go | 4 +- apps/nodectrl/internal/domain/utils/zipper.go | 9 +- apps/nodectrl/main.go | 3 +- 15 files changed, 315 insertions(+), 44 deletions(-) diff --git a/apps/nodectrl/Taskfile.yml b/apps/nodectrl/Taskfile.yml index c19312618..08b82a0d3 100644 --- a/apps/nodectrl/Taskfile.yml +++ b/apps/nodectrl/Taskfile.yml @@ -10,6 +10,7 @@ tasks: - ./main.go cmds: # - go run -tags dynamic main.go --dev + - cd ./.secrets && go run main.go - nodemon -e go --signal SIGKILL --exec 'go run main.go --dev || exit 1' docker-build: diff --git a/apps/nodectrl/internal/app/main.go b/apps/nodectrl/internal/app/main.go index 8f781dfc9..ccc3c8a6b 100644 --- a/apps/nodectrl/internal/app/main.go +++ b/apps/nodectrl/internal/app/main.go @@ -29,10 +29,9 @@ var Module = fx.Module("app", err := func() error { switch env.Action { - case "create": - + case "create-cluster": fmt.Println("needs to create node") - if err := pc.NewNode(ctx); err != nil { + if err := pc.CreateCluster(ctx); err != nil { return err } case "delete": diff --git a/apps/nodectrl/internal/domain/aws/main.go b/apps/nodectrl/internal/domain/aws/main.go index ac7d221ac..1a4bf5487 100644 --- a/apps/nodectrl/internal/domain/aws/main.go +++ b/apps/nodectrl/internal/domain/aws/main.go @@ -3,12 +3,19 @@ package aws import ( "context" "fmt" + "io/ioutil" + "os" "path" + "strings" "time" + "gopkg.in/yaml.v2" + "kloudlite.io/apps/nodectrl/internal/domain/common" + "kloudlite.io/apps/nodectrl/internal/domain/entities" "kloudlite.io/apps/nodectrl/internal/domain/utils" mongogridfs "kloudlite.io/pkg/mongo-gridfs" + "kloudlite.io/pkg/repos" ) type AwsProviderConfig struct { @@ -26,8 +33,9 @@ type AWSNode struct { } type awsClient struct { - gfs mongogridfs.GridFs - node AWSNode + gfs mongogridfs.GridFs + node AWSNode + tokenRepo repos.DbRepo[*entities.Token] accessKey string accessSecret string @@ -41,16 +49,11 @@ type awsClient struct { // CreateAndAttachNode implements common.ProviderClient func (a awsClient) CreateAndAttachNode(ctx context.Context) error { - privateKeyBytes, publicKeyBytes, err := utils.GenerateKeys() - if err != nil { - return err - } - - if err := a.NewNode(ctx, privateKeyBytes); err != nil { + if err := a.NewNode(ctx); err != nil { return err } - if err := a.AttachNode(ctx, publicKeyBytes); err != nil { + if err := a.AttachNode(ctx); err != nil { return err } @@ -58,19 +61,161 @@ func (a awsClient) CreateAndAttachNode(ctx context.Context) error { } // AttachNode implements common.ProviderClient -func (a awsClient) AttachNode(ctx context.Context, publicKeyBytes []byte) error { +func (a awsClient) AttachNode(ctx context.Context) error { /* check readyness, wait if not ready if ready install agent to install fetch */ - panic("unimplemented") + token, err := a.tokenRepo.FindOne(ctx, repos.Filter{"nodeId": a.node.NodeId, "accountId": a.accountId}) + if err != nil { + return err + } + + var out []byte + + if out, err = utils.GetOutput(path.Join(utils.Workdir, a.node.NodeId), "node-ip"); err != nil { + return err + } + + labels := func() []string { + l := []string{} + for k, v := range a.labels { + l = append(l, fmt.Sprintf("--node-label %s=%s", k, v)) + } + l = append(l, fmt.Sprintf("--node-label %s=%s", "kloudlite.io/public-ip", string(out))) + return l + }() + + count := 0 + + for { + if e := utils.ExecCmd( + fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s ls", + fmt.Sprintf("%v/access", a.SSHPath), + string(out)), + "checking if node is ready "); e == nil { + break + } + + count++ + if count > 24 { + return fmt.Errorf("node is not ready even after 6 minutes") + } + time.Sleep(time.Second * 15) + } + + // attach node + if e := utils.ExecCmd( + fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s sudo sh /tmp/k3s-install.sh agent --server %s --token %s %s --node-name %s --node-external-ip %s --node-ip %s", + fmt.Sprintf("%v/access", a.SSHPath), string(out), token.EndpointUrl, token.JoinToken, + strings.Join(labels, " "), a.node.NodeId, string(out), string(out)), + "attaching to cluster"); e != nil { + return e + } + + return nil } // CreateCluster implements common.ProviderClient -func (awsClient) CreateCluster() error { - panic("unimplemented") +func (a awsClient) CreateCluster(ctx context.Context) error { + /* + create node + check for rediness + install k3s + check for rediness + install maaster + */ + + const sshDir = "/tmp/ssh" + + if _, err := os.Stat(sshDir); err != nil { + return os.Mkdir(sshDir, os.ModePerm) + } + + file, err := ioutil.TempDir("/tmp/ssh", "ssh_") + if err != nil { + return err + } + + if err := a.NewNode(ctx); err != nil { + return err + } + + var ip []byte + + if ip, err = utils.GetOutput(path.Join(utils.Workdir, a.node.NodeId), "node-ip"); err != nil { + return err + } + + count := 0 + + for { + if e := utils.ExecCmd( + fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s ls", + fmt.Sprintf("%v/access", file), + string(ip)), + "checking if node is ready "); e == nil { + break + } + + count++ + if count > 24 { + return fmt.Errorf("node is not ready even after 6 minutes") + } + time.Sleep(time.Second * 15) + } + + // install k3s + cmd := fmt.Sprintf( + "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s sudo sh /tmp/k3s-install.sh server --token=%q --node-external-ip %s --flannel-backend wireguard-native --flannel-external-ip --disable traefik --node-name=%q", + file, + string(ip), + a.node.NodeId, + string(ip), + fmt.Sprintf("kl-master-%s", a.node.NodeId), + ) + + if err := utils.ExecCmd(cmd, cmd); err != nil { + return err + } + // needed to fetch kubeconfig + + configOut, err := utils.ExecCmdWithOutput(fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s cat /etc/rancher/k3s/k3s.yaml", file, string(ip)), "") + if err != nil { + return err + } + + var kubeconfig common.KubeConfigType + if err := yaml.Unmarshal(configOut, &kubeconfig); err != nil { + return err + } + + for i := range kubeconfig.Clusters { + kubeconfig.Clusters[i].Cluster.Server = fmt.Sprintf("https://%s:6443", string(ip)) + } + + kc, err := yaml.Marshal(kubeconfig) + if err != nil { + return err + } + + tokenOut, err := utils.ExecCmdWithOutput(fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s cat /var/lib/rancher/k3s/server/node-token", file, string(ip)), "") + if err != nil { + return err + } + + _, err = a.tokenRepo.Create(ctx, &entities.Token{ + JoinToken: string(tokenOut), + EndpointUrl: fmt.Sprintf("https://%s:6443", ip), + KubeConfig: string(kc), + NodeId: a.node.NodeId, + AccountName: a.accountId, + ClusterName: "", + }) + + return err } func parseValues(a awsClient) map[string]string { @@ -85,6 +230,8 @@ func parseValues(a awsClient) map[string]string { values["keys-path"] = a.SSHPath values["ami"] = a.node.ImageId + fmt.Print(values) + return values } @@ -100,7 +247,14 @@ func (a awsClient) SaveToDbGuranteed(ctx context.Context) { } // NewNode implements ProviderClient -func (a awsClient) NewNode(ctx context.Context, privateKeyBytes []byte) error { +func (a awsClient) NewNode(ctx context.Context) error { + file, err := ioutil.TempDir("/tmp/ssh", "ssh_") + if err != nil { + return err + } + + a.SSHPath = file + values := parseValues(a) /* @@ -175,10 +329,11 @@ func (a awsClient) DeleteNode(ctx context.Context) error { return nil } -func NewAwsProviderClient(node AWSNode, cpd common.CommonProviderData, apc AwsProviderConfig, gfs mongogridfs.GridFs) common.ProviderClient { +func NewAwsProviderClient(node AWSNode, cpd common.CommonProviderData, apc AwsProviderConfig, gfs mongogridfs.GridFs, tokenRepo repos.DbRepo[*entities.Token]) common.ProviderClient { return awsClient{ - node: node, - gfs: gfs, + node: node, + gfs: gfs, + tokenRepo: tokenRepo, accessKey: apc.AccessKey, accessSecret: apc.AccessSecret, diff --git a/apps/nodectrl/internal/domain/common/common.go b/apps/nodectrl/internal/domain/common/common.go index ec8c16d93..a1834895d 100644 --- a/apps/nodectrl/internal/domain/common/common.go +++ b/apps/nodectrl/internal/domain/common/common.go @@ -6,3 +6,31 @@ type CommonProviderData struct { Taints []string `yaml:"taints"` SSHPath string `yaml:"sshPath"` } + +type KubeConfigType struct { + APIVersion string `yaml:"apiVersion"` + Clusters []struct { + Cluster struct { + CertificateAuthorityData string `yaml:"certificate-authority-data"` + Server string `yaml:"server"` + } `yaml:"cluster"` + Name string `yaml:"name"` + } `yaml:"clusters"` + Contexts []struct { + Context struct { + Cluster string `yaml:"cluster"` + User string `yaml:"user"` + } `yaml:"context"` + Name string `yaml:"name"` + } `yaml:"contexts"` + CurrentContext string `yaml:"current-context"` + Kind string `yaml:"kind"` + Preferences struct{} `yaml:"preferences"` + Users []struct { + Name string `yaml:"name"` + User struct { + ClientCertificateData string `yaml:"client-certificate-data"` + ClientKeyData string `yaml:"client-key-data"` + } `yaml:"user"` + } `yaml:"users"` +} diff --git a/apps/nodectrl/internal/domain/common/interface.go b/apps/nodectrl/internal/domain/common/interface.go index 2f8140937..0d6908fe2 100644 --- a/apps/nodectrl/internal/domain/common/interface.go +++ b/apps/nodectrl/internal/domain/common/interface.go @@ -24,7 +24,7 @@ type ProviderClient interface { DeleteNode(ctx context.Context) error SaveToDbGuranteed(ctx context.Context) - CreateCluster() error + CreateCluster(ctx context.Context) error /* It will perform generation of ssh create node diff --git a/apps/nodectrl/internal/domain/do/main.go b/apps/nodectrl/internal/domain/do/main.go index b09106f02..f70586348 100644 --- a/apps/nodectrl/internal/domain/do/main.go +++ b/apps/nodectrl/internal/domain/do/main.go @@ -7,8 +7,10 @@ import ( "time" "kloudlite.io/apps/nodectrl/internal/domain/common" + "kloudlite.io/apps/nodectrl/internal/domain/entities" "kloudlite.io/apps/nodectrl/internal/domain/utils" mongogridfs "kloudlite.io/pkg/mongo-gridfs" + "kloudlite.io/pkg/repos" ) type DoProviderConfig struct { @@ -36,6 +38,21 @@ type doClient struct { taints []string } +// AttachNode implements common.ProviderClient. +func (doClient) AttachNode(ctx context.Context) error { + panic("unimplemented") +} + +// CreateAndAttachNode implements common.ProviderClient. +func (doClient) CreateAndAttachNode(ctx context.Context) error { + panic("unimplemented") +} + +// CreateCluster implements common.ProviderClient. +func (doClient) CreateCluster(ctx context.Context) error { + panic("unimplemented") +} + func parseValues(d doClient) map[string]string { values := map[string]string{} @@ -136,7 +153,7 @@ func (d doClient) DeleteNode(ctx context.Context) error { return nil } -func NewDoProviderClient(node DoNode, cpd common.CommonProviderData, dpc DoProviderConfig, gfs mongogridfs.GridFs) common.ProviderClient { +func NewDoProviderClient(node DoNode, cpd common.CommonProviderData, dpc DoProviderConfig, gfs mongogridfs.GridFs, tokenRepo repos.DbRepo[*entities.Token]) common.ProviderClient { return doClient{ node: node, gfs: gfs, diff --git a/apps/nodectrl/internal/domain/entities/agent-token.go b/apps/nodectrl/internal/domain/entities/agent-token.go index 317320036..507472b3c 100644 --- a/apps/nodectrl/internal/domain/entities/agent-token.go +++ b/apps/nodectrl/internal/domain/entities/agent-token.go @@ -1,16 +1,16 @@ package entities import ( - crdsv1 "github.com/kloudlite/operator/apis/crds/v1" - "kloudlite.io/pkg/repos" ) type Token struct { repos.BaseEntity `json:",inline"` - crdsv1.Secret `json:",inline"` - Token string `json:"token"` + JoinToken string `json:"join%oken"` + EndpointUrl string `json:"endpointUrl" yaml:"endPointUrl"` + KubeConfig string `json:"kubeConfig" yaml:"kubeConfig"` + NodeId string `json:"nodeId"` AccountName string `json:"accountName"` ClusterName string `json:"clusterName"` diff --git a/apps/nodectrl/internal/domain/main.go b/apps/nodectrl/internal/domain/main.go index 6e0bca9a0..d9c9f7f3d 100644 --- a/apps/nodectrl/internal/domain/main.go +++ b/apps/nodectrl/internal/domain/main.go @@ -15,6 +15,18 @@ type domain struct { tokenRepo repos.DbRepo[*entities.Token] } +func (d domain) GetEnv() *env.Env { + return d.env +} + +func (d domain) GetGRidFs() mongogridfs.GridFs { + return d.gfs +} + +func (d domain) GetTokenRepo() repos.DbRepo[*entities.Token] { + return d.tokenRepo +} + var Module = fx.Module("domain", fx.Provide( func(env *env.Env, gfs mongogridfs.GridFs, tokenRepo repos.DbRepo[*entities.Token]) Domain { diff --git a/apps/nodectrl/internal/domain/port.go b/apps/nodectrl/internal/domain/port.go index fa83c82de..32a1a7704 100644 --- a/apps/nodectrl/internal/domain/port.go +++ b/apps/nodectrl/internal/domain/port.go @@ -1,4 +1,15 @@ package domain +import ( + "kloudlite.io/apps/nodectrl/internal/domain/entities" + "kloudlite.io/apps/nodectrl/internal/env" + mongogridfs "kloudlite.io/pkg/mongo-gridfs" + "kloudlite.io/pkg/repos" +) + type Domain interface { + GetEnv() *env.Env + + GetGRidFs() mongogridfs.GridFs + GetTokenRepo() repos.DbRepo[*entities.Token] } diff --git a/apps/nodectrl/internal/domain/provider-client-fx.go b/apps/nodectrl/internal/domain/provider-client-fx.go index 790ee8cac..3782a98c3 100644 --- a/apps/nodectrl/internal/domain/provider-client-fx.go +++ b/apps/nodectrl/internal/domain/provider-client-fx.go @@ -1,17 +1,46 @@ package domain import ( + "fmt" + "io/ioutil" + "os" + "go.uber.org/fx" + "kloudlite.io/apps/nodectrl/internal/domain/aws" "kloudlite.io/apps/nodectrl/internal/domain/common" "kloudlite.io/apps/nodectrl/internal/domain/do" "kloudlite.io/apps/nodectrl/internal/domain/utils" "kloudlite.io/apps/nodectrl/internal/env" - mongogridfs "kloudlite.io/pkg/mongo-gridfs" ) var ProviderClientFx = fx.Module("provider-client-fx", - fx.Provide(func(env *env.Env, gfs mongogridfs.GridFs) (common.ProviderClient, error) { + fx.Provide(func(env *env.Env, d Domain) (common.ProviderClient, error) { + privateKeyBytes, publicKeyBytes, err := utils.GenerateKeys() + if err != nil { + return nil, err + } + + const sshDir = "/tmp/ssh" + + if _, err := os.Stat(sshDir); err != nil { + if e := os.Mkdir(sshDir, os.ModePerm); e != nil { + return nil, e + } + } + + file, err := ioutil.TempDir("/tmp/ssh", "ssh_") + if err != nil { + return nil, err + } + + if err := os.WriteFile(fmt.Sprintf("%s/access.pub", file), publicKeyBytes, os.ModePerm); err != nil { + return nil, err + } + + if err := os.WriteFile(fmt.Sprintf("%s/access", file), privateKeyBytes, os.ModePerm); err != nil { + return nil, err + } cpd := common.CommonProviderData{} @@ -30,11 +59,13 @@ var ProviderClientFx = fx.Module("provider-client-fx", apc := aws.AwsProviderConfig{} + fmt.Println("here......................", env.AWSProviderConfig) + if err := utils.Base64YamlDecode(env.AWSProviderConfig, &apc); err != nil { return nil, err } - return aws.NewAwsProviderClient(node, cpd, apc, gfs), nil + return aws.NewAwsProviderClient(node, cpd, apc, d.GetGRidFs(), d.GetTokenRepo()), nil case "azure": panic("not implemented") case "do": @@ -51,7 +82,7 @@ var ProviderClientFx = fx.Module("provider-client-fx", return nil, err } - return do.NewDoProviderClient(node, cpd, dpc, gfs), nil + return do.NewDoProviderClient(node, cpd, dpc, d.GetGRidFs(), d.GetTokenRepo()), nil case "gcp": panic("not implemented") } diff --git a/apps/nodectrl/internal/domain/utils/fs.go b/apps/nodectrl/internal/domain/utils/fs.go index d4571d9a7..2bb982e96 100644 --- a/apps/nodectrl/internal/domain/utils/fs.go +++ b/apps/nodectrl/internal/domain/utils/fs.go @@ -70,12 +70,12 @@ func MakeTfWorkFileReady(ctx context.Context, nodeId, tfPath string, gfs mongogr return err } - if s, err := Unzip(source, path.Join(Workdir)); err != nil { + if _, err := Unzip(source, path.Join(Workdir)); err != nil { return err } else { - for _, v := range s { - fmt.Print(v, " \n") - } + // for _, v := range s { + // fmt.Print(v, " \n") + // } } return nil diff --git a/apps/nodectrl/internal/domain/utils/main.go b/apps/nodectrl/internal/domain/utils/main.go index 5e2842757..65af4e8ca 100644 --- a/apps/nodectrl/internal/domain/utils/main.go +++ b/apps/nodectrl/internal/domain/utils/main.go @@ -54,3 +54,24 @@ func ExecCmd(cmdString string, logStr string) error { } return nil } + +func ExecCmdWithOutput(cmdString string, logStr string) ([]byte, error) { + r := csv.NewReader(strings.NewReader(cmdString)) + r.Comma = ' ' + cmdArr, err := r.Read() + if err != nil { + return nil, err + } + + if logStr != "" { + fmt.Printf("[#] %s\n", logStr) + } else { + fmt.Printf("[#] %s\n", strings.Join(cmdArr, " ")) + } + + cmd := exec.Command(cmdArr[0], cmdArr[1:]...) + cmd.Stderr = os.Stderr + // cmd.Stdout = os.Stdout + + return cmd.Output() +} diff --git a/apps/nodectrl/internal/domain/utils/tf.go b/apps/nodectrl/internal/domain/utils/tf.go index 08cda37a9..90a3fda0e 100644 --- a/apps/nodectrl/internal/domain/utils/tf.go +++ b/apps/nodectrl/internal/domain/utils/tf.go @@ -64,12 +64,12 @@ func DestroyNode(nodeId string, values map[string]string) error { func ApplyTF(folder string, values map[string]string) error { vars := []string{"apply", "-auto-approve"} - fmt.Printf("[#] terraform %s", strings.Join(vars, " ")) - for k, v := range values { vars = append(vars, fmt.Sprintf("-var=%s=%s", k, v)) } + fmt.Printf("[#] terraform %s", strings.Join(vars, " ")) + cmd := exec.Command("terraform", vars...) cmd.Dir = folder diff --git a/apps/nodectrl/internal/domain/utils/zipper.go b/apps/nodectrl/internal/domain/utils/zipper.go index 2accfe1d0..8b447df75 100644 --- a/apps/nodectrl/internal/domain/utils/zipper.go +++ b/apps/nodectrl/internal/domain/utils/zipper.go @@ -71,10 +71,8 @@ func ZipSource(source, target string) error { } func Unzip(src string, destination string) ([]string, error) { - var filenames []string r, err := zip.OpenReader(src) - if err != nil { return filenames, err } @@ -104,7 +102,6 @@ func Unzip(src string, destination string) ([]string, error) { outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) - if err != nil { return filenames, err } @@ -146,14 +143,13 @@ func ExtractZip(src, destination string) error { } defer os.RemoveAll(tempDirName) - if names, err := Unzip(src, tempDirName); err != nil { + if _, err := Unzip(src, tempDirName); err != nil { return err } else { - fmt.Println(names) + // fmt.Println(names) if err := copy.Copy(path.Join(tempDirName, destination), destination); err != nil { return err } - } } @@ -162,7 +158,6 @@ func ExtractZip(src, destination string) error { } func mutateOperation() error { - file, err := ioutil.TempFile("out", "prefix_") if err != nil { return err diff --git a/apps/nodectrl/main.go b/apps/nodectrl/main.go index 02ddc0276..7b61bafd9 100644 --- a/apps/nodectrl/main.go +++ b/apps/nodectrl/main.go @@ -4,6 +4,7 @@ import ( "flag" "go.uber.org/fx" + "kloudlite.io/apps/nodectrl/internal/env" "kloudlite.io/apps/nodectrl/internal/framework" "kloudlite.io/pkg/logging" @@ -15,7 +16,7 @@ func main() { flag.Parse() fx.New( fx.Provide(env.LoadEnv), - // fx.NopLogger, + fx.NopLogger, fx.Provide( func() (logging.Logger, error) { return logging.New(&logging.Options{Name: "nodectrl", Dev: isDev}) From 9b7adc80153cabfed6f6de595db21c0d8ee61206 Mon Sep 17 00:00:00 2001 From: Abdhesh Nayak Date: Thu, 8 Jun 2023 10:23:05 +0530 Subject: [PATCH 3/5] :construction: wip --- apps/nodectrl/Taskfile.yml | 9 +- apps/nodectrl/internal/app/main.go | 3 - apps/nodectrl/internal/domain/aws/main.go | 238 +++++++++++------- apps/nodectrl/internal/domain/do/main.go | 144 ----------- .../internal/domain/entities/agent-token.go | 34 --- apps/nodectrl/internal/domain/main.go | 21 +- apps/nodectrl/internal/domain/port.go | 6 - .../internal/domain/provider-client-fx.go | 22 +- apps/nodectrl/internal/domain/utils/fs.go | 23 +- apps/nodectrl/internal/framework/main.go | 5 +- go.mod | 13 +- go.sum | 37 ++- pkg/aws-s3/main.go | 138 ++++++++++ 13 files changed, 348 insertions(+), 345 deletions(-) delete mode 100644 apps/nodectrl/internal/domain/entities/agent-token.go create mode 100644 pkg/aws-s3/main.go diff --git a/apps/nodectrl/Taskfile.yml b/apps/nodectrl/Taskfile.yml index 08b82a0d3..70a51b2f2 100644 --- a/apps/nodectrl/Taskfile.yml +++ b/apps/nodectrl/Taskfile.yml @@ -4,13 +4,18 @@ dotenv: - .secrets/env tasks: - run: + gen-sec: sources: - ./internal/**/*.go - ./main.go cmds: - # - go run -tags dynamic main.go --dev - cd ./.secrets && go run main.go + + run: + sources: + - ./internal/**/*.go + - ./main.go + cmds: - nodemon -e go --signal SIGKILL --exec 'go run main.go --dev || exit 1' docker-build: diff --git a/apps/nodectrl/internal/app/main.go b/apps/nodectrl/internal/app/main.go index ccc3c8a6b..424a65001 100644 --- a/apps/nodectrl/internal/app/main.go +++ b/apps/nodectrl/internal/app/main.go @@ -8,14 +8,11 @@ import ( "kloudlite.io/apps/nodectrl/internal/domain" "kloudlite.io/apps/nodectrl/internal/domain/common" - "kloudlite.io/apps/nodectrl/internal/domain/entities" "kloudlite.io/apps/nodectrl/internal/domain/utils" "kloudlite.io/apps/nodectrl/internal/env" - "kloudlite.io/pkg/repos" ) var Module = fx.Module("app", - repos.NewFxMongoRepo[*entities.Token]("tokens", "tkn", entities.TokenIndexes), domain.Module, fx.Invoke( func(env *env.Env, pc common.ProviderClient, shutdowner fx.Shutdowner, lifecycle fx.Lifecycle) { diff --git a/apps/nodectrl/internal/domain/aws/main.go b/apps/nodectrl/internal/domain/aws/main.go index 1a4bf5487..02f992878 100644 --- a/apps/nodectrl/internal/domain/aws/main.go +++ b/apps/nodectrl/internal/domain/aws/main.go @@ -3,19 +3,15 @@ package aws import ( "context" "fmt" - "io/ioutil" "os" "path" - "strings" "time" "gopkg.in/yaml.v2" "kloudlite.io/apps/nodectrl/internal/domain/common" - "kloudlite.io/apps/nodectrl/internal/domain/entities" "kloudlite.io/apps/nodectrl/internal/domain/utils" - mongogridfs "kloudlite.io/pkg/mongo-gridfs" - "kloudlite.io/pkg/repos" + awss3 "kloudlite.io/pkg/aws-s3" ) type AwsProviderConfig struct { @@ -33,9 +29,8 @@ type AWSNode struct { } type awsClient struct { - gfs mongogridfs.GridFs - node AWSNode - tokenRepo repos.DbRepo[*entities.Token] + node AWSNode + awsS3Client awss3.AwsS3 accessKey string accessSecret string @@ -68,32 +63,29 @@ func (a awsClient) AttachNode(ctx context.Context) error { to install fetch */ - token, err := a.tokenRepo.FindOne(ctx, repos.Filter{"nodeId": a.node.NodeId, "accountId": a.accountId}) - if err != nil { - return err - } - + // var out []byte - if out, err = utils.GetOutput(path.Join(utils.Workdir, a.node.NodeId), "node-ip"); err != nil { + out, err := utils.GetOutput(path.Join(utils.Workdir, a.node.NodeId), "node-ip") + if err != nil { return err } - labels := func() []string { - l := []string{} - for k, v := range a.labels { - l = append(l, fmt.Sprintf("--node-label %s=%s", k, v)) - } - l = append(l, fmt.Sprintf("--node-label %s=%s", "kloudlite.io/public-ip", string(out))) - return l - }() + // labels := func() []string { + // l := []string{} + // for k, v := range a.labels { + // l = append(l, fmt.Sprintf("--node-label %s=%s", k, v)) + // } + // l = append(l, fmt.Sprintf("--node-label %s=%s", "kloudlite.io/public-ip", string(out))) + // return l + // }() count := 0 for { if e := utils.ExecCmd( fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s ls", - fmt.Sprintf("%v/access", a.SSHPath), + fmt.Sprintf("%s/access", a.SSHPath), string(out)), "checking if node is ready "); e == nil { break @@ -106,13 +98,94 @@ func (a awsClient) AttachNode(ctx context.Context) error { time.Sleep(time.Second * 15) } - // attach node - if e := utils.ExecCmd( - fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s sudo sh /tmp/k3s-install.sh agent --server %s --token %s %s --node-name %s --node-external-ip %s --node-ip %s", - fmt.Sprintf("%v/access", a.SSHPath), string(out), token.EndpointUrl, token.JoinToken, - strings.Join(labels, " "), a.node.NodeId, string(out), string(out)), - "attaching to cluster"); e != nil { - return e + // // attach node + // if e := utils.ExecCmd( + // fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s sudo sh /tmp/k3s-install.sh agent --server %s --token %s %s --node-name %s --node-external-ip %s --node-ip %s", + // fmt.Sprintf("%v/access", a.SSHPath), string(out), token.EndpointUrl, token.JoinToken, + // strings.Join(labels, " "), a.node.NodeId, string(out), string(out)), + // "attaching to cluster"); e != nil { + // return e + // } + + return nil +} + +func (a awsClient) SetupSSH() error { + const sshDir = "/tmp/ssh" + + if _, err := os.Stat(sshDir); err != nil { + return os.Mkdir(sshDir, os.ModePerm) + } + + destDir := path.Join(sshDir, a.accountId) + fileName := fmt.Sprintf("%s.zip", a.accountId) + + if err := a.awsS3Client.IsFileExists(fileName); err != nil { + + if _, err := os.Stat(destDir); err == nil { + if err := os.RemoveAll(destDir); err != nil { + return err + } + } + + if e := os.Mkdir(destDir, os.ModePerm); e != nil { + return e + } + + privateKeyBytes, publicKeyBytes, err := utils.GenerateKeys() + if err != nil { + return err + } + + if err := os.WriteFile(fmt.Sprintf("%s/access.pub", destDir), publicKeyBytes, os.ModePerm); err != nil { + return err + } + + if err := os.WriteFile(fmt.Sprintf("%s/access", destDir), privateKeyBytes, 0400); err != nil { + return err + } + return nil + } + + err := a.awsS3Client.DownloadFile(path.Join(sshDir, fileName), fileName) + if err != nil { + return err + } + + _, err = utils.Unzip(path.Join(sshDir, fileName), destDir) + if err != nil { + return err + } + + return nil +} + +func (a awsClient) saveForSure() error { + count := 0 + for { + if err := a.saveSSH(); err == nil { + return nil + } + if count >= 10 { + return fmt.Errorf("coudn't save the state") + } + + time.Sleep(time.Second * 20) + count++ + } +} + +func (a awsClient) saveSSH() error { + const sshDir = "/tmp/ssh" + destDir := path.Join(sshDir, a.accountId) + fileName := fmt.Sprintf("%s.zip", a.accountId) + + if err := utils.ZipSource(destDir, path.Join(sshDir, fileName)); err != nil { + return err + } + + if err := a.awsS3Client.UploadFile(path.Join(sshDir, fileName), fileName); err != nil { + return err } return nil @@ -128,24 +201,18 @@ func (a awsClient) CreateCluster(ctx context.Context) error { install maaster */ - const sshDir = "/tmp/ssh" - - if _, err := os.Stat(sshDir); err != nil { - return os.Mkdir(sshDir, os.ModePerm) - } - - file, err := ioutil.TempDir("/tmp/ssh", "ssh_") - if err != nil { + if err := a.SetupSSH(); err != nil { return err } + defer a.saveForSure() + a.SSHPath = path.Join("/tmp/ssh", a.accountId) if err := a.NewNode(ctx); err != nil { return err } - var ip []byte - - if ip, err = utils.GetOutput(path.Join(utils.Workdir, a.node.NodeId), "node-ip"); err != nil { + ip, err := utils.GetOutput(path.Join(utils.Workdir, a.node.NodeId), "node-ip") + if err != nil { return err } @@ -154,9 +221,10 @@ func (a awsClient) CreateCluster(ctx context.Context) error { for { if e := utils.ExecCmd( fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s ls", - fmt.Sprintf("%v/access", file), - string(ip)), - "checking if node is ready "); e == nil { + fmt.Sprintf("%v/access", a.SSHPath), + string(ip), + ), + ""); e == nil { break } @@ -170,7 +238,7 @@ func (a awsClient) CreateCluster(ctx context.Context) error { // install k3s cmd := fmt.Sprintf( "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s sudo sh /tmp/k3s-install.sh server --token=%q --node-external-ip %s --flannel-backend wireguard-native --flannel-external-ip --disable traefik --node-name=%q", - file, + a.SSHPath, string(ip), a.node.NodeId, string(ip), @@ -182,7 +250,7 @@ func (a awsClient) CreateCluster(ctx context.Context) error { } // needed to fetch kubeconfig - configOut, err := utils.ExecCmdWithOutput(fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s cat /etc/rancher/k3s/k3s.yaml", file, string(ip)), "") + configOut, err := utils.ExecCmdWithOutput(fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s cat /etc/rancher/k3s/k3s.yaml", a.SSHPath, string(ip)), "") if err != nil { return err } @@ -196,24 +264,24 @@ func (a awsClient) CreateCluster(ctx context.Context) error { kubeconfig.Clusters[i].Cluster.Server = fmt.Sprintf("https://%s:6443", string(ip)) } - kc, err := yaml.Marshal(kubeconfig) - if err != nil { - return err - } - - tokenOut, err := utils.ExecCmdWithOutput(fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s cat /var/lib/rancher/k3s/server/node-token", file, string(ip)), "") - if err != nil { - return err - } - - _, err = a.tokenRepo.Create(ctx, &entities.Token{ - JoinToken: string(tokenOut), - EndpointUrl: fmt.Sprintf("https://%s:6443", ip), - KubeConfig: string(kc), - NodeId: a.node.NodeId, - AccountName: a.accountId, - ClusterName: "", - }) + // kc, err := yaml.Marshal(kubeconfig) + // if err != nil { + // return err + // } + + // tokenOut, err := utils.ExecCmdWithOutput(fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s cat /var/lib/rancher/k3s/server/node-token", a.SSHPath, string(ip)), "") + // if err != nil { + // return err + // } + + // _, err = a.tokenRepo.Create(ctx, &entities.Token{ + // JoinToken: string(tokenOut), + // EndpointUrl: fmt.Sprintf("https://%s:6443", ip), + // KubeConfig: string(kc), + // NodeId: a.node.NodeId, + // AccountName: a.accountId, + // ClusterName: "", + // }) return err } @@ -237,7 +305,7 @@ func parseValues(a awsClient) map[string]string { func (a awsClient) SaveToDbGuranteed(ctx context.Context) { for { - if err := utils.SaveToDb(ctx, a.node.NodeId, a.gfs); err == nil { + if err := utils.SaveToDb(a.node.NodeId, a.awsS3Client); err == nil { break } else { fmt.Println(err) @@ -248,25 +316,9 @@ func (a awsClient) SaveToDbGuranteed(ctx context.Context) { // NewNode implements ProviderClient func (a awsClient) NewNode(ctx context.Context) error { - file, err := ioutil.TempDir("/tmp/ssh", "ssh_") - if err != nil { - return err - } - - a.SSHPath = file - values := parseValues(a) - /* - steps: - - check if state present in db - - if present load that to working dir - - else initialize new tf dir - - apply terraform - - upload the final state with defer - */ - - if err := utils.MakeTfWorkFileReady(ctx, a.node.NodeId, path.Join(a.tfTemplates, "aws"), a.gfs, true); err != nil { + if err := utils.MakeTfWorkFileReady(a.node.NodeId, path.Join(a.tfTemplates, "aws"), a.awsS3Client, true); err != nil { return err } @@ -305,7 +357,7 @@ func (a awsClient) DeleteNode(ctx context.Context) error { - delete final state */ - if err := utils.MakeTfWorkFileReady(ctx, a.node.NodeId, path.Join(a.tfTemplates, "aws"), a.gfs, false); err != nil { + if err := utils.MakeTfWorkFileReady(a.node.NodeId, path.Join(a.tfTemplates, "aws"), a.awsS3Client, false); err != nil { return err } @@ -320,20 +372,18 @@ func (a awsClient) DeleteNode(ctx context.Context) error { return err } - filename := fmt.Sprintf("%s.zip", a.node.NodeId) - - if err := a.gfs.DeleteAllWithFilename(filename); err != nil { - return err - } - return nil } -func NewAwsProviderClient(node AWSNode, cpd common.CommonProviderData, apc AwsProviderConfig, gfs mongogridfs.GridFs, tokenRepo repos.DbRepo[*entities.Token]) common.ProviderClient { +func NewAwsProviderClient(node AWSNode, cpd common.CommonProviderData, apc AwsProviderConfig) (common.ProviderClient, error) { + awsS3Client, err := awss3.NewAwsS3Client(apc.AccessKey, apc.AccessSecret, node.NodeId) + if err != nil { + return nil, err + } + return awsClient{ - node: node, - gfs: gfs, - tokenRepo: tokenRepo, + node: node, + awsS3Client: awsS3Client, accessKey: apc.AccessKey, accessSecret: apc.AccessSecret, @@ -343,5 +393,5 @@ func NewAwsProviderClient(node AWSNode, cpd common.CommonProviderData, apc AwsPr labels: cpd.Labels, taints: cpd.Taints, SSHPath: cpd.SSHPath, - } + }, nil } diff --git a/apps/nodectrl/internal/domain/do/main.go b/apps/nodectrl/internal/domain/do/main.go index f70586348..6c9c68ff3 100644 --- a/apps/nodectrl/internal/domain/do/main.go +++ b/apps/nodectrl/internal/domain/do/main.go @@ -1,18 +1,5 @@ package do -import ( - "context" - "fmt" - "path" - "time" - - "kloudlite.io/apps/nodectrl/internal/domain/common" - "kloudlite.io/apps/nodectrl/internal/domain/entities" - "kloudlite.io/apps/nodectrl/internal/domain/utils" - mongogridfs "kloudlite.io/pkg/mongo-gridfs" - "kloudlite.io/pkg/repos" -) - type DoProviderConfig struct { ApiToken string `yaml:"apiToken"` AccountId string `yaml:"accountId"` @@ -26,7 +13,6 @@ type DoNode struct { } type doClient struct { - gfs mongogridfs.GridFs node DoNode apiToken string @@ -37,133 +23,3 @@ type doClient struct { labels map[string]string taints []string } - -// AttachNode implements common.ProviderClient. -func (doClient) AttachNode(ctx context.Context) error { - panic("unimplemented") -} - -// CreateAndAttachNode implements common.ProviderClient. -func (doClient) CreateAndAttachNode(ctx context.Context) error { - panic("unimplemented") -} - -// CreateCluster implements common.ProviderClient. -func (doClient) CreateCluster(ctx context.Context) error { - panic("unimplemented") -} - -func parseValues(d doClient) map[string]string { - values := map[string]string{} - - values["do-token"] = d.apiToken - values["accountId"] = d.accountId - - values["do-image-id"] = "ubuntu-22-10-x64" - values["nodeId"] = d.node.NodeId - values["size"] = d.node.Size - values["keys-path"] = d.SSHPath - - return values -} - -// SaveToDbGuranteed implements ProviderClient -func (d doClient) SaveToDbGuranteed(ctx context.Context) { - for { - if err := utils.SaveToDb(ctx, d.node.NodeId, d.gfs); err == nil { - break - } else { - fmt.Println(err) - } - time.Sleep(time.Second * 20) - } -} - -// NewNode implements ProviderClient -func (d doClient) NewNode(ctx context.Context) error { - values := parseValues(d) - - /* - steps: - - check if state present in db - - if present load that to working dir - - else initialize new tf dir - - apply terraform - - upload the final state with defer - */ - - if err := utils.MakeTfWorkFileReady(ctx, d.node.NodeId, path.Join(d.tfTemplates, "do"), d.gfs, true); err != nil { - return err - } - - defer d.SaveToDbGuranteed(ctx) - - // apply the tf file - if err := func() error { - if err := utils.InitTFdir(path.Join(utils.Workdir, d.node.NodeId)); err != nil { - return err - } - - if err := utils.ApplyTF(path.Join(utils.Workdir, d.node.NodeId), values); err != nil { - return err - } - - return nil - }(); err != nil { - return err - } - - return nil -} - -// DeleteNode implements ProviderClient -func (d doClient) DeleteNode(ctx context.Context) error { - values := parseValues(d) - - /* - steps: - - check if state present in db - - if present load that to working dir - - else initialize new tf dir - - destroy node with terraform - - delete final state - */ - - if err := utils.MakeTfWorkFileReady(ctx, d.node.NodeId, path.Join(d.tfTemplates, "aws"), d.gfs, false); err != nil { - return err - } - - // destroy the tf file - if err := func() error { - if err := utils.DestroyNode(d.node.NodeId, values); err != nil { - return err - } - - return nil - }(); err != nil { - return err - } - - filename := fmt.Sprintf("%s.zip", d.node.NodeId) - - if err := d.gfs.DeleteAllWithFilename(filename); err != nil { - return err - } - - return nil -} - -func NewDoProviderClient(node DoNode, cpd common.CommonProviderData, dpc DoProviderConfig, gfs mongogridfs.GridFs, tokenRepo repos.DbRepo[*entities.Token]) common.ProviderClient { - return doClient{ - node: node, - gfs: gfs, - - apiToken: dpc.ApiToken, - accountId: dpc.AccountId, - - tfTemplates: cpd.TfTemplates, - labels: cpd.Labels, - taints: cpd.Taints, - SSHPath: cpd.SSHPath, - } -} diff --git a/apps/nodectrl/internal/domain/entities/agent-token.go b/apps/nodectrl/internal/domain/entities/agent-token.go deleted file mode 100644 index 507472b3c..000000000 --- a/apps/nodectrl/internal/domain/entities/agent-token.go +++ /dev/null @@ -1,34 +0,0 @@ -package entities - -import ( - "kloudlite.io/pkg/repos" -) - -type Token struct { - repos.BaseEntity `json:",inline"` - - JoinToken string `json:"join%oken"` - EndpointUrl string `json:"endpointUrl" yaml:"endPointUrl"` - KubeConfig string `json:"kubeConfig" yaml:"kubeConfig"` - - NodeId string `json:"nodeId"` - AccountName string `json:"accountName"` - ClusterName string `json:"clusterName"` -} - -var TokenIndexes = []repos.IndexField{ - { - Field: []repos.IndexKey{ - {Key: "id", Value: repos.IndexAsc}, - }, - Unique: true, - }, - { - Field: []repos.IndexKey{ - {Key: "nodeId", Value: repos.IndexAsc}, - {Key: "accountName", Value: repos.IndexAsc}, - {Key: "clusterName", Value: repos.IndexAsc}, - }, - Unique: true, - }, -} diff --git a/apps/nodectrl/internal/domain/main.go b/apps/nodectrl/internal/domain/main.go index d9c9f7f3d..a4aada84c 100644 --- a/apps/nodectrl/internal/domain/main.go +++ b/apps/nodectrl/internal/domain/main.go @@ -3,37 +3,22 @@ package domain import ( "go.uber.org/fx" - "kloudlite.io/apps/nodectrl/internal/domain/entities" "kloudlite.io/apps/nodectrl/internal/env" - mongogridfs "kloudlite.io/pkg/mongo-gridfs" - "kloudlite.io/pkg/repos" ) type domain struct { - env *env.Env - gfs mongogridfs.GridFs - tokenRepo repos.DbRepo[*entities.Token] + env *env.Env } func (d domain) GetEnv() *env.Env { return d.env } -func (d domain) GetGRidFs() mongogridfs.GridFs { - return d.gfs -} - -func (d domain) GetTokenRepo() repos.DbRepo[*entities.Token] { - return d.tokenRepo -} - var Module = fx.Module("domain", fx.Provide( - func(env *env.Env, gfs mongogridfs.GridFs, tokenRepo repos.DbRepo[*entities.Token]) Domain { + func(env *env.Env) Domain { return domain{ - env: env, - gfs: gfs, - tokenRepo: tokenRepo, + env: env, } }, ), diff --git a/apps/nodectrl/internal/domain/port.go b/apps/nodectrl/internal/domain/port.go index 32a1a7704..76c8e5115 100644 --- a/apps/nodectrl/internal/domain/port.go +++ b/apps/nodectrl/internal/domain/port.go @@ -1,15 +1,9 @@ package domain import ( - "kloudlite.io/apps/nodectrl/internal/domain/entities" "kloudlite.io/apps/nodectrl/internal/env" - mongogridfs "kloudlite.io/pkg/mongo-gridfs" - "kloudlite.io/pkg/repos" ) type Domain interface { GetEnv() *env.Env - - GetGRidFs() mongogridfs.GridFs - GetTokenRepo() repos.DbRepo[*entities.Token] } diff --git a/apps/nodectrl/internal/domain/provider-client-fx.go b/apps/nodectrl/internal/domain/provider-client-fx.go index 3782a98c3..df2d1d281 100644 --- a/apps/nodectrl/internal/domain/provider-client-fx.go +++ b/apps/nodectrl/internal/domain/provider-client-fx.go @@ -2,7 +2,6 @@ package domain import ( "fmt" - "io/ioutil" "os" "go.uber.org/fx" @@ -16,11 +15,6 @@ import ( var ProviderClientFx = fx.Module("provider-client-fx", fx.Provide(func(env *env.Env, d Domain) (common.ProviderClient, error) { - privateKeyBytes, publicKeyBytes, err := utils.GenerateKeys() - if err != nil { - return nil, err - } - const sshDir = "/tmp/ssh" if _, err := os.Stat(sshDir); err != nil { @@ -29,19 +23,6 @@ var ProviderClientFx = fx.Module("provider-client-fx", } } - file, err := ioutil.TempDir("/tmp/ssh", "ssh_") - if err != nil { - return nil, err - } - - if err := os.WriteFile(fmt.Sprintf("%s/access.pub", file), publicKeyBytes, os.ModePerm); err != nil { - return nil, err - } - - if err := os.WriteFile(fmt.Sprintf("%s/access", file), privateKeyBytes, os.ModePerm); err != nil { - return nil, err - } - cpd := common.CommonProviderData{} if err := utils.Base64YamlDecode(env.ProviderConfig, &cpd); err != nil { @@ -65,7 +46,7 @@ var ProviderClientFx = fx.Module("provider-client-fx", return nil, err } - return aws.NewAwsProviderClient(node, cpd, apc, d.GetGRidFs(), d.GetTokenRepo()), nil + return aws.NewAwsProviderClient(node, cpd, apc) case "azure": panic("not implemented") case "do": @@ -82,7 +63,6 @@ var ProviderClientFx = fx.Module("provider-client-fx", return nil, err } - return do.NewDoProviderClient(node, cpd, dpc, d.GetGRidFs(), d.GetTokenRepo()), nil case "gcp": panic("not implemented") } diff --git a/apps/nodectrl/internal/domain/utils/fs.go b/apps/nodectrl/internal/domain/utils/fs.go index 2bb982e96..44e691d05 100644 --- a/apps/nodectrl/internal/domain/utils/fs.go +++ b/apps/nodectrl/internal/domain/utils/fs.go @@ -1,14 +1,13 @@ package utils import ( - "context" "fmt" "os" "path" "github.com/containerd/continuity/fs" - mongogridfs "kloudlite.io/pkg/mongo-gridfs" + awss3 "kloudlite.io/pkg/aws-s3" ) func CreateNodeWorkDir(nodeId string) error { @@ -35,16 +34,12 @@ func SetupGetWorkDir() error { return nil } -func MakeTfWorkFileReady(ctx context.Context, nodeId, tfPath string, gfs mongogridfs.GridFs, createIfNotExists bool) error { +func MakeTfWorkFileReady(nodeId, tfPath string, awss3client awss3.AwsS3, createIfNotExists bool) error { filename := fmt.Sprintf("%s.zip", nodeId) // check if file exists in db - gf, err := gfs.FetchFileRef(ctx, filename) + err := awss3client.IsFileExists(filename) if err != nil { - return err - } - // not found create new dir - if gf == nil { if !createIfNotExists { return fmt.Errorf("no state file found with the nodeId %s to operate", nodeId) } @@ -62,26 +57,22 @@ func MakeTfWorkFileReady(ctx context.Context, nodeId, tfPath string, gfs mongogr } // found file in db, download and extract to the workdir - fmt.Println(gf.Name, "found, extract it by downloading") + fmt.Println("found, extract it by downloading") source := path.Join(Workdir, filename) // Download from db - if err := gfs.Download(ctx, filename, source); err != nil { + if err := awss3client.DownloadFile(source, filename); err != nil { return err } if _, err := Unzip(source, path.Join(Workdir)); err != nil { return err - } else { - // for _, v := range s { - // fmt.Print(v, " \n") - // } } return nil } -func SaveToDb(ctx context.Context, nodeId string, gfs mongogridfs.GridFs) error { +func SaveToDb(nodeId string, awss3client awss3.AwsS3) error { /* Steps: - compress the workdir into zip @@ -104,7 +95,7 @@ func SaveToDb(ctx context.Context, nodeId string, gfs mongogridfs.GridFs) error return err } - if err := gfs.Upsert(ctx, filename, source); err != nil { + if err := awss3client.UploadFile(source, filename); err != nil { return err } diff --git a/apps/nodectrl/internal/framework/main.go b/apps/nodectrl/internal/framework/main.go index 1f1218b02..c44df3125 100644 --- a/apps/nodectrl/internal/framework/main.go +++ b/apps/nodectrl/internal/framework/main.go @@ -2,10 +2,9 @@ package framework import ( "go.uber.org/fx" + "kloudlite.io/apps/nodectrl/internal/app" "kloudlite.io/apps/nodectrl/internal/env" - mongogridfs "kloudlite.io/pkg/mongo-gridfs" - mongoDb "kloudlite.io/pkg/repos" ) type fm struct { @@ -21,7 +20,5 @@ var Module = fx.Module( fx.Provide(func(env *env.Env) *fm { return &fm{env} }), - mongogridfs.NewMongoGridFsClientFx[*fm](), - mongoDb.NewMongoClientFx[*fm](), app.Module, ) diff --git a/go.mod b/go.mod index 88428ffcc..edf9297ef 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.18 require ( github.com/99designs/gqlgen v0.17.28 github.com/Masterminds/sprig/v3 v3.2.3 + github.com/aws/aws-sdk-go v1.44.277 github.com/bradleyfalzon/ghinstallation/v2 v2.0.4 github.com/codingconcepts/env v0.0.0-20200821220118-a8fbf8d84482 github.com/containerd/continuity v0.4.1 @@ -21,6 +22,7 @@ require ( github.com/kloudlite/wg-operator v0.0.0-20230329090407-183297dc23b8 github.com/matoous/go-nanoid/v2 v2.0.0 github.com/miekg/dns v1.1.41 + github.com/moby/buildkit v0.11.6 github.com/otiai10/copy v1.11.0 github.com/pkg/errors v0.9.1 github.com/sendgrid/sendgrid-go v3.11.1+incompatible @@ -52,9 +54,18 @@ require ( ) require ( + github.com/containerd/containerd v1.6.20 // indirect + github.com/containerd/typeurl v1.0.2 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/gogo/googleapis v1.4.1 // indirect github.com/google/gnostic v0.6.9 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 // indirect + go.opentelemetry.io/otel v1.10.0 // indirect + go.opentelemetry.io/otel/trace v1.10.0 // indirect golang.org/x/net v0.8.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) @@ -110,7 +121,7 @@ require ( github.com/imdario/mergo v0.3.13 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.15.11 // indirect + github.com/klauspost/compress v1.15.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect diff --git a/go.sum b/go.sum index da47ba48f..a08a41a28 100644 --- a/go.sum +++ b/go.sum @@ -68,6 +68,10 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= +github.com/aws/aws-sdk-go v1.44.105 h1:UUwoD1PRKIj3ltrDUYTDQj5fOTK3XsnqolLpRTMmSEM= +github.com/aws/aws-sdk-go v1.44.105/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.277 h1:YHmyzBPARTJ7LLYV1fxbfEbQOaUh3kh52hb7nBvX3BQ= +github.com/aws/aws-sdk-go v1.44.277/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -95,8 +99,12 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/codingconcepts/env v0.0.0-20200821220118-a8fbf8d84482 h1:5/aEFreBh9hH/0G+33xtczJCvMaulqsm9nDuu2BZUEo= github.com/codingconcepts/env v0.0.0-20200821220118-a8fbf8d84482/go.mod h1:TM9ug+H/2cI3EjyIDr5xKCkFGyNE59URgH1wu5NyU8E= +github.com/containerd/containerd v1.6.20 h1:+itjwpdqXpzHB/QAiWc/BZCjjVfcNgw69w/oIeF4Oy0= +github.com/containerd/containerd v1.6.20/go.mod h1:apei1/i5Ux2FzrK6+DM/suEsGuK/MeVOfy8tR2q7Wnw= github.com/containerd/continuity v0.4.1 h1:wQnVrjIyQ8vhU2sgOiL5T07jo+ouqc2bnKsv5/EqGhU= github.com/containerd/continuity v0.4.1/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY= +github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -142,8 +150,11 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= @@ -170,6 +181,8 @@ github.com/gofiber/utils v0.1.2 h1:1SH2YEz4RlNS0tJlMJ0bGwO0JkqPqvq6TbHK9tXZKtk= github.com/gofiber/utils v0.1.2/go.mod h1:pacRFtghAE3UoknMOUiXh2Io/nLWSUHtQCi/3QASsOc= github.com/gofiber/websocket/v2 v2.0.21 h1:mQEiLXBqFsNNlJc5dzFgSGeoqoEXYvIcdBQzAZBdbL0= github.com/gofiber/websocket/v2 v2.0.21/go.mod h1:AOdLDGRGMr9MXH0GjHD43xR17x5lzs0pd5E0/cEKYX8= +github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0= +github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -261,6 +274,8 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4GkDEdKCRJduHpTxT3/jcw= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -290,6 +305,10 @@ github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -309,8 +328,8 @@ github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.14.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.4/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= -github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM= +github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/kloudlite/cluster-operator v0.0.0-20230329090334-40fc9f00d55e h1:GkOuSFWhsl2AINzhIHc67XOydGGha3sSHOIRr72Id1M= github.com/kloudlite/cluster-operator v0.0.0-20230329090334-40fc9f00d55e/go.mod h1:TYrwSdXGOXd3SVBlzm6BDr3+zItMEls6HfuidhqjvMo= github.com/kloudlite/operator v0.0.0-20230519115853-9aa81fd2e1f7 h1:njEoULGhPluSLU1aVU6GQFJJysNCsZUnDCc8fgTzn7A= @@ -353,6 +372,8 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/buildkit v0.11.6 h1:VYNdoKk5TVxN7k4RvZgdeM4GOyRvIi4Z8MXOY7xvyUs= +github.com/moby/buildkit v0.11.6/go.mod h1:GCqKfHhz+pddzfgaR7WmHVEE3nKKZMMDPpK8mh3ZLv4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -371,6 +392,7 @@ github.com/onsi/ginkgo/v2 v2.6.0 h1:9t9b9vRUbFq3C4qKFCGkVuq/fIHji802N1nrtkh1mNc= github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin/zipkin-go v0.3.0 h1:XtuXmOLIXLjiU2XduuWREDT0LOKtSgos/g7i7RYyoZQ= github.com/otiai10/copy v1.11.0 h1:OKBD80J/mLBrwnzXqGtFCzprFSGioo30JcmR4APsNwc= github.com/otiai10/copy v1.11.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= @@ -525,6 +547,12 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 h1:xFSRQBbXF6VvYRf2lqMJXxoB72XI1K/azav8TekHHSw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0/go.mod h1:h8TWwRAhQpOd0aM5nYsRD8+flnkj+526GEIVlarH7eY= +go.opentelemetry.io/otel v1.10.0 h1:Y7DTJMR6zs1xkS/upamJYk0SxxN4C9AqRd77jmZnyY4= +go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= +go.opentelemetry.io/otel/trace v1.10.0 h1:npQMbR8o7mum8uF95yFbOEJffhs1sbCOfDh8zAJiH5E= +go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -630,6 +658,7 @@ golang.org/x/net v0.0.0-20220111093109-d55c255bac03/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= @@ -706,12 +735,14 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= @@ -835,6 +866,7 @@ google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -905,6 +937,7 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/aws-s3/main.go b/pkg/aws-s3/main.go new file mode 100644 index 000000000..1940e83dd --- /dev/null +++ b/pkg/aws-s3/main.go @@ -0,0 +1,138 @@ +package awss3 + +import ( + "fmt" + "io" + "os" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" +) + +type AwsS3 interface { + DownloadFile(filePath, fileKey string) error + UploadFile(filePath, fileKey string) error + IsFileExists(fileKey string) error +} + +func (a awsS3) createBucket() error { + _, err := a.svc.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(a.bucketName), + }) + if err != nil { + return err + } + + return nil +} + +func (a awsS3) IsFileExists(fileKey string) error { + _, err := a.svc.HeadObject(&s3.HeadObjectInput{ + Bucket: aws.String(a.bucketName), + Key: aws.String(fileKey), + }) + if err != nil { + // If the file does not exist or there is an error, handle the error + fmt.Println(err) + return err + } + return nil +} + +func (a awsS3) checkS3Created() error { + _, err := a.svc.HeadBucket(&s3.HeadBucketInput{ + Bucket: aws.String(a.bucketName), + }) + if err != nil { + return err + } + return nil +} + +func (a awsS3) createBucketIfNotCreated() error { + if err := a.checkS3Created(); err != nil { + return a.createBucket() + } + return nil +} + +func getS3Client(accessKey, accessSecret string) (*s3.S3, error) { + sess, err := session.NewSession(&aws.Config{ + Region: aws.String("us-west-1"), // Replace with your desired region + Credentials: credentials.NewStaticCredentials(accessKey, accessSecret, ""), + }) + if err != nil { + return nil, err + } + + return s3.New(sess), nil +} + +type awsS3 struct { + svc *s3.S3 + bucketName string +} + +// downloadFile implements AwsS3. +func (a awsS3) DownloadFile(filePath, fileKey string) error { + file, err := os.Create(filePath) + if err != nil { + return err + } + defer file.Close() + + // Download the file from S3 + resp, err := a.svc.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(a.bucketName), + Key: aws.String(fileKey), + }) + if err != nil { + return err + } + + // Write the downloaded file data to the file + _, err = io.Copy(file, resp.Body) + if err != nil { + return err + } + return nil +} + +// uploadFile implements AwsS3. +func (a awsS3) UploadFile(filePath, fileKey string) error { + file, err := os.Open(filePath) + if err != nil { + return err + } + defer file.Close() + + _, err = a.svc.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(a.bucketName), + Key: aws.String(fileKey), + Body: file, + }) + if err != nil { + return err + } + + return nil +} + +func NewAwsS3Client(accessKey, accessSec, bucketName string) (AwsS3, error) { + svc, err := getS3Client(accessKey, accessSec) + if err != nil { + return nil, err + } + + a := awsS3{ + svc: svc, + bucketName: bucketName, + } + if err := a.createBucketIfNotCreated(); err != nil { + return nil, err + } + + return a, nil +} From 5487bd52b604fb7d854093cbced9e03f9e0da59a Mon Sep 17 00:00:00 2001 From: Abdhesh Nayak Date: Thu, 8 Jun 2023 14:10:19 +0530 Subject: [PATCH 4/5] :construction: wip --- apps/nodectrl/internal/app/main.go | 20 +- apps/nodectrl/internal/domain/aws/main.go | 250 +++++++++++++----- .../internal/domain/common/interface.go | 7 +- .../internal/domain/provider-client-fx.go | 5 +- apps/nodectrl/internal/domain/utils/main.go | 2 +- 5 files changed, 214 insertions(+), 70 deletions(-) diff --git a/apps/nodectrl/internal/app/main.go b/apps/nodectrl/internal/app/main.go index 424a65001..4be72a05e 100644 --- a/apps/nodectrl/internal/app/main.go +++ b/apps/nodectrl/internal/app/main.go @@ -18,7 +18,7 @@ var Module = fx.Module("app", func(env *env.Env, pc common.ProviderClient, shutdowner fx.Shutdowner, lifecycle fx.Lifecycle) { lifecycle.Append(fx.Hook{ OnStart: func(context.Context) error { - go func() error { + runner := func() error { ctx := context.Background() if err := utils.SetupGetWorkDir(); err != nil { return err @@ -31,6 +31,18 @@ var Module = fx.Module("app", if err := pc.CreateCluster(ctx); err != nil { return err } + case "add-master": + fmt.Println("needs to attach master") + if err := pc.AddMaster(ctx); err != nil { + return err + } + + case "add-worker": + fmt.Println("needs to attach worker") + if err := pc.AddWorker(ctx); err != nil { + return err + } + case "delete": fmt.Println("needs to delete node") if err := pc.DeleteNode(ctx); err != nil { @@ -52,6 +64,12 @@ var Module = fx.Module("app", return err } return nil + } + + go func() { + if err := runner(); err != nil { + shutdowner.Shutdown() + } }() return nil diff --git a/apps/nodectrl/internal/domain/aws/main.go b/apps/nodectrl/internal/domain/aws/main.go index 02f992878..881e04298 100644 --- a/apps/nodectrl/internal/domain/aws/main.go +++ b/apps/nodectrl/internal/domain/aws/main.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path" + "strings" "time" "gopkg.in/yaml.v2" @@ -26,6 +27,7 @@ type AWSNode struct { InstanceType string `yaml:"instanceType"` VPC string `yaml:"vpc"` ImageId string `yaml:"imageId"` + IsGpu bool `yaml:"isGpu"` } type awsClient struct { @@ -42,43 +44,123 @@ type awsClient struct { taints []string } -// CreateAndAttachNode implements common.ProviderClient -func (a awsClient) CreateAndAttachNode(ctx context.Context) error { +type tokenAndKubeconfig struct { + Token string `json:"token"` + Kubeconfig string `json:"kubeconfig"` + ServerIp string `json:"serverIp"` +} + +// AddMaster implements common.ProviderClient. +func (a awsClient) AddMaster(ctx context.Context) error { + // fetch token + a.SSHPath = path.Join("/tmp/ssh", a.accountId) + + tokenFileName := fmt.Sprintf("%s-config.yaml", a.accountId) + + if err := a.awsS3Client.IsFileExists(tokenFileName); err != nil { + return err + } + + tokenPath := path.Join(a.SSHPath, "config.yaml") + if err := a.awsS3Client.DownloadFile(tokenPath, tokenFileName); err != nil { + return err + } + + b, err := os.ReadFile(tokenPath) + if err != nil { + return err + } + + kc := tokenAndKubeconfig{} + + if err := yaml.Unmarshal(b, &kc); err != nil { + return err + } + + // setup ssh + + if err := a.SetupSSH(); err != nil { + return err + } + defer a.saveForSure() + + // create node and wait for ready if err := a.NewNode(ctx); err != nil { return err } - if err := a.AttachNode(ctx); err != nil { + ip, err := utils.GetOutput(path.Join(utils.Workdir, a.node.NodeId), "node-ip") + if err != nil { return err } - return nil + count := 0 + + for { + if e := utils.ExecCmd( + fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s ls", + fmt.Sprintf("%v/access", a.SSHPath), + string(ip), + ), + ""); e == nil { + break + } + + count++ + if count > 24 { + return fmt.Errorf("node is not ready even after 6 minutes") + } + time.Sleep(time.Second * 5) + } + + // attach to cluster as master + + panic("unimplemented") } -// AttachNode implements common.ProviderClient -func (a awsClient) AttachNode(ctx context.Context) error { - /* - check readyness, wait if not ready - if ready install agent - to install fetch - */ +func (a awsClient) AddWorker(ctx context.Context) error { + // fetch token + + a.SSHPath = path.Join("/tmp/ssh", a.accountId) + + tokenFileName := fmt.Sprintf("%s-config.yaml", a.accountId) - // - var out []byte + if err := a.awsS3Client.IsFileExists(tokenFileName); err != nil { + return err + } - out, err := utils.GetOutput(path.Join(utils.Workdir, a.node.NodeId), "node-ip") + tokenPath := path.Join(a.SSHPath, "config.yaml") + if err := a.awsS3Client.DownloadFile(tokenPath, tokenFileName); err != nil { + return err + } + + b, err := os.ReadFile(tokenPath) if err != nil { return err } - // labels := func() []string { - // l := []string{} - // for k, v := range a.labels { - // l = append(l, fmt.Sprintf("--node-label %s=%s", k, v)) - // } - // l = append(l, fmt.Sprintf("--node-label %s=%s", "kloudlite.io/public-ip", string(out))) - // return l - // }() + kc := tokenAndKubeconfig{} + + if err := yaml.Unmarshal(b, &kc); err != nil { + return err + } + + // setup ssh + + if err := a.SetupSSH(); err != nil { + return err + } + defer a.saveForSure() + + // create node and wait for ready + if err := a.NewNode(ctx); err != nil { + return err + } + + ip, err := utils.GetOutput(path.Join(utils.Workdir, a.node.NodeId), "node-ip") + if err != nil { + return err + } count := 0 @@ -86,8 +168,9 @@ func (a awsClient) AttachNode(ctx context.Context) error { if e := utils.ExecCmd( fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s ls", fmt.Sprintf("%s/access", a.SSHPath), - string(out)), - "checking if node is ready "); e == nil { + string(ip), + ), + ""); e == nil { break } @@ -95,17 +178,47 @@ func (a awsClient) AttachNode(ctx context.Context) error { if count > 24 { return fmt.Errorf("node is not ready even after 6 minutes") } - time.Sleep(time.Second * 15) + time.Sleep(time.Second * 5) } - // // attach node - // if e := utils.ExecCmd( - // fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s sudo sh /tmp/k3s-install.sh agent --server %s --token %s %s --node-name %s --node-external-ip %s --node-ip %s", - // fmt.Sprintf("%v/access", a.SSHPath), string(out), token.EndpointUrl, token.JoinToken, - // strings.Join(labels, " "), a.node.NodeId, string(out), string(out)), - // "attaching to cluster"); e != nil { - // return e - // } + labels := func() []string { + l := []string{} + for k, v := range map[string]string{ + "kloudlite.io/public-ip": string(ip), + } { + l = append(l, fmt.Sprintf("--node-label %s=%s", k, v)) + } + + for k, v := range a.labels { + l = append(l, fmt.Sprintf("--node-label %s=%s", k, v)) + } + return l + }() + + // attach to cluster as workernode + + cmd := fmt.Sprintf( + "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s/access root@%s sudo sh /tmp/k3s-install.sh agent --token=%s --server https://%s:6443 --node-external-ip %s --node-name %s %s %s", + a.SSHPath, + ip, + strings.TrimSpace(string(kc.Token)), + kc.ServerIp, + ip, + fmt.Sprintf("kl-worker-%s", a.node.NodeId), + strings.Join(labels, " "), + func() string { + if a.node.IsGpu { + // return "--docker" + // return "--docker" + return "" + } + return "" + }(), + ) + + if err := utils.ExecCmd(cmd, ""); err != nil { + return err + } return nil } @@ -147,12 +260,16 @@ func (a awsClient) SetupSSH() error { return nil } + if err := os.RemoveAll(destDir); err != nil { + return err + } + err := a.awsS3Client.DownloadFile(path.Join(sshDir, fileName), fileName) if err != nil { return err } - _, err = utils.Unzip(path.Join(sshDir, fileName), destDir) + _, err = utils.Unzip(path.Join(sshDir, fileName), sshDir) if err != nil { return err } @@ -232,15 +349,14 @@ func (a awsClient) CreateCluster(ctx context.Context) error { if count > 24 { return fmt.Errorf("node is not ready even after 6 minutes") } - time.Sleep(time.Second * 15) + time.Sleep(time.Second * 5) } // install k3s cmd := fmt.Sprintf( - "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s sudo sh /tmp/k3s-install.sh server --token=%q --node-external-ip %s --flannel-backend wireguard-native --flannel-external-ip --disable traefik --node-name=%q", + "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s/access root@%s sudo sh /tmp/k3s-install.sh server --node-external-ip %s --flannel-backend wireguard-native --flannel-external-ip --disable traefik --node-name=%s", a.SSHPath, string(ip), - a.node.NodeId, string(ip), fmt.Sprintf("kl-master-%s", a.node.NodeId), ) @@ -250,7 +366,7 @@ func (a awsClient) CreateCluster(ctx context.Context) error { } // needed to fetch kubeconfig - configOut, err := utils.ExecCmdWithOutput(fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s cat /etc/rancher/k3s/k3s.yaml", a.SSHPath, string(ip)), "") + configOut, err := utils.ExecCmdWithOutput(fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s/access root@%s cat /etc/rancher/k3s/k3s.yaml", a.SSHPath, string(ip)), "") if err != nil { return err } @@ -264,24 +380,36 @@ func (a awsClient) CreateCluster(ctx context.Context) error { kubeconfig.Clusters[i].Cluster.Server = fmt.Sprintf("https://%s:6443", string(ip)) } - // kc, err := yaml.Marshal(kubeconfig) - // if err != nil { - // return err - // } + kc, err := yaml.Marshal(kubeconfig) + if err != nil { + return err + } - // tokenOut, err := utils.ExecCmdWithOutput(fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s cat /var/lib/rancher/k3s/server/node-token", a.SSHPath, string(ip)), "") - // if err != nil { - // return err - // } + tokenOut, err := utils.ExecCmdWithOutput(fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s/access root@%s cat /var/lib/rancher/k3s/server/node-token", a.SSHPath, string(ip)), "") + if err != nil { + return err + } - // _, err = a.tokenRepo.Create(ctx, &entities.Token{ - // JoinToken: string(tokenOut), - // EndpointUrl: fmt.Sprintf("https://%s:6443", ip), - // KubeConfig: string(kc), - // NodeId: a.node.NodeId, - // AccountName: a.accountId, - // ClusterName: "", - // }) + st := tokenAndKubeconfig{ + Token: string(tokenOut), + Kubeconfig: string(kc), + ServerIp: string(ip), + } + + b, err := yaml.Marshal(st) + if err != nil { + return err + } + + tokenPath := path.Join(a.SSHPath, "config.yaml") + + if err := os.WriteFile(tokenPath, b, os.ModePerm); err != nil { + return err + } + + if err := a.awsS3Client.UploadFile(tokenPath, fmt.Sprintf("%s-config.yaml", a.accountId)); err != nil { + return err + } return err } @@ -298,8 +426,6 @@ func parseValues(a awsClient) map[string]string { values["keys-path"] = a.SSHPath values["ami"] = a.node.ImageId - fmt.Print(values) - return values } @@ -318,11 +444,13 @@ func (a awsClient) SaveToDbGuranteed(ctx context.Context) { func (a awsClient) NewNode(ctx context.Context) error { values := parseValues(a) - if err := utils.MakeTfWorkFileReady(a.node.NodeId, path.Join(a.tfTemplates, "aws"), a.awsS3Client, true); err != nil { - return err - } + if true { + if err := utils.MakeTfWorkFileReady(a.node.NodeId, path.Join(a.tfTemplates, "aws"), a.awsS3Client, true); err != nil { + return err + } - defer a.SaveToDbGuranteed(ctx) + defer a.SaveToDbGuranteed(ctx) + } // upload the final state to the db, upsert if db is already present @@ -376,7 +504,7 @@ func (a awsClient) DeleteNode(ctx context.Context) error { } func NewAwsProviderClient(node AWSNode, cpd common.CommonProviderData, apc AwsProviderConfig) (common.ProviderClient, error) { - awsS3Client, err := awss3.NewAwsS3Client(apc.AccessKey, apc.AccessSecret, node.NodeId) + awsS3Client, err := awss3.NewAwsS3Client(apc.AccessKey, apc.AccessSecret, apc.AccountId) if err != nil { return nil, err } diff --git a/apps/nodectrl/internal/domain/common/interface.go b/apps/nodectrl/internal/domain/common/interface.go index 0d6908fe2..a47028362 100644 --- a/apps/nodectrl/internal/domain/common/interface.go +++ b/apps/nodectrl/internal/domain/common/interface.go @@ -12,19 +12,20 @@ Tasks needs to be performed by this job */ type ProviderClient interface { - CreateAndAttachNode(ctx context.Context) error /* ssh generation create node AttachNode */ NewNode(ctx context.Context) error - AttachNode(ctx context.Context) error - DeleteNode(ctx context.Context) error SaveToDbGuranteed(ctx context.Context) CreateCluster(ctx context.Context) error + + AddWorker(ctx context.Context) error + AddMaster(ctx context.Context) error + /* It will perform generation of ssh create node diff --git a/apps/nodectrl/internal/domain/provider-client-fx.go b/apps/nodectrl/internal/domain/provider-client-fx.go index df2d1d281..f1f7368da 100644 --- a/apps/nodectrl/internal/domain/provider-client-fx.go +++ b/apps/nodectrl/internal/domain/provider-client-fx.go @@ -1,7 +1,6 @@ package domain import ( - "fmt" "os" "go.uber.org/fx" @@ -14,7 +13,7 @@ import ( ) var ProviderClientFx = fx.Module("provider-client-fx", - fx.Provide(func(env *env.Env, d Domain) (common.ProviderClient, error) { + fx.Provide(func(env *env.Env) (common.ProviderClient, error) { const sshDir = "/tmp/ssh" if _, err := os.Stat(sshDir); err != nil { @@ -40,8 +39,6 @@ var ProviderClientFx = fx.Module("provider-client-fx", apc := aws.AwsProviderConfig{} - fmt.Println("here......................", env.AWSProviderConfig) - if err := utils.Base64YamlDecode(env.AWSProviderConfig, &apc); err != nil { return nil, err } diff --git a/apps/nodectrl/internal/domain/utils/main.go b/apps/nodectrl/internal/domain/utils/main.go index 65af4e8ca..48f38b5d5 100644 --- a/apps/nodectrl/internal/domain/utils/main.go +++ b/apps/nodectrl/internal/domain/utils/main.go @@ -49,7 +49,7 @@ func ExecCmd(cmdString string, logStr string) error { // cmd.Stdout = os.Stdout if err := cmd.Run(); err != nil { - fmt.Printf("err occurred: %s\n", err.Error()) + fmt.Printf("err occurred: %v\n", err.Error()) return err } return nil From 861cb219e18e99de87f60352fd79c365333cefdb Mon Sep 17 00:00:00 2001 From: Abdhesh Nayak Date: Fri, 9 Jun 2023 11:48:45 +0530 Subject: [PATCH 5/5] :construction: wip --- apps/nodectrl/internal/domain/aws/main.go | 65 ++++++++++++++----- apps/nodectrl/internal/domain/utils/fs.go | 2 +- apps/nodectrl/internal/domain/utils/main.go | 2 +- apps/nodectrl/internal/domain/utils/zipper.go | 8 +++ apps/nodectrl/terraform/aws/resource.tf | 14 ++++ 5 files changed, 72 insertions(+), 19 deletions(-) diff --git a/apps/nodectrl/internal/domain/aws/main.go b/apps/nodectrl/internal/domain/aws/main.go index 881e04298..4621b8552 100644 --- a/apps/nodectrl/internal/domain/aws/main.go +++ b/apps/nodectrl/internal/domain/aws/main.go @@ -8,6 +8,7 @@ import ( "strings" "time" + guuid "github.com/google/uuid" "gopkg.in/yaml.v2" "kloudlite.io/apps/nodectrl/internal/domain/common" @@ -45,9 +46,10 @@ type awsClient struct { } type tokenAndKubeconfig struct { - Token string `json:"token"` - Kubeconfig string `json:"kubeconfig"` - ServerIp string `json:"serverIp"` + Token string `json:"token"` + Kubeconfig string `json:"kubeconfig"` + ServerIp string `json:"serverIp"` + MasterToken string `json:"masterToken"` } // AddMaster implements common.ProviderClient. @@ -61,6 +63,12 @@ func (a awsClient) AddMaster(ctx context.Context) error { return err } + if _, err := os.Stat(a.SSHPath); err != nil { + if e := os.Mkdir(a.SSHPath, os.ModePerm); e != nil { + return e + } + } + tokenPath := path.Join(a.SSHPath, "config.yaml") if err := a.awsS3Client.DownloadFile(tokenPath, tokenFileName); err != nil { return err @@ -102,7 +110,7 @@ func (a awsClient) AddMaster(ctx context.Context) error { fmt.Sprintf("%v/access", a.SSHPath), string(ip), ), - ""); e == nil { + "checking if node is ready"); e == nil { break } @@ -114,8 +122,21 @@ func (a awsClient) AddMaster(ctx context.Context) error { } // attach to cluster as master + cmd := fmt.Sprintf( + "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s/access root@%s sudo sh /tmp/k3s-install.sh server --server https://%s:6443 --token %s --node-external-ip %s --flannel-backend wireguard-native --flannel-external-ip --disable traefik --node-name=%s", + a.SSHPath, + string(ip), + kc.ServerIp, + strings.TrimSpace(string(kc.Token)), + string(ip), + fmt.Sprintf("kl-master-%s", a.node.NodeId), + ) - panic("unimplemented") + if err := utils.ExecCmd(cmd, "attaching to cluster as a master"); err != nil { + return err + } + + return nil } func (a awsClient) AddWorker(ctx context.Context) error { @@ -123,6 +144,12 @@ func (a awsClient) AddWorker(ctx context.Context) error { a.SSHPath = path.Join("/tmp/ssh", a.accountId) + if _, err := os.Stat(a.SSHPath); err != nil { + if e := os.Mkdir(a.SSHPath, os.ModePerm); e != nil { + return e + } + } + tokenFileName := fmt.Sprintf("%s-config.yaml", a.accountId) if err := a.awsS3Client.IsFileExists(tokenFileName); err != nil { @@ -170,7 +197,7 @@ func (a awsClient) AddWorker(ctx context.Context) error { fmt.Sprintf("%s/access", a.SSHPath), string(ip), ), - ""); e == nil { + "checking if node ready"); e == nil { break } @@ -198,11 +225,11 @@ func (a awsClient) AddWorker(ctx context.Context) error { // attach to cluster as workernode cmd := fmt.Sprintf( - "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s/access root@%s sudo sh /tmp/k3s-install.sh agent --token=%s --server https://%s:6443 --node-external-ip %s --node-name %s %s %s", + "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s/access root@%s sudo sh /tmp/k3s-install.sh agent --server https://%s:6443 --token=%s --node-external-ip %s --node-name %s %s %s", a.SSHPath, ip, - strings.TrimSpace(string(kc.Token)), kc.ServerIp, + strings.TrimSpace(string(kc.Token)), ip, fmt.Sprintf("kl-worker-%s", a.node.NodeId), strings.Join(labels, " "), @@ -216,7 +243,7 @@ func (a awsClient) AddWorker(ctx context.Context) error { }(), ) - if err := utils.ExecCmd(cmd, ""); err != nil { + if err := utils.ExecCmd(cmd, "attaching to cluster as a worker node"); err != nil { return err } @@ -341,7 +368,7 @@ func (a awsClient) CreateCluster(ctx context.Context) error { fmt.Sprintf("%v/access", a.SSHPath), string(ip), ), - ""); e == nil { + "checking is node is ready"); e == nil { break } @@ -352,21 +379,24 @@ func (a awsClient) CreateCluster(ctx context.Context) error { time.Sleep(time.Second * 5) } + masterToken := guuid.New() + // install k3s cmd := fmt.Sprintf( - "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s/access root@%s sudo sh /tmp/k3s-install.sh server --node-external-ip %s --flannel-backend wireguard-native --flannel-external-ip --disable traefik --node-name=%s", + "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s/access root@%s sudo sh /tmp/k3s-install.sh server --token=%s --node-external-ip %s --flannel-backend wireguard-native --flannel-external-ip --disable traefik --node-name=%s --cluster-init", a.SSHPath, string(ip), + masterToken.String(), string(ip), fmt.Sprintf("kl-master-%s", a.node.NodeId), ) - if err := utils.ExecCmd(cmd, cmd); err != nil { + if err := utils.ExecCmd(cmd, "installing k3s"); err != nil { return err } // needed to fetch kubeconfig - configOut, err := utils.ExecCmdWithOutput(fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s/access root@%s cat /etc/rancher/k3s/k3s.yaml", a.SSHPath, string(ip)), "") + configOut, err := utils.ExecCmdWithOutput(fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s/access root@%s cat /etc/rancher/k3s/k3s.yaml", a.SSHPath, string(ip)), "fetching kubeconfig from the cluster") if err != nil { return err } @@ -385,15 +415,16 @@ func (a awsClient) CreateCluster(ctx context.Context) error { return err } - tokenOut, err := utils.ExecCmdWithOutput(fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s/access root@%s cat /var/lib/rancher/k3s/server/node-token", a.SSHPath, string(ip)), "") + tokenOut, err := utils.ExecCmdWithOutput(fmt.Sprintf("ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s/access root@%s cat /var/lib/rancher/k3s/server/node-token", a.SSHPath, string(ip)), "fetching node token from the cluster") if err != nil { return err } st := tokenAndKubeconfig{ - Token: string(tokenOut), - Kubeconfig: string(kc), - ServerIp: string(ip), + Token: string(tokenOut), + Kubeconfig: string(kc), + ServerIp: string(ip), + MasterToken: masterToken.String(), } b, err := yaml.Marshal(st) diff --git a/apps/nodectrl/internal/domain/utils/fs.go b/apps/nodectrl/internal/domain/utils/fs.go index 44e691d05..11659fb4e 100644 --- a/apps/nodectrl/internal/domain/utils/fs.go +++ b/apps/nodectrl/internal/domain/utils/fs.go @@ -57,7 +57,7 @@ func MakeTfWorkFileReady(nodeId, tfPath string, awss3client awss3.AwsS3, createI } // found file in db, download and extract to the workdir - fmt.Println("found, extract it by downloading") + fmt.Println("-> found, extract it by downloading") source := path.Join(Workdir, filename) // Download from db diff --git a/apps/nodectrl/internal/domain/utils/main.go b/apps/nodectrl/internal/domain/utils/main.go index 48f38b5d5..1ff978f1c 100644 --- a/apps/nodectrl/internal/domain/utils/main.go +++ b/apps/nodectrl/internal/domain/utils/main.go @@ -21,7 +21,7 @@ func Base64YamlDecode(in string, out interface{}) error { return err } - fmt.Println(string(rawDecodedText)) + // fmt.Println(string(rawDecodedText)) return yaml.Unmarshal(rawDecodedText, out) } diff --git a/apps/nodectrl/internal/domain/utils/zipper.go b/apps/nodectrl/internal/domain/utils/zipper.go index 8b447df75..b8ae8a200 100644 --- a/apps/nodectrl/internal/domain/utils/zipper.go +++ b/apps/nodectrl/internal/domain/utils/zipper.go @@ -15,6 +15,9 @@ import ( ) func ZipSource(source, target string) error { + fmt.Printf("\n[#] compressing %s -> %s\n", source, target) + defer fmt.Printf("\n[#] compressed %s -> %s\n", source, target) + // 1. Create a ZIP file and zip.Writer f, err := os.Create(target) if err != nil { @@ -71,6 +74,8 @@ func ZipSource(source, target string) error { } func Unzip(src string, destination string) ([]string, error) { + fmt.Printf("\n[#] extracting %s -> %s\n", src, destination) + defer fmt.Printf("\n[#] extracted %s -> %s\n", src, destination) var filenames []string r, err := zip.OpenReader(src) if err != nil { @@ -125,6 +130,9 @@ func Unzip(src string, destination string) ([]string, error) { } func ExtractZip(src, destination string) error { + fmt.Printf("[#] extracting %s -> %s", src, destination) + defer fmt.Printf("[#] extracted %s -> %s", src, destination) + if _, err := os.Stat(destination); err == nil { if er := os.RemoveAll(destination); er != nil { return err diff --git a/apps/nodectrl/terraform/aws/resource.tf b/apps/nodectrl/terraform/aws/resource.tf index e1ec3ae56..0ad99ab34 100644 --- a/apps/nodectrl/terraform/aws/resource.tf +++ b/apps/nodectrl/terraform/aws/resource.tf @@ -42,6 +42,20 @@ resource "aws_security_group" "sg" { cidr_blocks = ["0.0.0.0/0"] } + ingress { + from_port = 2379 + protocol = "tcp" + to_port = 2379 + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + from_port = 2380 + protocol = "tcp" + to_port = 2380 + cidr_blocks = ["0.0.0.0/0"] + } + ingress { from_port = 6443 protocol = "tcp"