jakolehm · jakolehm · Apr 7, 2020 · Apr 7, 2020
diff --git a/.travis.yml b/.travis.yml
@@ -0,0 +1,23 @@
+language: go
+sudo: required
+go: 1.13
+stages:
+  - name: e2e
+    if: type IN (pull_request, cron)
+jobs:
+  include:
+    - stage: e2e
+      name: "e2e: ubuntu 18.04"
+      script: ./e2e/travis.sh
+      env:
+        - FOOTLOOSE_IMAGE=quay.io/footloose/ubuntu18.04
+    - stage: e2e
+      name: "e2e: centos7"
+      script: ./e2e/travis.sh
+      env:
+        - FOOTLOOSE_IMAGE=quay.io/footloose/centos7
+    - stage: e2e
+      name: "e2e: debian10"
+      script: ./e2e/travis.sh
+      env:
+        - FOOTLOOSE_IMAGE=quay.io/footloose/debian10
diff --git a/README.md b/README.md
@@ -29,7 +29,7 @@ hosts:
     role: master
 ```
 
-### Fill cluster.yml example
+### Full cluster.yml example
 
 ```yaml
 token: verysecret
@@ -48,4 +48,6 @@ hosts:
     sshPort: 22
     extraArgs:
       - "--node-label foo=bar"
+manifests:
+  - ./manifests/*.yaml
 ```
diff --git a/cmd/up.go b/cmd/up.go
@@ -50,7 +50,9 @@ func clusterUp(ctx *cli.Context) error {
 	phaseManager := phases.NewManager(&cluster)
 
 	phaseManager.AddPhase(&phases.ConnectPhase{})
+	phaseManager.AddPhase(&phases.GatherHostFactsPhase{})
 	phaseManager.AddPhase(&phases.SetupMastersPhase{})
+	phaseManager.AddPhase(&phases.CopyManifestsPhase{})
 	phaseManager.AddPhase(&phases.SetupWorkersPhase{})
 	phaseManager.AddPhase(&phases.DisconnectPhase{})
 

diff --git a/e2e/cluster.yml b/e2e/cluster.yml
@@ -0,0 +1,18 @@
+hosts:
+  - address: "127.0.0.1"
+    sshPort: 9022
+    user: "root"
+    role: "master"
+    sshKeyPath: ~/.ssh/id_rsa_travis
+  - address: "127.0.0.1"
+    sshPort: 9023
+    user: "root"
+    role: "worker"
+    sshKeyPath: ~/.ssh/id_rsa_travis
+  - address: "127.0.0.1"
+    sshPort: 9024
+    user: "root"
+    role: "worker"
+    sshKeyPath: ~/.ssh/id_rsa_travis
+manifests:
+  - ./e2e/manifests/*.yaml
diff --git a/e2e/footloose.yaml b/e2e/footloose.yaml
@@ -0,0 +1,34 @@
+cluster:
+  name: trieres
+  privateKey: ~/.ssh/id_rsa_travis
+machines:
+- count: 1
+  backend: docker
+  spec:
+    image: $FOOTLOOSE_IMAGE
+    name: master%d
+    privileged: true
+    volumes:
+    - type: volume
+      destination: /var/lib/rancher
+    - type: volume
+      destination: /var/lib/containerd
+    portMappings:
+    - containerPort: 22
+      hostPort: 9022
+    - containerPort: 6443
+      hostPort: 6443
+- count: 2
+  backend: docker
+  spec:
+    image: $FOOTLOOSE_IMAGE
+    name: worker%d
+    privileged: true
+    volumes:
+    - type: volume
+      destination: /var/lib/rancher
+    - type: volume
+      destination: /var/lib/containerd
+    portMappings:
+    - containerPort: 22
+      hostPort: 9022
diff --git a/e2e/manifests/redis.yaml b/e2e/manifests/redis.yaml
@@ -0,0 +1,7 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: redis
+  namespace: kube-system
+spec:
+  chart: stable/redis
diff --git a/e2e/travis.sh b/e2e/travis.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+# shellcheck disable=SC2039 disable=SC1091
+
+set -ue
+
+go build -o trieres main.go
+
+ssh-keygen -t rsa -f ~/.ssh/id_rsa_travis -N ""
+cat ~/.ssh/id_rsa_travis.pub > ~/.ssh/authorized_keys
+chmod 0600 ~/.ssh/authorized_keys
+
+envsubst < e2e/cluster.yml > cluster.yml
+envsubst < e2e/footloose.yaml > footloose.yaml
+
+curl -L https://github.com/weaveworks/footloose/releases/download/0.6.3/footloose-0.6.3-linux-x86_64 > ./footloose
+chmod +x ./footloose
+./footloose create
+./footloose ssh root@master0 -- 'apt-get install -y curl || yum install -y curl which openssh-clients'
+./footloose ssh root@worker0 -- 'apt-get install -y curl || yum install -y curl which openssh-clients'
+./footloose ssh root@worker1 -- 'apt-get install -y curl || yum install -y curl which openssh-clients'
+
+./trieres
+./trieres -v
+./trieres --debug up
+./trieres kubeconfig > kubeconfig.e2e
+
+export KUBECONFIG=./kubeconfig.e2e
+
+echo "==> Test with sonobuoy"
+curl -L https://github.com/vmware-tanzu/sonobuoy/releases/download/v0.18.0/sonobuoy_0.18.0_linux_amd64.tar.gz | tar xzv
+chmod +x ./sonobuoy
+sleep 30
+./sonobuoy run --mode quick --timeout 600 --wait
+
diff --git a/go.mod b/go.mod
@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	github.com/avast/retry-go v2.5.0+incompatible
+	github.com/bramvdbogaerde/go-scp v0.0.0-20200119201711-987556b8bdd7
 	github.com/go-playground/validator/v10 v10.1.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/sirupsen/logrus v1.4.2

diff --git a/go.sum b/go.sum
@@ -6,6 +6,8 @@ github.com/alexellis/k3sup v0.0.0-20200125142521-8e396b7faa78/go.mod h1:BZUF5ulP
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/avast/retry-go v2.5.0+incompatible h1:8SaFqliw34WeeaPs+GEtMMkiwEsC2S6+YyqnLqI55Ks=
 github.com/avast/retry-go v2.5.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
+github.com/bramvdbogaerde/go-scp v0.0.0-20200119201711-987556b8bdd7 h1:G24EOzrFCngJcgnDQgPWXTCBe3JP7lXE6n/Mnnn1yyM=
+github.com/bramvdbogaerde/go-scp v0.0.0-20200119201711-987556b8bdd7/go.mod h1:aiQFnN5G0MivefWD+J4Em1a+CDyu/UBEmbNP5+8Gtd4=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@@ -17,6 +19,7 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=

diff --git a/main.go b/main.go
@@ -17,6 +17,11 @@ var (
 func init() {
 	logrus.SetOutput(os.Stdout)
 	logrus.SetLevel(logrus.InfoLevel)
+
+	customFormatter := new(logrus.TextFormatter)
+	customFormatter.TimestampFormat = "2006-01-02 15:04:05"
+	customFormatter.FullTimestamp = true
+	logrus.SetFormatter(customFormatter)
 }
 
 func main() {

diff --git a/pkg/cluster/config.go b/pkg/cluster/config.go
@@ -11,9 +11,10 @@ import (
 
 // Config describes cluster.yml
 type Config struct {
-	Hosts   hosts.Hosts `validate:"required,dive,required,gt=0"`
-	Token   string
-	Version string
+	Hosts     hosts.Hosts `validate:"required,dive,required,gt=0"`
+	Token     string      `validate:"omitempty,gt=12"`
+	Manifests []string
+	Version   string
 }
 
 // FromYaml parses config from YAML

diff --git a/pkg/hosts/host.go b/pkg/hosts/host.go
@@ -5,8 +5,13 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"os"
 	"path"
+	"strconv"
+	"strings"
+	"time"
 
+	"github.com/bramvdbogaerde/go-scp"
 	"github.com/mitchellh/go-homedir"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/crypto/ssh"
@@ -19,22 +24,44 @@ type RemoteHost interface {
 
 // Config for host
 type Config struct {
-	Address    string   `yaml:"address" validate:"required,hostname|ip"`
-	User       string   `yaml:"user"`
-	SSHPort    int      `yaml:"sshPort" validate:"gt=0,lte=65535"`
-	SSHKeyPath string   `yaml:"sshKeyPath" validate:"file"`
-	Role       string   `yaml:"role" validate:"oneof=master worker"`
-	ExtraArgs  []string `yaml:"extraArgs"`
+	Address          string   `yaml:"address" validate:"required,hostname|ip"`
+	User             string   `yaml:"user"`
+	SSHPort          int      `yaml:"sshPort" validate:"gt=0,lte=65535"`
+	SSHKeyPath       string   `yaml:"sshKeyPath" validate:"file"`
+	Role             string   `yaml:"role" validate:"oneof=master worker"`
+	ExtraArgs        []string `yaml:"extraArgs"`
+	PrivateInterface string   `validate:"omitempty,gt=2"`
+}
+
+// HostMetadata resolved metadata for host
+type HostMetadata struct {
+	Hostname        string
+	InternalAddress string
 }
 
 // Host describes connectable host
 type Host struct {
 	Config
 	sshClient *ssh.Client
+	Metadata  *HostMetadata
 }
 
+// Hosts array of hosts
 type Hosts []*Host
 
+// FullAddress returns address and non-standard ssh port
+func (h *Host) FullAddress() string {
+	address := h.Address
+	if h.Metadata != nil {
+		address = h.Metadata.Hostname
+	}
+	if h.SSHPort != 22 {
+		address = fmt.Sprintf("%s:%s", address, strconv.Itoa(h.SSHPort))
+	}
+
+	return address
+}
+
 // Connect to the host
 func (h *Host) Connect() error {
 	key, err := ioutil.ReadFile(h.SSHKeyPath)
@@ -88,29 +115,46 @@ func (h *Host) Exec(cmd string) error {
 	outputScanner := bufio.NewScanner(multiReader)
 
 	for outputScanner.Scan() {
-		logrus.Infof("%s:  %s", h.Address, outputScanner.Text())
+		logrus.Debugf("%s:  %s", h.FullAddress(), outputScanner.Text())
 	}
 	if err := outputScanner.Err(); err != nil {
-		logrus.Errorf("%s:  %s", h.Address, err.Error())
+		logrus.Errorf("%s:  %s", h.FullAddress(), err.Error())
 	}
 
 	return nil
 }
 
-// Exec a command on the host and return output
-func (h *Host) ExecWithOutput(cmd string) ([]byte, error) {
+// ExecWithOutput execs a command on the host and return output
+func (h *Host) ExecWithOutput(cmd string) (string, error) {
 	session, err := h.sshClient.NewSession()
 	if err != nil {
-		return []byte{}, err
+		return "", err
 	}
 	defer session.Close()
 
 	output, err := session.CombinedOutput(cmd)
 	if err != nil {
-		return []byte{}, nil
+		return "", nil
+	}
+
+	return strings.TrimSpace(string(output)), nil
+}
+
+// CopyFile copies a local file to host
+func (h *Host) CopyFile(file os.File, remotePath string, permissions string) error {
+	session, err := h.sshClient.NewSession()
+	if err != nil {
+		return err
+	}
+	defer session.Close()
+
+	scpClient := scp.Client{
+		Session:      session,
+		Timeout:      time.Second * 60,
+		RemoteBinary: "scp",
 	}
 
-	return output, nil
+	return scpClient.CopyFromFile(file, remotePath, permissions)
 }
 
 // Disconnect from the host
@@ -122,21 +166,26 @@ func (h *Host) Disconnect() error {
 	return h.sshClient.Close()
 }
 
+// UnmarshalYAML unmarshals yaml
 func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	type rawConfig Config
 	homeDir, _ := homedir.Dir()
 	raw := rawConfig{
-		Address:    "127.0.0.1",
-		User:       "root",
-		SSHKeyPath: path.Join(homeDir, ".ssh", "id_rsa"),
-		SSHPort:    22,
-		Role:       "worker",
-		ExtraArgs:  []string{},
+		Address:          "127.0.0.1",
+		User:             "root",
+		SSHKeyPath:       path.Join(homeDir, ".ssh", "id_rsa"),
+		SSHPort:          22,
+		Role:             "worker",
+		ExtraArgs:        []string{},
+		PrivateInterface: "eth0",
 	}
 
 	if err := unmarshal(&raw); err != nil {
 		return err
 	}
+	if strings.HasPrefix(raw.SSHKeyPath, "~") {
+		raw.SSHKeyPath = path.Join(homeDir, raw.SSHKeyPath[2:])
+	}
 
 	*c = Config(raw)
 	return nil

diff --git a/pkg/phases/connect_phase.go b/pkg/phases/connect_phase.go
@@ -30,19 +30,19 @@ func (p *ConnectPhase) connectHost(host *hosts.Host, wg *sync.WaitGroup) error {
 	defer wg.Done()
 	err := retry.Do(
 		func() error {
-			logrus.Infof("%s: opening SSH connection", host.Address)
+			logrus.Infof("%s: opening SSH connection", host.FullAddress())
 			err := host.Connect()
 			if err != nil {
-				logrus.Errorf("%s: failed to connect -> %s", host.Address, err.Error())
+				logrus.Errorf("%s: failed to connect -> %s", host.FullAddress(), err.Error())
 			}
 			return err
 		},
 	)
 	if err != nil {
-		logrus.Errorf("%s: failed to open connection", host.Address)
+		logrus.Errorf("%s: failed to open connection", host.FullAddress())
 		return err
 	}
 
-	logrus.Printf("%s: SSH connection opened", host.Address)
+	logrus.Printf("%s: SSH connection opened", host.FullAddress())
 	return nil
 }