#!/bin/sh

source ./common.sh

source ./utils.sh

set -e

echo "Using Project: ${PROJECT}"

echo "Using Zone: ${ZONE}"

echo "Using Cluster Name: ${CLUSTER_NAME}"

echo "Using K8S Version: ${K8S_VERSION}"

export NUM_K8S_NODES=600

echo "Overriding num nodes to: $NUM_K8S_NODES"

check_gcloud_exists

# Create a Kubernetes cluster. This setup is designed for creating large clusters.

# Allocating a big CIDR is necessary for large clusters (>1008 Nodes)

# See more details https://stackoverflow.com/questions/42129327/gke-cluster-creation-fails-because-the-network-default-does-not-have-available

gcloud beta container clusters create \

--num-nodes $NUM_K8S_NODES \

--accelerator type=nvidia-tesla-k80,count=1 \

--machine-type n1-standard-2 \

--disk-size 20 \

--preemptible \

--cluster-ipv4-cidr=10.0.0.0/10 \

--zone=$ZONE \

--cluster-version=$K8S_VERSION \

--project=$PROJECT \

$CLUSTER_NAME

# Fetch its credentials so we can use kubectl locally

gcloud container clusters get-credentials $CLUSTER_NAME --project $PROJECT --zone $ZONE

create_gcs_bucket

create_service_account_key

# Import the credentials into the cluster as a secret

kubectl create secret generic ${SERVICE_ACCOUNT}-creds --from-file=service-account.json=${SERVICE_ACCOUNT_KEY_LOCATION}

echo "Initializing GPUs"

# Install the NVIDIA drivers on each of the nodes in the cluster that will have

# GPU workers.

kubectl apply -f gpu-provision-daemonset.yaml

# TODO(kashomon): How can I automate this?

echo "--------------------------------------------------------------"

echo "To check that GPUS have been initialized, run:"

echo "kubectl get no -w -o yaml | grep -E 'hostname:|nvidia.com/gpu'"