Repository navigation
Expand file tree
/
Copy pathminithor
More file actions
executable file
·3932 lines (3633 loc) · 144 KB
/
Copy pathminithor
File metadata and controls
executable file
·3932 lines (3633 loc) · 144 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env bash
# Single-quoted `sh -c` scripts are expanded inside pods (so secrets arrive over
# stdin instead of host command lines), not by this shell
# shellcheck disable=SC2016
# -E makes ERR traps fire inside functions (kctl and every helper is one)
set -Eeuo pipefail
########################################
# Shared utilities
########################################
OS="$(uname -s)"
ARCH="$(uname -m)"
DRIVER=""
CONTAINER_CMD=""
# In-cluster container registry, referenced by its Kubernetes service DNS name.
REGISTRY_HOST="registry.thorium.svc.cluster.local"
REGISTRY_PORT=5000
# Select which minikube profile (cluster) every minikube call targets. minikube
# reads MINIKUBE_PROFILE itself, so exporting it scopes all of its subcommands;
# the profile name is also the primary node's (and container's) name, with any
# extra nodes named "<profile>-m02", "<profile>-m03", ...
#
# set_profile [name]
set_profile() {
PROFILE="${1:-minikube}"
if ! [[ "$PROFILE" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]]; then
echo "ERROR: --profile must be a lowercase DNS label" >&2
exit 1
fi
export MINIKUBE_PROFILE="$PROFILE"
}
set_profile "${MINIKUBE_PROFILE:-}"
# The critical hosts and CIDR ranges that must never traverse an HTTP proxy so
# the minikube node and host can reach them directly: the node subnet (docker
# driver), the service and pod CIDRs, the control-plane host, loopback, and the
# cluster-internal registry/DNS suffixes (CoreDNS is pod-facing only, so the
# node's container runtime can't route these through a proxy). Leading-dot
# entries match subdomains in Go's httpproxy matcher used by the runtime. This
# is unioned with the host's own NO_PROXY by compute_no_proxy().
MINIKUBE_NO_PROXY="control-plane.minikube.internal,192.168.49.0/24,10.96.0.0/12,10.244.0.0/16,localhost,127.0.0.1,${REGISTRY_HOST},.svc,.svc.cluster.local,cluster.local"
# Union the host's existing NO_PROXY/no_proxy with MINIKUBE_NO_PROXY, emitting
# each entry exactly once (order preserved). This is the single source of truth
# for the NO_PROXY handed to minikube and baked into the node, so install,
# start, and expose stay consistent. HTTP_PROXY/HTTPS_PROXY are always taken
# as-is from the host environment and never constructed here.
compute_no_proxy() {
# Every profile but the first gets its own container network, so add the
# current profile's node subnet once its network exists (docker/podman only)
local profile_subnet=""
case "$CONTAINER_CMD" in
docker)
profile_subnet=$(env -u DOCKER_HOST docker network inspect "$PROFILE" \
-f '{{range .IPAM.Config}}{{.Subnet}}{{end}}' 2>/dev/null) || true
;;
podman)
profile_subnet=$(podman network inspect "$PROFILE" \
-f '{{range .Subnets}}{{.Subnet}}{{end}}' 2>/dev/null) || true
;;
esac
printf '%s,%s,%s,%s' "${NO_PROXY:-}" "${no_proxy:-}" "$MINIKUBE_NO_PROXY" "$profile_subnet" \
| tr ',' '\n' \
| awk 'NF && !seen[$0]++' \
| paste -sd, -
}
# Persist the computed NO_PROXY into a shell rc file so ad-hoc kubectl/curl
# use in interactive shells (outside minithor, which fixes this up itself on
# every cluster-touching command via ensure_minikube_running/minikube_start)
# also bypasses the proxy for the minikube node. Manages its own marked block
# so re-running install on an existing rc file only replaces that block and
# never touches any other proxy settings the user has in the file.
persist_no_proxy_rc() {
local rc_file="$1"
local no_proxy_value="$2"
local marker_start="# >>> minithor NO_PROXY (managed, do not edit) >>>"
local marker_end="# <<< minithor NO_PROXY (managed, do not edit) <<<"
touch "$rc_file"
if grep -qF "$marker_start" "$rc_file" 2>/dev/null; then
# Delete the managed block along with the single blank separator line
# this function prepends, so repeated installs don't accumulate blank
# lines at the removal site.
awk -v s="$marker_start" -v e="$marker_end" '
$0 == s { if (buffered_blank) buffered_blank=0; in_block=1; next }
in_block { if ($0 == e) in_block=0; next }
$0 == "" { if (buffered_blank) print ""; buffered_blank=1; next }
{ if (buffered_blank) { print ""; buffered_blank=0 }; print }
END { if (buffered_blank) print "" }
' "$rc_file" > "${rc_file}.tmp"
# Write back through the existing file rather than mv so a symlinked rc
# file (e.g. managed by a dotfile manager) stays a symlink
cat "${rc_file}.tmp" > "$rc_file"
rm -f "${rc_file}.tmp"
fi
{
echo ""
echo "$marker_start"
echo "# Required for minikube (192.168.49.0/24 etc.) to bypass the proxy."
echo "export NO_PROXY=\"$no_proxy_value\""
echo "export no_proxy=\"\$NO_PROXY\""
echo "$marker_end"
} >> "$rc_file"
}
kctl() { minikube kubectl -- "$@"; }
# Print the name of every node in the current minikube profile, one per line
list_minikube_nodes() { minikube node list 2>/dev/null | awk 'NF {print $1}'; }
# Print the profile's node names as a YAML flow sequence for the ThoriumCluster
# scaler node list, e.g. ["minikube","minikube-m02"]
k8s_nodes_yaml() {
local nodes
nodes=$(kctl get nodes -o jsonpath='{.items[*].metadata.name}')
# node names never contain whitespace, so splitting gives one name per item
# shellcheck disable=SC2086
printf '[%s]' "$(printf '"%s",' $nodes | sed 's/,$//')"
}
log() { echo -e "\n====== $1 ======\n"; }
########################################
# Port-forward + loopback alias helpers (shared)
#
# Used by `expose`, `expose --stop`, `cleanup`, and `minikube delete` so that
# dev port-forwards and the loopback aliases they create are always torn down
# together.
########################################
# Where expose writes its logs and tracks the loopback aliases it created. This
# is per-user and private: the alias list is later fed to `sudo ip addr del`, so
# it must not live in a shared location another user could write to.
MINITHOR_STATE_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/minithor"
EXPOSE_LOG_DIR="${MINITHOR_STATE_DIR}/expose-logs"
LOOPBACK_STATE_FILE="${MINITHOR_STATE_DIR}/loopback-aliases"
CURRENT_UID="$(id -u)"
# Create the private state directories used by expose
ensure_state_dir() {
mkdir -p "$EXPOSE_LOG_DIR"
chmod 700 "$MINITHOR_STATE_DIR" "$EXPOSE_LOG_DIR"
}
SCYLLA_CQL_PORT=9042
# Local API listen port written into a `--local` config (80 needs root)
LOCAL_API_PORT=8888
# Component versions, pinned to match megathor's defaults so what is tested
# here is what megathor deploys
ECK_CHART_VERSION="2.16.0"
CERT_MANAGER_CHART_VERSION="v1.18.2"
# Pinned: newer operators use curl in probes/hooks, which the pinned
# scylla-manager-agent image lacks
SCYLLA_OPERATOR_CHART_VERSION="1.21.1"
QUICKWIT_CHART_VERSION="0.8.17"
REDIS_IMAGE="docker.io/redis:7.4.11"
JAEGER_IMAGE="docker.io/jaegertracing/jaeger-query:1.76.0"
# Pinned because `weed mini` defaults may change between SeaweedFS releases
SEAWEEDFS_IMAGE="docker.io/chrislusf/seaweedfs:4.48"
THORIUM_REGISTRY="ghcr.io/cisagov/thorium/infrastructure/thorium"
# The cluster-wide operators every instance shares
ELASTIC_OPERATOR_NAMESPACE="elastic-system"
SCYLLA_OPERATOR_NAMESPACE="scylla-operator"
# The optional in-cluster registry only exists for the default instance, since
# the node's container runtime trusts REGISTRY_HOST from cluster creation
REGISTRY_NAMESPACE="thorium"
# Select which Thorium instance the namespace and service DNS variables below
# describe. The default instance (no name) uses the plain namespaces (thorium,
# redis, ...); a named instance prefixes each with "<name>-" so several Thorium
# deployments can share the cluster for deployment testing only, since the scaler
# does not support sharing a cluster with another instance. The cluster-wide
# operators (ECK, Scylla, cert-manager, Kubegres, and the Thorium operator) are
# shared by all instances.
#
# set_instance [name]
set_instance() {
INSTANCE="${1:-}"
if [ -n "$INSTANCE" ] && ! [[ "$INSTANCE" =~ ^[a-z0-9]([-a-z0-9]{0,30}[a-z0-9])?$ ]]; then
echo "ERROR: --instance must be a lowercase DNS label of at most 32 characters" >&2
exit 1
fi
local prefix="${INSTANCE:+$INSTANCE-}"
THORIUM_NAMESPACE="${prefix}thorium"
REDIS_NAMESPACE="${prefix}redis"
SCYLLA_NAMESPACE="${prefix}scylla"
QUICKWIT_NAMESPACE="${prefix}quickwit"
SEAWEEDFS_NAMESPACE="${prefix}seaweedfs"
JAEGER_NAMESPACE="${prefix}jaeger"
# where deploy keeps the credentials it generated (see load_credentials)
CREDENTIALS_NAMESPACE="${prefix}minithor"
# the default instance keeps Elasticsearch alongside the ECK operator
if [ -n "$INSTANCE" ]; then
ELASTIC_NAMESPACE="${prefix}elastic"
else
ELASTIC_NAMESPACE="$ELASTIC_OPERATOR_NAMESPACE"
fi
# In-cluster service DNS names baked into the thorium secret. Also used by
# `get-config --local` to rewrite the config for host access via exposed ports.
ELASTIC_CLUSTER_HOST="elastic-es-http.${ELASTIC_NAMESPACE}.svc.cluster.local"
REDIS_CLUSTER_HOST="redis.${REDIS_NAMESPACE}.svc.cluster.local"
SEAWEEDFS_CLUSTER_HOST="seaweedfs.${SEAWEEDFS_NAMESPACE}.svc.cluster.local"
SCYLLA_CLUSTER_HOST="scylla-client.${SCYLLA_NAMESPACE}.svc.cluster.local"
POSTGRES_CLUSTER_HOST="postgres.${QUICKWIT_NAMESPACE}.svc.cluster.local"
QUICKWIT_INDEXER_HOST="quickwit-indexer.${QUICKWIT_NAMESPACE}.svc.cluster.local"
QUICKWIT_SEARCHER_HOST="quickwit-searcher.${QUICKWIT_NAMESPACE}.svc.cluster.local"
# each instance binds its own "thorium" service account (used by its scaler)
THORIUM_BINDING="thorium-operator-binding${INSTANCE:+-$INSTANCE}"
# nginx routes on host, so each instance's ingress needs its own
INGRESS_HOST="${INSTANCE:+$INSTANCE.}localhost"
# get-config writes ~/thorium.yml for the default instance, ~/thorium-<name>.yml otherwise
CONFIG_NAME="thorium${INSTANCE:+-$INSTANCE}"
}
set_instance "${MINITHOR_INSTANCE:-}"
# List the minikube kubectl port-forward processes on this host as
# "<uid> <pid> <args...>". The `[p]ort` trick keeps grep from matching itself;
# `.*svc/` tolerates an interspersed `--address` flag (used for the Scylla
# forwards). The uid is used for ownership because procps truncates long user
# names. `-ww` stops BSD/macOS ps from truncating args to the terminal width.
find_forwards() {
ps -A -ww -o uid= -o pid= -o args= 2>/dev/null \
| grep 'minikube kubectl.*[p]ort-forward.*svc/' \
|| true
}
# List only the port-forwards owned by the invoking user
find_own_forwards() { find_forwards | awk -v u="$CURRENT_UID" '$1 == u'; }
parse_service() { echo "$1" | sed -n 's/.*svc\/\([^ ]*\).*/\1/p'; }
parse_ports() { echo "$1" | awk '{print $NF}'; }
# Print a user name for a uid, falling back to the raw uid
uid_name() { id -nu "$1" 2>/dev/null || echo "uid $1"; }
# Stop a process started by this user along with its children (the minikube
# wrapper spawns the real kubectl as a child)
kill_own_process() {
local pid=$1
pkill -P "$pid" 2>/dev/null || true
kill "$pid" 2>/dev/null || true
}
# Is a port-forward to this service already running? Services are matched with
# their namespace since every instance uses the same service names.
#
# is_forward_alive <service> <namespace>
is_forward_alive() { find_forwards | grep -qE "svc/$1 -n $2( |$)"; }
parse_namespace() { echo "$1" | sed -n 's/.* -n \([^ ]*\).*/\1/p'; }
stop_all_forwards() {
local found=false line uid pid service others
while IFS= read -r line; do
[ -n "$line" ] || continue
found=true
read -r uid pid _ <<< "$line"
service=$(parse_service "$line")
kill_own_process "$pid"
echo "Stopped $service (PID $pid)"
done < <(find_own_forwards)
if [ "$found" = false ]; then
echo "No port-forwards running"
fi
# Other users' forwards are theirs to manage; only report them
others=$(find_forwards | awk -v u="$CURRENT_UID" '$1 != u' | wc -l | tr -d ' ')
if [ "$others" -gt 0 ]; then
echo "Left $others port-forward(s) owned by other users running"
fi
}
# Stop this user's port-forwards into the selected instance's namespaces and
# remove the Scylla loopback aliases they used, leaving other instances' alone
stop_instance_forwards() {
local line pid ns service
local namespaces=" $THORIUM_NAMESPACE $ELASTIC_NAMESPACE $REDIS_NAMESPACE $SEAWEEDFS_NAMESPACE $SCYLLA_NAMESPACE "
while IFS= read -r line; do
[ -n "$line" ] || continue
ns=$(parse_namespace "$line")
[[ "$namespaces" == *" $ns "* ]] || continue
read -r _ pid _ <<< "$line"
service=$(parse_service "$line")
kill_own_process "$pid"
echo "Stopped $service ($ns, PID $pid)"
done < <(find_own_forwards)
# the instance's Scylla nodes advertise the addresses its aliases were made for
local svc ip
while read -r svc ip; do
[ -n "$ip" ] || continue
loopback_alias_del "$ip"
if [ -f "$LOOPBACK_STATE_FILE" ]; then
grep -vxF "$ip" "$LOOPBACK_STATE_FILE" > "${LOOPBACK_STATE_FILE}.tmp" || true
mv "${LOOPBACK_STATE_FILE}.tmp" "$LOOPBACK_STATE_FILE"
fi
done <<< "$(discover_scylla_nodes)"
}
show_status() {
local found=false line uid pid service ports
while IFS= read -r line; do
[ -n "$line" ] || continue
found=true
read -r uid pid _ <<< "$line"
service=$(parse_service "$line")
ports=$(parse_ports "$line")
echo " $service ($(parse_namespace "$line")): running as $(uid_name "$uid") (PID $pid) -> $ports"
done < <(find_forwards)
if [ "$found" = false ]; then
echo " No port-forwards running"
fi
}
# Start a single port-forward. Accepts an optional leading `--address <ip>` to
# bind the forward to a specific local IP (used for Scylla, see forward_scylla_dev).
#
# start_forward [--address <ip>] <name> <namespace> <service> <ports...>
start_forward() {
local bind_addr=""
if [ "${1:-}" = "--address" ]; then
bind_addr="$2"
shift 2
fi
local name=$1 namespace=$2 service=$3
shift 3
local ports=("$@")
if is_forward_alive "$service" "$namespace"; then
local existing uid pid
existing=$(find_forwards | grep -E "svc/$service -n $namespace( |$)" | head -1)
read -r uid pid _ <<< "$existing"
echo " $name: already running as $(uid_name "$uid") (PID $pid), skipping"
return 0
fi
if ! kctl get svc "$service" -n "$namespace" &>/dev/null; then
echo " $name: SKIPPED (service $service not found in namespace $namespace)"
return 1
fi
# Find any process holding the local port. When bound to a specific address,
# scope the check to that address so sibling forwards on the same port (e.g.
# multiple Scylla nodes on :9042) aren't matched.
local local_port="${ports[0]%%:*}"
local holder_pid=""
if command -v ss &>/dev/null; then
if [ -n "$bind_addr" ]; then
holder_pid=$(ss -tlnp "src ${bind_addr}:${local_port}" 2>/dev/null | sed -n 's/.*pid=\([0-9]*\).*/\1/p' | head -1 || true)
else
holder_pid=$(ss -tlnp "sport = :$local_port" 2>/dev/null | sed -n 's/.*pid=\([0-9]*\).*/\1/p' | head -1 || true)
fi
elif [ -n "$bind_addr" ]; then
holder_pid=$(lsof -nP -ti "TCP@${bind_addr}:${local_port}" -sTCP:LISTEN 2>/dev/null | head -1 || true)
else
holder_pid=$(lsof -nP -ti "TCP:$local_port" -sTCP:LISTEN 2>/dev/null | head -1 || true)
fi
if [ -n "$holder_pid" ]; then
# Only a stale port-forward of our own is safe to replace; anything else
# (a local Redis, another user's forward, ...) is left alone
local holder_uid holder_args
holder_uid=$(ps -o uid= -p "$holder_pid" 2>/dev/null | tr -d ' ' || true)
holder_args=$(ps -ww -o args= -p "$holder_pid" 2>/dev/null || true)
if [ "$holder_uid" = "$CURRENT_UID" ] && [[ "$holder_args" == *port-forward* ]]; then
echo " $name: replacing stale port-forward on port ${bind_addr:+$bind_addr:}$local_port (PID $holder_pid)"
kill_own_process "$holder_pid"
sleep 1
else
echo " $name: FAILED (port ${bind_addr:+$bind_addr:}$local_port is in use by PID $holder_pid, owner $(uid_name "${holder_uid:-?}"))"
return 1
fi
fi
local addr_flag=()
[ -n "$bind_addr" ] && addr_flag=(--address "$bind_addr")
local log_file="$EXPOSE_LOG_DIR/$name.log"
kctl port-forward ${addr_flag[@]+"${addr_flag[@]}"} "svc/$service" -n "$namespace" "${ports[@]}" >/dev/null 2>"$log_file" &
local pid=$!
sleep 1
if is_forward_alive "$service" "$namespace"; then
echo " $name: forwarding ${bind_addr:+$bind_addr }${ports[*]} (PID $pid)"
else
echo " $name: FAILED to start (port-forward exited immediately)"
if [ -s "$log_file" ]; then
echo " $name: log: $(cat "$log_file")"
fi
return 1
fi
}
# Is the given IP currently configured as a loopback alias?
loopback_alias_present() {
local ip=$1
case "$OS" in
Linux) ip addr show dev lo 2>/dev/null | grep -qw "$ip" ;;
Darwin) ifconfig lo0 2>/dev/null | grep -qw "$ip" ;;
*) return 1 ;;
esac
}
# Add an IP as a loopback alias so a port-forward can bind to it, recording it
# in the state file so teardown knows to remove it. Idempotent.
loopback_alias_add() {
local ip=$1
ensure_state_dir
if ! { [ -f "$LOOPBACK_STATE_FILE" ] && grep -qxF "$ip" "$LOOPBACK_STATE_FILE"; }; then
echo "$ip" >> "$LOOPBACK_STATE_FILE"
fi
if loopback_alias_present "$ip"; then
return 0
fi
case "$OS" in
Linux) sudo ip addr add "$ip/32" dev lo ;;
Darwin) sudo ifconfig lo0 alias "$ip" 255.255.255.255 ;;
*) echo " WARNING: don't know how to add a loopback alias on $OS" >&2; return 1 ;;
esac
}
# Remove a single loopback alias if present.
loopback_alias_del() {
local ip=$1
if loopback_alias_present "$ip"; then
case "$OS" in
Linux) sudo ip addr del "$ip/32" dev lo 2>/dev/null || true ;;
Darwin) sudo ifconfig lo0 -alias "$ip" 2>/dev/null || true ;;
esac
echo " Removed loopback alias $ip"
fi
}
# Remove every loopback alias expose created and clear the state file.
remove_all_loopback_aliases() {
[ -f "$LOOPBACK_STATE_FILE" ] || return 0
local ip
while IFS= read -r ip; do
[ -n "$ip" ] || continue
loopback_alias_del "$ip"
done < "$LOOPBACK_STATE_FILE"
rm -f "$LOOPBACK_STATE_FILE"
}
# Discover Thorium's Scylla nodes and the address each advertises to clients
# (its broadcast_rpc_address, which the Scylla driver re-dials after the initial
# connection). With scylla-operator each node gets a per-pod ClusterIP "member"
# service whose name matches the pod and whose ClusterIP is that advertised
# address. Prints "<member-service-name> <advertised-ip>" per node.
discover_scylla_nodes() {
local pods pod ip
pods=$(kctl get pods -n "$SCYLLA_NAMESPACE" \
-l app.kubernetes.io/name=scylla \
-o jsonpath='{.items[*].metadata.name}' 2>/dev/null) || true
for pod in $pods; do
ip=$(kctl get svc "$pod" -n "$SCYLLA_NAMESPACE" \
-o jsonpath='{.spec.clusterIP}' 2>/dev/null) || true
if [ -n "$ip" ] && [ "$ip" != "None" ]; then
echo "$pod $ip"
fi
done
}
# Start a port-forward without aborting the caller on failure, counting the
# failure in EXPOSE_FAILURES so expose can report it and exit non-zero at the end
run_forward() {
start_forward "$@" || EXPOSE_FAILURES=$((EXPOSE_FAILURES + 1))
}
# Set up dev port-forwards for Scylla. A plain localhost forward does NOT work:
# the driver connects to the seed, learns each node's advertised
# broadcast_rpc_address (the per-pod ClusterIP) and re-dials *that* address,
# which a localhost tunnel can't satisfy. So for each node we add its advertised
# IP as a loopback alias and bind the forward to that exact IP, so both the seed
# connection and the driver's re-dial land on the forward.
forward_scylla_dev() {
local nodes
nodes=$(discover_scylla_nodes)
if [ -z "$nodes" ]; then
echo " scylla: SKIPPED (no Scylla nodes found in namespace $SCYLLA_NAMESPACE)"
return 1
fi
local svc ip
while read -r svc ip; do
[ -n "$svc" ] || continue
if ! loopback_alias_add "$ip"; then
echo " scylla[$svc]: FAILED to add loopback alias $ip"
EXPOSE_FAILURES=$((EXPOSE_FAILURES + 1))
continue
fi
run_forward --address "$ip" "${INSTANCE:+$INSTANCE-}scylla-$svc" "$SCYLLA_NAMESPACE" "$svc" \
"$SCYLLA_CQL_PORT:$SCYLLA_CQL_PORT"
done <<< "$nodes"
}
generate_password() {
openssl rand -base64 32 | tr -d '/+=' | head -c 32
}
# Where deploy keeps the backend credentials it generated, so re-running deploy
# reuses them instead of rotating passwords the stateful services already hold.
# `cleanup` deletes this namespace along with the data, so a fresh deploy
# starts with fresh credentials.
CREDENTIALS_SECRET="minithor-credentials"
# Print a short, stable hash of a string (used as a pod annotation so a changed
# credential rolls the pods that read it)
config_hash() {
if command -v sha256sum >/dev/null 2>&1; then
printf '%s' "$1" | sha256sum | cut -c1-16
else
printf '%s' "$1" | shasum -a 256 | cut -c1-16
fi
}
# Escape a string for embedding inside a JSON string literal
json_escape() { printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g'; }
# Print one decoded key of a secret, or nothing if the secret or key is missing
#
# read_secret_key <namespace> <secret> <key>
read_secret_key() {
kctl get secret "$2" -n "$1" \
-o go-template="{{with index .data \"$3\"}}{{. | base64decode}}{{end}}" 2>/dev/null || true
}
# Print a credential from a deployment made before credentials were saved, read
# back from the resources that hold it, or nothing when there is none
legacy_credential() {
local jsonpath=""
case "$1" in
redis-password) jsonpath='{.items[0].spec.config.redis.password}' ;;
scylla-password) jsonpath='{.items[0].spec.config.scylla.auth.password}' ;;
elastic-password) jsonpath='{.items[0].spec.config.elastic.password}' ;;
s3-access-key) jsonpath='{.items[0].spec.config.thorium.s3.access_key}' ;;
s3-secret-key) jsonpath='{.items[0].spec.config.thorium.s3.secret_token}' ;;
thorium-secret-key) jsonpath='{.items[0].spec.config.thorium.secret_key}' ;;
postgres-password)
read_secret_key "$QUICKWIT_NAMESPACE" postgres-cluster-auth superUserPassword
return 0
;;
esac
[ -n "$jsonpath" ] || return 0
kctl get thoriumclusters -n "$THORIUM_NAMESPACE" -o jsonpath="$jsonpath" 2>/dev/null || true
}
# Load every backend credential into the caller's variables: the saved value if
# there is one, else the value an existing deployment already uses, else a new
# random one. Values are embedded unquoted in YAML, SQL, and sed expressions, so
# anything outside a conservative character set is rejected.
load_credentials() {
local pair key var value
for pair in redis-password:REDIS_PASS postgres-password:PG_PASS \
scylla-password:SCYLLA_PASS elastic-password:ES_PASS \
s3-access-key:S3_AK s3-secret-key:S3_SK thorium-secret-key:THORIUM_SECRET; do
key="${pair%%:*}"
var="${pair#*:}"
value=$(read_secret_key "$CREDENTIALS_NAMESPACE" "$CREDENTIALS_SECRET" "$key")
if [ -z "$value" ]; then
value=$(legacy_credential "$key")
if [ -n "$value" ]; then
echo " $key: reusing the value from the existing deployment"
fi
fi
if [ -z "$value" ]; then
value=$(generate_password)
fi
if ! [[ "$value" =~ ^[A-Za-z0-9._~-]+$ ]]; then
echo "ERROR: stored $key contains unsupported characters; run 'minithor cleanup --confirm' for fresh credentials" >&2
return 1
fi
printf -v "$var" '%s' "$value"
done
}
# Persist the credentials in the caller's variables (plus the admin user when a
# random admin password is in use) to the credentials secret. The manifest is
# piped over stdin so no secret appears on a command line.
#
# save_credentials [admin_user admin_password]
save_credentials() {
local admin_user="${1:-}" admin_pass="${2:-}"
local admin_lines=""
if [ -n "$admin_user" ]; then
admin_lines=" admin-user: \"$(json_escape "$admin_user")\"
admin-password: \"$(json_escape "$admin_pass")\""
fi
kctl apply -f - >/dev/null <<EOF
---
apiVersion: v1
kind: Namespace
metadata:
name: $CREDENTIALS_NAMESPACE
---
apiVersion: v1
kind: Secret
metadata:
name: $CREDENTIALS_SECRET
namespace: $CREDENTIALS_NAMESPACE
type: Opaque
stringData:
redis-password: "$REDIS_PASS"
postgres-password: "$PG_PASS"
scylla-password: "$SCYLLA_PASS"
elastic-password: "$ES_PASS"
s3-access-key: "$S3_AK"
s3-secret-key: "$S3_SK"
thorium-secret-key: "$THORIUM_SECRET"
${admin_lines}
EOF
}
# Substitute the \${NAME} credential, namespace, service host, and node list
# placeholders in a user-supplied cluster config with this deploy's values, so
# one file works for any instance and node count. \${K8S_NODES} becomes a YAML
# list of every node in the cluster. Only these names are replaced, so any
# other \$ in the file is left as written.
#
# render_cluster_config <source> <destination>
render_cluster_config() {
sed \
-e "s|\\\${REDIS_PASS}|$REDIS_PASS|g" \
-e "s|\\\${SCYLLA_PASS}|$SCYLLA_PASS|g" \
-e "s|\\\${ES_PASS}|$ES_PASS|g" \
-e "s|\\\${S3_AK}|$S3_AK|g" \
-e "s|\\\${S3_SK}|$S3_SK|g" \
-e "s|\\\${THORIUM_SECRET}|$THORIUM_SECRET|g" \
-e "s|\\\${THORIUM_NAMESPACE}|$THORIUM_NAMESPACE|g" \
-e "s|\\\${REDIS_HOST}|$REDIS_CLUSTER_HOST|g" \
-e "s|\\\${SCYLLA_HOST}|$SCYLLA_CLUSTER_HOST|g" \
-e "s|\\\${ELASTIC_HOST}|$ELASTIC_CLUSTER_HOST|g" \
-e "s|\\\${S3_HOST}|$SEAWEEDFS_CLUSTER_HOST|g" \
-e "s|\\\${QUICKWIT_INDEXER_HOST}|$QUICKWIT_INDEXER_HOST|g" \
-e "s|\\\${K8S_NODES}|$(k8s_nodes_yaml)|g" \
"$1" > "$2"
}
has_podman() {
command -v podman >/dev/null 2>&1 &&
podman info >/dev/null 2>&1
}
has_docker() {
command -v docker >/dev/null 2>&1 &&
env -u DOCKER_HOST docker info >/dev/null 2>&1
}
has_kvm2() {
[ "$OS" = "Linux" ] || return 1
command -v virsh >/dev/null 2>&1 &&
[ -e /dev/kvm ] &&
(
systemctl is-active --quiet libvirtd 2>/dev/null ||
systemctl is-active --quiet virtqemud 2>/dev/null
)
}
select_driver() {
if has_podman; then
DRIVER="podman"
CONTAINER_CMD="podman"
elif has_docker; then
DRIVER="docker"
CONTAINER_CMD="docker"
elif has_kvm2; then
DRIVER="kvm2"
CONTAINER_CMD=""
else
echo "No supported and working minikube driver found."
return 1
fi
}
# Set DRIVER/CONTAINER_CMD from the existing minikube profile, which is the
# driver the cluster was actually created with. Returns 1 when no profile exists.
detect_profile_driver() {
local profile_driver
profile_driver=$(minikube profile list -o json 2>/dev/null \
| grep -o '"Driver": *"[^"]*"' | head -1 | sed 's/.*"\([^"]*\)"$/\1/') || true
[ -n "$profile_driver" ] || return 1
DRIVER="$profile_driver"
case "$DRIVER" in
docker|podman) CONTAINER_CMD="$DRIVER" ;;
*) CONTAINER_CMD="" ;;
esac
}
# minikube_start [extra minikube start args...]
#
# Extra args (e.g. --nodes/--cpus/--memory) only take effect when the profile is
# first created; restarting an existing profile brings back all of its nodes.
minikube_start() {
# Trust the in-cluster registry as insecure (HTTP) at cluster-creation time.
# The node's container runtime defaults to HTTPS for every registry; without
# this, pulling from the HTTP-only internal registry fails the TLS handshake
# with "unexpected EOF" -> ErrImagePull/ImagePullBackOff. The flag must be set
# when the node is created since it bakes into the runtime's daemon config.
local insecure_registry="${REGISTRY_HOST}:${REGISTRY_PORT}"
# Hand minikube a clean, deduped NO_PROXY (host entries + critical cluster
# hosts/CIDRs). HTTP_PROXY/HTTPS_PROXY are inherited from the host env as-is.
# Note: minikube's own propagation of these into the node runtime is
# unreliable (it can bake empty values), so setup_node_pull_env() re-asserts
# the proxy directly on the node after start — this just keeps what minikube
# sees consistent.
NO_PROXY="$(compute_no_proxy)"
export NO_PROXY
export no_proxy="$NO_PROXY"
if [ "$DRIVER" = "docker" ]; then
env -u DOCKER_HOST minikube start --cni calico --driver="$DRIVER" --insecure-registry="$insecure_registry" "$@"
elif [ "$DRIVER" = "podman" ]; then
minikube start --cni calico --container-runtime=containerd --driver="$DRIVER" --insecure-registry="$insecure_registry" "$@"
else
minikube start --cni calico --driver="$DRIVER" --insecure-registry="$insecure_registry" "$@"
fi
}
fix_container_config() {
if [ -n "$CONTAINER_CMD" ]; then
local api_port cluster
# the primary node's container is named after the profile
api_port="$($CONTAINER_CMD port "$PROFILE" 8443/tcp | awk -F: '{print $2}')"
cluster="$(kctl config view --minify -o jsonpath='{.contexts[0].context.cluster}')"
kctl config set-cluster "$cluster" --server="https://127.0.0.1:${api_port}"
fi
}
ensure_minikube_running() {
# Every subcommand that reaches the cluster calls this first, so fixing up
# NO_PROXY here (rather than in each cmd_* function) guarantees local
# kubectl/port-forward traffic to the node bypasses any configured HTTP
# proxy, regardless of what the invoking shell happened to have set.
NO_PROXY="$(compute_no_proxy)"
export NO_PROXY
export no_proxy="$NO_PROXY"
local status_output
status_output="$(minikube status 2>/dev/null)" || true
local host_status
# multi-node clusters list every node; the first block is the control plane
host_status="$(echo "$status_output" | awk '/^host:/ {print $2; exit}')" || true
if [[ "$host_status" != "Running" ]]; then
echo "ERROR: minikube is not running. Start it with: minithor start" >&2
exit 1
fi
# Auto-fix stale kubeconfig (e.g. after host reboot changed the container port mapping)
local kube_status
kube_status="$(echo "$status_output" | awk '/^kubeconfig:/ {print $2; exit}')" || true
if [[ "$kube_status" != "Configured" ]]; then
echo "Kubeconfig is stale, updating context..." >&2
minikube update-context >/dev/null 2>&1
# update-context points container drivers at the node's internal IP,
# which isn't reachable from macOS hosts, so re-apply the published port
if detect_profile_driver; then
fix_container_config >/dev/null
fi
fi
}
wait_for_rollout() {
local resource=$1 namespace=$2 timeout=${3:-300}
kctl rollout status --watch --timeout="${timeout}s" "$resource" -n "$namespace"
}
wait_for_resource() {
local resource_type=$1 resource_name=$2 namespace=$3 timeout=${4:-300}
local elapsed=0
local ns_flag=()
if [ -n "$namespace" ]; then
ns_flag=(-n "$namespace")
fi
echo "Waiting for $resource_type/$resource_name${namespace:+ in $namespace} (timeout: ${timeout}s)..."
until kctl get "$resource_type" "$resource_name" ${ns_flag[@]+"${ns_flag[@]}"} &>/dev/null; do
sleep 5
elapsed=$((elapsed + 5))
if [ "$elapsed" -ge "$timeout" ]; then
echo "ERROR: Timed out waiting for $resource_type/$resource_name${namespace:+ in $namespace}" >&2
return 1
fi
done
}
# Wait until at least one pod matching a selector exists and all matching pods
# are Ready. A bare `kubectl wait -l` fails immediately with "no matching
# resources" when the controller hasn't created the pods yet.
#
# wait_for_pods_ready <namespace> <selector> [timeout]
wait_for_pods_ready() {
local namespace=$1 selector=$2 timeout=${3:-300}
local elapsed=0
echo "Waiting for pods ($selector) in $namespace to be Ready (timeout: ${timeout}s)..."
until [ -n "$(kctl get pods -n "$namespace" -l "$selector" -o name 2>/dev/null)" ]; do
sleep 5
elapsed=$((elapsed + 5))
if [ "$elapsed" -ge "$timeout" ]; then
echo "ERROR: Timed out waiting for pods ($selector) in $namespace to be created" >&2
return 1
fi
done
kctl wait --for=condition=Ready pod -l "$selector" -n "$namespace" --timeout="$((timeout - elapsed))s"
}
# Wait for a Job to succeed, failing fast (with its logs) once the Job is marked
# Failed instead of waiting out the whole timeout
#
# wait_for_job <namespace> <job> [timeout]
wait_for_job() {
local namespace=$1 job=$2 timeout=${3:-300}
local elapsed=0 succeeded failed
while :; do
succeeded=$(kctl get job "$job" -n "$namespace" -o jsonpath='{.status.succeeded}' 2>/dev/null || true)
if [ "${succeeded:-0}" -ge 1 ]; then
return 0
fi
failed=$(kctl get job "$job" -n "$namespace" \
-o jsonpath='{.status.conditions[?(@.type=="Failed")].status}' 2>/dev/null || true)
if [ "$failed" = "True" ] || [ "$elapsed" -ge "$timeout" ]; then
if [ "$failed" = "True" ]; then
echo "ERROR: job/$job in $namespace failed; last logs:" >&2
else
echo "ERROR: Timed out after ${timeout}s waiting for job/$job in $namespace; last logs:" >&2
fi
kctl logs -n "$namespace" "job/$job" --tail=50 >&2 || true
return 1
fi
sleep 5
elapsed=$((elapsed + 5))
done
}
wait_for_port_forward() {
local port=$1 max_attempts=${2:-20}
for attempt in $(seq 1 "$max_attempts"); do
if curl -sf -o /dev/null "http://localhost:$port" 2>/dev/null; then return 0; fi
sleep 1
done
echo "ERROR: Port-forward on port $port not ready after ${max_attempts}s" >&2
return 1
}
# Error-tolerant execution — uses FAILURES global set by the caller
run_step() {
if ! "$@"; then
FAILURES=$((FAILURES + 1))
echo "ERROR: Command failed (continuing): $*" >&2
fi
}
# Raise the kernel limits Scylla/Elastic need on the host or on minikube nodes
#
# tune_kernel_params [host|minikube|node:<name>]
#
# "minikube" tunes every node in the profile; "node:<name>" tunes one node.
tune_kernel_params() {
local target="${1:-host}"
# tune each node separately since every node has its own kernel settings
if [ "$target" = "minikube" ]; then
local node
for node in $(list_minikube_nodes); do
echo " Node $node:"
tune_kernel_params "node:$node"
done
return 0
fi
local node_name="${target#node:}"
# fs.aio-max-nr is kernel-wide and each Scylla node reserves a large share,
# so a second instance's Scylla fails to start at the 65536 default
local params=(
"fs.aio-max-nr=2097152"
"fs.inotify.max_user_instances=1024"
"fs.file-max=2097152"
"fs.nr_open=1048576"
"vm.max_map_count=262144"
)
local persisted=()
for param in "${params[@]}"; do
local key="${param%%=*}"
local desired="${param#*=}"
local current
if [ "$target" != "host" ]; then
current=$(minikube ssh -n "$node_name" -- /sbin/sysctl -n "$key" 2>/dev/null | tr -d '\r') || true
else
current=$(sysctl -n "$key" 2>/dev/null) || true
fi
if [ -z "$current" ]; then
echo " $key: could not read current value, skipping"
continue
fi
if [ "$current" -ge "$desired" ] 2>/dev/null; then
echo " $key = $current (already >= $desired)"
# persist the higher current value so a reboot never lowers it
persisted+=("$key=$current")
else
persisted+=("$param")
echo " $key: $current -> $desired"
if [ "$target" != "host" ]; then
minikube ssh -n "$node_name" -- sudo /sbin/sysctl -w "$param" >/dev/null 2>&1 || echo " WARNING: failed to set $key on node $node_name"
else
sudo sysctl -w "$param" >/dev/null 2>&1 || echo " WARNING: failed to set $key (may require root)"
fi
fi
done
if [ "$target" = "host" ] && [ "$OS" = "Linux" ] && [ "${#persisted[@]}" -gt 0 ]; then
local conf="/etc/sysctl.d/99-minithor.conf"
echo " Persisting to $conf"
printf '%s\n' "${persisted[@]}" | sudo tee "$conf" >/dev/null 2>&1 || echo " WARNING: failed to write $conf"
fi
}
# Map the in-cluster registry's service DNS name to its ClusterIP in every minikube
# node's /etc/hosts. Each node's container runtime pulls images using the node's
# resolver (not CoreDNS), so without this it cannot resolve REGISTRY_HOST and
# image pulls fail with "no such host". Idempotent; no-op when the registry isn't
# deployed. The docker driver regenerates /etc/hosts on node restart, so this is
# re-asserted on every `minithor start`.
ensure_registry_node_dns() {
local ip
ip="$(kctl get svc registry -n "$REGISTRY_NAMESPACE" -o jsonpath='{.spec.clusterIP}' 2>/dev/null)" || return 0
[ -n "$ip" ] || return 0
local node
for node in $(list_minikube_nodes); do
echo " Mapping ${REGISTRY_HOST} -> ${ip} on node $node"
# /etc/hosts is a bind-mount in the node; rewrite in place (rename-based edits
# like `sed -i` fail with "Device or resource busy").
minikube ssh -n "$node" -- "sudo cp /etc/hosts /tmp/minithor-hosts && \
grep -v ' ${REGISTRY_HOST}\$' /tmp/minithor-hosts | sudo tee /etc/hosts >/dev/null && \
echo '${ip} ${REGISTRY_HOST}' | sudo tee -a /etc/hosts >/dev/null" >/dev/null 2>&1 \
|| echo " WARNING: failed to set registry hosts entry on node $node"
done
}
# Make every minikube node able to pull images through the host's HTTP proxy
# and, behind a TLS-intercepting proxy, trust its CA. This is required because
# minikube's own proxy propagation into the node runtime is unreliable (it can
# bake empty HTTP_PROXY/HTTPS_PROXY), so the container runtime otherwise dials
# external registries (quay.io, ghcr.io, ...) directly and times out.
#
# setup_node_pull_env [certs_path] [force]
#
# certs_path (optional): a .crt file, or a directory of *.crt files, to install
# into the node's trust store. Gated — cert injection only happens when given.
# force (optional, "true"/"false", default false): when true, overwrite the CA
# and the runtime proxy drop-in on the node even if they already exist. By
# default an existing file is left in place so a manual fix isn't clobbered.
# Proxy config is driven entirely by the host's HTTP_PROXY/HTTPS_PROXY and is a
# no-op on unproxied hosts. The drop-in targets whichever runtime pulls images on
# the node: dockerd for the docker/kvm2 drivers' default runtime, containerd for
# podman (started with --container-runtime=containerd). Files are copied with
# `minikube cp` (piping through `minikube ssh` stdin hangs), which also keeps
# proxy URLs (which may embed credentials) off command lines and out of shell
# quoting. Driver-agnostic: everything goes through `minikube cp` / `minikube
# ssh`, so it works for docker, podman, and kvm2.
setup_node_pull_env() {
local node
for node in $(list_minikube_nodes); do
setup_one_node_pull_env "$node" "$@"
done
}
# Configure a single node's image pulls (see setup_node_pull_env)
#
# setup_one_node_pull_env <node> [certs_path] [force]
setup_one_node_pull_env() {
local node="$1"
local certs_path="${2:-}"
local force="${3:-false}"
local changed=false
# The node reports its runtime as e.g. "docker://28.1.1" or "containerd://1.7"
local runtime_service
runtime_service=$(kctl get node "$node" -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}' 2>/dev/null | cut -d: -f1) || true
case "$runtime_service" in
docker|containerd) ;;
*)
echo " WARNING: could not detect node $node's container runtime, assuming containerd" >&2
runtime_service=containerd
;;
esac
# --- Certs (gated on a caller-supplied path) ---
if [ -n "$certs_path" ]; then
local certs=()
if [ -f "$certs_path" ]; then
certs=("$certs_path")
elif [ -d "$certs_path" ]; then
local f
for f in "$certs_path"/*.crt; do
[ -e "$f" ] && certs+=("$f")
done
else
echo " WARNING: --certs path '$certs_path' is not a file or directory, skipping cert install" >&2
fi
if [ "${#certs[@]}" -eq 0 ]; then
echo " No .crt files found at '$certs_path', skipping cert install"
else
local cert base