[root@learning-1 memsql.namespace]# kubectl get all
NAME READY STATUS RESTARTS AGE
pod/node-sdb-cluster-aggregator-0 1/2 Running 3 (8m8s ago) 11m
pod/node-sdb-cluster-leaf-ag1-0 1/2 Running 0 11m
pod/node-sdb-cluster-leaf-ag1-1 0/2 Pending 0 11m
pod/node-sdb-cluster-leaf-ag2-0 0/2 Pending 0 30m
pod/node-sdb-cluster-leaf-ag2-1 0/2 Pending 0 30m
pod/node-sdb-cluster-master-0 1/2 Running 4 (7m34s ago) 14m
pod/sdb-operator-84bf4b74dd-jjd2m 1/1 Running 0 79m
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 82m
service/svc-sdb-cluster ClusterIP None <none> 3306/TCP 79m
service/svc-sdb-cluster-ddl LoadBalancer 10.103.65.200 192.168.10.0 3306:31624/TCP 79m
service/svc-sdb-cluster-dml LoadBalancer 10.107.201.204 192.168.10.1 3306:32060/TCP 30m
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/sdb-operator 1/1 1 1 79m
NAME DESIRED CURRENT READY AGE
replicaset.apps/sdb-operator-84bf4b74dd 1 1 1 79m
NAME READY AGE
statefulset.apps/node-sdb-cluster-aggregator 0/1 30m
statefulset.apps/node-sdb-cluster-leaf-ag1 0/2 30m
statefulset.apps/node-sdb-cluster-leaf-ag2 0/2 30m
statefulset.apps/node-sdb-cluster-master 0/1 30m
[root@learning-1 memsql.namespace]# kubectl get ep
NAME ENDPOINTS AGE
kubernetes 10.128.0.4:6443 83m
svc-sdb-cluster 10.244.95.89:3306,10.244.95.90:3306,10.244.95.91:3306 79m
svc-sdb-cluster-ddl 79m
svc-sdb-cluster-dml 30m
[root@learning-1 memsql.namespace]# kubectl describe pod/node-sdb-cluster-master-0
Name: node-sdb-cluster-master-0
Namespace: default
Priority: 0
Node: learning-2/10.138.0.5
Start Time: Sat, 05 Nov 2022 15:18:24 +0000
Labels: app.kubernetes.io/component=master
app.kubernetes.io/instance=sdb-cluster
app.kubernetes.io/name=memsql-cluster
controller-revision-hash=node-sdb-cluster-master-698c4b6f8
memsql.com/role-tier=aggregator
memsql.com/workspace=singlestore-central
optional=label
statefulset.kubernetes.io/pod-name=node-sdb-cluster-master-0
Annotations: cni.projectcalico.org/containerID: f1e2b176f7d5a8481fa5fb716fa21e3f6f471489b126c039796c3874aad591be
cni.projectcalico.org/podIP: 10.244.95.89/32
cni.projectcalico.org/podIPs: 10.244.95.89/32
hash.configmap.memsql.com/node-sdb-cluster-master: 7ac8c27d96325970b6a2eb8be1c4a1e6075e647345c5d10d7387bedc414249d3
hash.secret.memsql.com/sdb-cluster: 144b336661ab827af5e4cdb0eb030b60df028684349adc43a81ff342c71151d2
optional: annotation
prometheus.io/port: 9104
prometheus.io/scrape: true
Status: Running
IP: 10.244.95.89
IPs:
IP: 10.244.95.89
Controlled By: StatefulSet/node-sdb-cluster-master
Containers:
node:
Container ID: docker://49668fe66271abc769088c0ee8cc97597313b596ea63c318324425f8c11af4e7
Image: singlestore/node:alma-7.8.17-69cee1f1a3
Image ID: docker-pullable://singlestore/node@sha256:58cc920cc63552e6a07e38a3f39d6e7b2880dd792fe6c6db723444a699ec2681
Port: <none>
Host Port: <none>
State: Running
Started: Sat, 05 Nov 2022 15:25:40 +0000
Last State: Terminated
Reason: Completed
Exit Code: 0
Started: Sat, 05 Nov 2022 15:24:32 +0000
Finished: Sat, 05 Nov 2022 15:25:39 +0000
Ready: False
Restart Count: 4
Limits:
cpu: 1
memory: 1Gi
Requests:
cpu: 1
memory: 1Gi
Liveness: exec [/etc/memsql/scripts/liveness-probe] delay=10s timeout=10s period=10s #success=1 #failure=3
Readiness: exec [/etc/memsql/scripts/readiness-probe] delay=10s timeout=10s period=10s #success=1 #failure=3
Startup: exec [/etc/memsql/scripts/startup-probe] delay=0s timeout=120s period=5s #success=1 #failure=2147483647
Environment:
RELEASE_ID:
ROOT_PASSWORD: <set to the key 'ROOT_PASSWORD' in secret 'sdb-cluster'> Optional: false
PRE_START_SCRIPT: /etc/memsql/scripts/update-config-script
MALLOC_ARENA_MAX: 4
Mounts:
/etc/memsql/extra from additional-files (rw)
/etc/memsql/extra-secret from additional-secrets (rw)
/etc/memsql/scripts from scripts (rw)
/etc/memsql/share from global-additional-files (rw)
/var/lib/memsql from pv-storage (rw)
/var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-dbnwh (ro)
exporter:
Container ID: docker://1df49ee39a3fc8e90bf41ec887865d6d28d87f6b166ecfd7b625355402af8122
Image: singlestore/node:alma-7.8.17-69cee1f1a3
Image ID: docker-pullable://singlestore/node@sha256:58cc920cc63552e6a07e38a3f39d6e7b2880dd792fe6c6db723444a699ec2681
Port: 9104/TCP
Host Port: 0/TCP
Command:
/etc/memsql/scripts/exporter-startup-script
State: Running
Started: Sat, 05 Nov 2022 15:18:27 +0000
Ready: True
Restart Count: 0
Limits:
cpu: 100m
memory: 100Mi
Requests:
cpu: 100m
memory: 90Mi
Environment:
RELEASE_ID:
DATA_SOURCE_NAME: <set to the key 'DATA_SOURCE_NAME' in secret 'sdb-cluster'> Optional: false
Mounts:
/etc/memsql/extra from additional-files (rw)
/etc/memsql/extra-secret from additional-secrets (rw)
/etc/memsql/scripts from scripts (rw)
/etc/memsql/share from global-additional-files (rw)
/var/lib/memsql from pv-storage (rw)
/var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-dbnwh (ro)
Conditions:
Type Status
Initialized True
Ready False
ContainersReady False
PodScheduled True
Volumes:
pv-storage:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: pv-storage-node-sdb-cluster-master-0
ReadOnly: false
scripts:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: node-sdb-cluster-master
Optional: false
additional-files:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: sdb-cluster-additional-files
Optional: true
additional-secrets:
Type: Secret (a volume populated by a Secret)
SecretName: sdb-cluster-additional-secrets
Optional: true
global-additional-files:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: global-additional-files
Optional: true
kube-api-access-dbnwh:
Type: Projected (a volume that contains injected data from multiple sources)
TokenExpirationSeconds: 3607
ConfigMapName: kube-root-ca.crt
ConfigMapOptional: <nil>
DownwardAPI: true
QoS Class: Burstable
Node-Selectors: <none>
Tolerations: node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled 16m default-scheduler Successfully assigned default/node-sdb-cluster-master-0 to learning-2
Normal Pulled 16m kubelet Container image "singlestore/node:alma-7.8.17-69cee1f1a3" already present on machine
Normal Created 16m kubelet Created container node
Normal Started 16m kubelet Started container node
Normal Pulled 16m kubelet Container image "singlestore/node:alma-7.8.17-69cee1f1a3" already present on machine
Normal Created 16m kubelet Created container exporter
Normal Started 16m kubelet Started container exporter
Warning Unhealthy 66s (x81 over 15m) kubelet Readiness probe failed: Error 1045: Access denied for user 'root'@'localhost' (using password: YES)
2022-11-05T16:06:53.830Z INFO controller/configmaps_secrets.go:55 skipping reconcile request because cluster spec has not changed
2022-11-05T16:06:53.846Z INFO controller/configmaps_secrets.go:55 skipping reconcile request because cluster spec has not changed
2022-11-05T16:06:53.846Z INFO controller/controller.go:114 Reconciling MemSQL Cluster. {"Request.Namespace": "default", "Request.Name": "sdb-cluster"}
2022-11-05T16:06:53.847Z INFO memsql/metrics.go:58 can't find operator deployment, trying uncached client {"key": "default/operator-sdb-cluster"}
2022-11-05T16:06:53.943Z ERROR memsql/metrics.go:61 can't find operator deployment, metrics service will not be created {"error": "deployments.apps \"operator-sdb-cluster\" not found"}
freya/kube/memsql.NewMetricsServiceAction.func1
freya/kube/memsql/metrics.go:61
freya/kube/memsql.ComposeActions.func1
freya/kube/memsql/action.go:22
freya/kube/controller.(*Reconciler).Reconcile
freya/kube/controller/controller.go:296
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile
sigs.k8s.io/controller-runtime@v0.11.0/pkg/internal/controller/controller.go:114
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
sigs.k8s.io/controller-runtime@v0.11.0/pkg/internal/controller/controller.go:311
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
sigs.k8s.io/controller-runtime@v0.11.0/pkg/internal/controller/controller.go:266
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
sigs.k8s.io/controller-runtime@v0.11.0/pkg/internal/controller/controller.go:227
2022-11-05T16:06:54.232Z INFO memsql/nodes.go:153 Wait for first STS pod {"STS name": "node-sdb-cluster-master"}
2022-11-05T16:06:54.233Z INFO memsql/retry.go:39 WARN: Error {"Cause": "Wait for MA connection", "Retry after": "5s"}
2022-11-05T16:06:54.330Z INFO memsql/connection.go:24 Connect to the Master Aggregator
2022-11-05T16:06:54.409Z ERROR memsql/connection.go:31 Failed to connect to voting member {"index": 0, "error": "Error 1045: Access denied for user 'root'@'10.244.95.84' (using password: YES)"}
freya/kube/memsql.connectToMaster
freya/kube/memsql/connection.go:31
freya/kube/memsql.NewClusteringMasterAction.func1
freya/kube/memsql/clustering_master.go:30
freya/kube/memsql.ComposeActions.func1
freya/kube/memsql/action.go:22
freya/kube/controller.(*Reconciler).Reconcile
freya/kube/controller/controller.go:296
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile
sigs.k8s.io/controller-runtime@v0.11.0/pkg/internal/controller/controller.go:114
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
sigs.k8s.io/controller-runtime@v0.11.0/pkg/internal/controller/controller.go:311
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
sigs.k8s.io/controller-runtime@v0.11.0/pkg/internal/controller/controller.go:266
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
sigs.k8s.io/controller-runtime@v0.11.0/pkg/internal/controller/controller.go:227
2022-11-05T16:06:54.772Z ERROR controller/errors.go:95 Reconciler error {"will retry after": "10m0s", "error": "Error 1045: Access denied for user 'root'@'10.244.95.84' (using password: YES)"}
freya/kube/controller.(*ErrorManager).Update
freya/kube/controller/errors.go:95
freya/kube/controller.(*Reconciler).Reconcile
freya/kube/controller/controller.go:342
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile
sigs.k8s.io/controller-runtime@v0.11.0/pkg/internal/controller/controller.go:114
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
sigs.k8s.io/controller-runtime@v0.11.0/pkg/internal/controller/controller.go:311
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
sigs.k8s.io/controller-runtime@v0.11.0/pkg/internal/controller/controller.go:266
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
sigs.k8s.io/controller-runtime@v0.11.0/pkg/internal/controller/controller.go:227
Was anything changed/cycled after the initial deployment?
It looks like the operator lost the root password from the associated secret. I can see the Pods were cycled sooner than the statefulsets:
pod/node-sdb-cluster-aggregator-0 1/2 Running 3 (8m8s ago) 11m
pod/node-sdb-cluster-leaf-ag1-0 1/2 Running 0 11m
pod/node-sdb-cluster-leaf-ag1-1 0/2 Pending 0 11m
NAME READY AGE
statefulset.apps/node-sdb-cluster-aggregator 0/1 30m
statefulset.apps/node-sdb-cluster-leaf-ag1 0/2 30m
statefulset.apps/node-sdb-cluster-leaf-ag2 0/2 30m
statefulset.apps/node-sdb-cluster-master 0/1 30m
It appears the external load balancer is assigning internal IPs:
service/svc-sdb-cluster-ddl LoadBalancer 10.103.65.200 192.168.10.0 3306:31624/TCP 79m
Is this intended?
Can you go over your steps for deployment? I’m guessing something was missed during this process. It may be best to restart the deployment (make sure to clear out everything from the prior deployment).
You can also attempt restarting the statefulsets:
kubectl rollout restart statefulset
Have similar error. Cluster is operating, but in operator log there is error about “not found deployment” and I cannot find anywhere deployment “operator-singlestore-general-tier-memsql”
Error:
2022-12-16T16:07:44.878Z INFO memsql/connection.go:24 Connect to the Master Aggregator
2022-12-16T16:07:44.880Z INFO memsql/clustering_master.go:73 Engine Version {"is": "7.8.14"}
2022-12-16T16:07:44.892Z INFO memsql/role_check.go:67 Pod role {"pod": "node-singlestore-general-tier-memsql-aggregator-1", "role": "AGGREGATOR"}
2022-12-16T16:07:44.894Z INFO memsql/role_check.go:67 Pod role {"pod": "node-singlestore-general-tier-memsql-leaf-ag2-1", "role": "LEAF"}
2022-12-16T16:07:44.895Z INFO memsql/role_check.go:67 Pod role {"pod": "node-singlestore-general-tier-memsql-master-0", "role": "MASTER_AGGREGATOR"}
2022-12-16T16:07:44.897Z INFO memsql/role_check.go:67 Pod role {"pod": "node-singlestore-general-tier-memsql-leaf-ag1-1", "role": "LEAF"}
2022-12-16T16:07:44.899Z INFO memsql/role_check.go:67 Pod role {"pod": "node-singlestore-general-tier-memsql-leaf-ag2-0", "role": "LEAF"}
2022-12-16T16:07:44.900Z INFO memsql/role_check.go:67 Pod role {"pod": "node-singlestore-general-tier-memsql-leaf-ag1-0", "role": "LEAF"}
2022-12-16T16:07:44.902Z INFO memsql/role_check.go:67 Pod role {"pod": "node-singlestore-general-tier-memsql-aggregator-0", "role": "AGGREGATOR"}
2022-12-16T16:07:44.906Z INFO memsql/clustering_master.go:73 Engine Version {"is": "7.8.14"}
2022-12-16T16:07:44.906Z INFO memsql/clustering_master.go:55 Ensure Master Aggregator is setup
2022-12-16T16:07:44.909Z INFO memsql/clustering.go:101 Gathering StatefulSet Info
2022-12-16T16:07:44.910Z INFO memsql/clustering.go:185 Ensuring all node-singlestore-general-tier-memsql-leaf-ag1 are added to the MA
2022-12-16T16:07:44.912Z INFO memsql/clustering.go:185 Ensuring all node-singlestore-general-tier-memsql-leaf-ag2 are added to the MA
2022-12-16T16:07:44.912Z INFO memsql/clustering.go:185 Ensuring all node-singlestore-general-tier-memsql-aggregator are added to the MA
2022-12-16T16:07:44.990Z INFO memsql/clustering.go:185 Ensuring all node-singlestore-general-tier-memsql-master are added to the MA
2022-12-16T16:07:44.991Z INFO memsql/clustering.go:262 Ensuring all leaves are updated to the latest statefulset changes
2022-12-16T16:07:44.992Z INFO memsql/clustering.go:1493 In paired mode, will do rebalance if leaf node removed
2022-12-16T16:07:45.015Z INFO controller/errors.go:67 Reconciler success {"will retry after": "5m0s"}
2022-12-16T16:12:45.015Z INFO controller/controller.go:114 Reconciling MemSQL Cluster. {"Request.Namespace": "namespace", "Request.Name": "singlestore-general-tier-memsql"}
2022-12-16T16:12:45.016Z INFO memsql/metrics.go:58 can't find operator deployment, trying uncached client {"key": "namespace/operator-singlestore-general-tier-memsql"}
2022-12-16T16:12:45.018Z ERROR memsql/metrics.go:61 can't find operator deployment, metrics service will not be created {"error": "deployments.apps \"operator-singlestore-general-tier-memsql\" not found"}
freya/kube/memsql.NewMetricsServiceAction.func1
freya/kube/memsql/metrics.go:61
freya/kube/memsql.ComposeActions.func1
freya/kube/memsql/action.go:22
freya/kube/controller.(*Reconciler).Reconcile
freya/kube/controller/controller.go:296
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile
sigs.k8s.io/controller-runtime@v0.11.0/pkg/internal/controller/controller.go:114
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler
sigs.k8s.io/controller-runtime@v0.11.0/pkg/internal/controller/controller.go:311
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem
sigs.k8s.io/controller-runtime@v0.11.0/pkg/internal/controller/controller.go:266
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2
sigs.k8s.io/controller-runtime@v0.11.0/pkg/internal/controller/controller.go:227
kubectl -n namespace get memsql
NAME AGGREGATORS LEAVES REDUNDANCY LEVEL AGE
singlestore-general-tier-memsql 3 2 2 27d
kb get pods| grep 'singlestore-general-tier'
node-singlestore-general-tier-memsql-aggregator-0 2/2 Running 0 26h
node-singlestore-general-tier-memsql-aggregator-1 2/2 Running 0 26h
node-singlestore-general-tier-memsql-leaf-ag1-0 2/2 Running 0 26h
node-singlestore-general-tier-memsql-leaf-ag1-1 2/2 Running 0 26h
node-singlestore-general-tier-memsql-leaf-ag2-0 2/2 Running 0 26h
node-singlestore-general-tier-memsql-leaf-ag2-1 2/2 Running 0 26h
node-singlestore-general-tier-memsql-master-0 2/2 Running 0 26h
operator-memsql-singlestore-general-tier-795cb88876-zlkfv 1/1 Running 0 9m29s
kb describe deploy/operator-memsql-singlestore-general-tier
Name: operator-memsql-singlestore-general-tier
Namespace: namespace
CreationTimestamp: Fri, 16 Dec 2022 15:59:41 +0000
Labels: app=memsql
app.kubernetes.io/instance=singlestore-general-tier
app.kubernetes.io/managed-by=Helm
app.kubernetes.io/name=memsql
app.kubernetes.io/version=7.5.19
helm.sh/chart=memsql-2.0.11
release=singlestore-general-tier
Annotations: deployment.kubernetes.io/revision: 1
meta.helm.sh/release-name: singlestore-general-tier
meta.helm.sh/release-namespace: namespace
Selector: app.kubernetes.io/instance=singlestore-general-tier,app.kubernetes.io/managed-by=Helm,app.kubernetes.io/name=memsql
Replicas: 1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType: RollingUpdate
MinReadySeconds: 0
RollingUpdateStrategy: 25% max unavailable, 25% max surge
Pod Template:
Labels: app=memsql
app.kubernetes.io/instance=singlestore-general-tier
app.kubernetes.io/managed-by=Helm
app.kubernetes.io/name=memsql
app.kubernetes.io/version=7.5.19
helm.sh/chart=memsql-2.0.11
release=singlestore-general-tier
Service Account: operator-memsql-singlestore-general-tier
Containers:
memsql:
Image: singlestore/operator:3.0.60-c818b3a1
Port: <none>
Host Port: <none>
Args:
--merge-service-annotations
--cores-per-unit=1
--memory-per-unit=4
--fs-group-id=5555
--cluster-id
singlestore-general-tier-memsql
Environment:
WATCH_NAMESPACE: (v1:metadata.namespace)
POD_NAME: (v1:metadata.name)
OPERATOR_NAME: operator-memsql-singlestore-general-tier
in the sdb-cluster.yaml
please change
serviceSpec:
type: “LoadBalancer”
to
serviceSpec:
type: “NodePort”
And run kubectl apply to apply the change, system will work