# Samples of updating management cluster
# Specified cafile is invalid
[root@tca /home/admin/v2.3]# ./update_ca.py verify-mgmtclusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/multi-certs.crt
cafile /home/admin/v2.3/multi-certs.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is invalid: the specified cafile contains more than one certificate, it is expected ONLY ONE Certficate, the root CA certficate in the certficate chain or the self-signed certificate

[root@tca /home/admin/v2.3]# ./update_ca.py verify-mgmtclusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root2.crt
cafile /home/admin/v2.3/root2.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is invalid: SSLError(MaxRetryError("HTTPSConnectionPool(host='lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net', port=443): Max retries exceeded with url: / (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1123)')))"))

# Update tkgcontext if you haven't done it before
./update_ca.py update-tkgcontexts --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: updated tkgcontext 279aa222-af53-4af6-a320-329bfc0ecb4a with response <Response [200]>
update_ca[INFO]: updated tkgcontext a421f9ef-d036-4718-a3d2-5fbddd7464bc with response <Response [200]>
update_ca[INFO]: updated tkgcontext 51afe1e3-8a9e-4dc6-999e-9ea3b8e8ec14 with response <Response [200]>
update_ca[INFO]: updated tkgcontext 3f05b6fc-245f-43c8-bb34-9342b7181ebe with response <Response [200]>

./update_ca.py update-mgmtclusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name mc1-230
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: update cluster kapp-controller-config successfully
update_ca[INFO]: update secret [tkg-pkg-tkg-system-values] in namespace [tkg-system] successfully
update_ca[INFO]: update secret [tkr-source-controller-values] in namespace [tkg-system] successfully
update_ca[INFO]: update secret [tkr-vsphere-resolver-values] in namespace [tkg-system] successfully
update_ca[INFO]: update management cluster tkr-controller-config successfully
update_ca[INFO]: update clusterclass mgmt cluster [mc1-230] in namespace tkg-system successfully
update_ca[INFO]: start updating legacy management clusters' nodes
update_ca[INFO]: no legacy cluster node to update, skip
update_ca[INFO]: end updating management clusters' nodes successfully
update_ca[INFO]: start updating management clusters' resources: config maps/kcp/kct
update_ca[INFO]: management clusters configuration has been updated successfully!
update_ca[WARNING]: relevant management cluster nodes may be rolling updated, please check the results by running the verify-mgmtclusters command

[root@tca /home/admin/v2.3]# ./update_ca.py verify-mgmtclusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name mc1-230
update_ca[INFO]: ########## verifying management clusters ##########
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: # verifying management cluster[mc1-230], id[82b7e000-c260-417d-9ffd-4a03c51e1579]
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: secret tkg-pkg-tkg-system-values/tkg-system: up to date
update_ca[INFO]: secret [tkr-source-controller-values] in namespace [tkg-system]: up to date
update_ca[INFO]: secret [tkr-vsphere-resolver-values] in namespace [tkg-system]: up to date
update_ca[INFO]: configmap tkr-controller-config: up to date
update_ca[INFO]: cluster [mc1-230] in namespace tkg-system: up to date
update_ca[ERROR]: node[10.162.176.192]: out of date, SSL certificate problem
update_ca[INFO]: node[10.162.177.136]: up to date
update_ca[INFO]: node[10.162.177.7]: up to date
update_ca[ERROR]: node[10.162.176.86]: out of date, SSL certificate problem
[root@tca /home/admin/v2.3]#
[root@tca /home/admin/v2.3]# ./update_ca.py verify-mgmtclusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name mc1-230
update_ca[INFO]: ########## verifying management clusters ##########
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: # verifying management cluster[mc1-230], id[82b7e000-c260-417d-9ffd-4a03c51e1579]
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: secret tkg-pkg-tkg-system-values/tkg-system: up to date
update_ca[INFO]: secret [tkr-source-controller-values] in namespace [tkg-system]: up to date
update_ca[INFO]: secret [tkr-vsphere-resolver-values] in namespace [tkg-system]: up to date
update_ca[INFO]: configmap tkr-controller-config: up to date
update_ca[INFO]: cluster [mc1-230] in namespace tkg-system: up to date
update_ca[INFO]: node[10.162.177.136]: up to date
update_ca[INFO]: node[10.162.177.7]: up to date

[root@tca /home/admin/v2.3]# ./update_ca.py verify-mgmtclusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name mc2-230
update_ca[INFO]: ########## verifying management clusters ##########
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: # verifying management cluster[mc2-230], id[4bd3e2b9-b26f-4ae3-8df8-11a88807631b]
update_ca[ERROR]: configmap kapp-controller-config/tkg-system: out of date
update_ca[ERROR]: secret tkg-pkg-tkg-system-values/tkg-system: out of date
update_ca[ERROR]: secret [tkr-source-controller-values] in namespace [tkg-system]: out of date
update_ca[ERROR]: secret [tkr-vsphere-resolver-values] in namespace [tkg-system]: out of date
update_ca[ERROR]: configmap tkr-controller-config: out of date
update_ca[ERROR]: cluster [mc2-230] in namespace tkg-system: out of date
update_ca[ERROR]: cluster [mc2-230] in namespace tkg-system: out of date
update_ca[ERROR]: node[10.162.178.205]: out of date, SSL certificate problem
update_ca[ERROR]: node[10.162.181.31]: out of date, SSL certificate problem

[root@tca /home/admin/v2.3]# ./update_ca.py update-mgmtclusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name mc2-230
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: update cluster kapp-controller-config successfully
update_ca[INFO]: update secret [tkg-pkg-tkg-system-values] in namespace [tkg-system] successfully
update_ca[INFO]: update secret [tkr-source-controller-values] in namespace [tkg-system] successfully
update_ca[INFO]: update secret [tkr-vsphere-resolver-values] in namespace [tkg-system] successfully
update_ca[INFO]: update management cluster tkr-controller-config successfully
update_ca[INFO]: update clusterclass mgmt cluster [mc2-230] in namespace tkg-system successfully
update_ca[INFO]: start updating legacy management clusters' nodes
update_ca[INFO]: no legacy cluster node to update, skip
update_ca[INFO]: end updating management clusters' nodes successfully
update_ca[INFO]: start updating management clusters' resources: config maps/kcp/kct
update_ca[INFO]: management clusters configuration has been updated successfully!
update_ca[WARNING]: relevant management cluster nodes may be rolling updated, please check the results by running the verify-mgmtclusters command

[root@tca /home/admin/v2.3]# ./update_ca.py verify-mgmtclusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name mc2-230
update_ca[INFO]: ########## verifying management clusters ##########
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: # verifying management cluster[mc2-230], id[4bd3e2b9-b26f-4ae3-8df8-11a88807631b]
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: secret tkg-pkg-tkg-system-values/tkg-system: up to date
update_ca[INFO]: secret [tkr-source-controller-values] in namespace [tkg-system]: up to date
update_ca[INFO]: secret [tkr-vsphere-resolver-values] in namespace [tkg-system]: up to date
update_ca[INFO]: configmap tkr-controller-config: up to date
update_ca[INFO]: cluster [mc2-230] in namespace tkg-system: up to date
update_ca[ERROR]: node[10.162.178.205]: out of date, SSL certificate problem
update_ca[ERROR]: node[10.162.181.31]: out of date, SSL certificate problem

[root@tca /home/admin/v2.3]# ./update_ca.py verify-mgmtclusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name mc2-230
update_ca[INFO]: ########## verifying management clusters ##########
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: # verifying management cluster[mc2-230], id[4bd3e2b9-b26f-4ae3-8df8-11a88807631b]
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: secret tkg-pkg-tkg-system-values/tkg-system: up to date
update_ca[INFO]: secret [tkr-source-controller-values] in namespace [tkg-system]: up to date
update_ca[INFO]: secret [tkr-vsphere-resolver-values] in namespace [tkg-system]: up to date
update_ca[INFO]: configmap tkr-controller-config: up to date
update_ca[INFO]: cluster [mc2-230] in namespace tkg-system: up to date
update_ca[ERROR]: node[10.162.178.205]: out of date, SSL certificate problem
update_ca[ERROR]: node[10.162.181.31]: out of date, SSL certificate problem
update_ca[INFO]: node[10.162.179.69]: up to date

[root@tca /home/admin/v2.3]# ./update_ca.py verify-mgmtclusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name mc2-230
update_ca[INFO]: ########## verifying management clusters ##########
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: # verifying management cluster[mc2-230], id[4bd3e2b9-b26f-4ae3-8df8-11a88807631b]
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: secret tkg-pkg-tkg-system-values/tkg-system: up to date
update_ca[INFO]: secret [tkr-source-controller-values] in namespace [tkg-system]: up to date
update_ca[INFO]: secret [tkr-vsphere-resolver-values] in namespace [tkg-system]: up to date
update_ca[INFO]: configmap tkr-controller-config: up to date
update_ca[INFO]: cluster [mc2-230] in namespace tkg-system: up to date
update_ca[INFO]: node[10.162.182.222]: up to date
update_ca[ERROR]: node[10.162.178.205]: out of date, SSL certificate problem
update_ca[ERROR]: node[10.162.181.31]: out of date, SSL certificate problem
update_ca[INFO]: node[10.162.179.69]: up to date

[root@tca /home/admin/v2.3]# ./update_ca.py verify-mgmtclusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name mc2-230
update_ca[INFO]: ########## verifying management clusters ##########
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: # verifying management cluster[mc2-230], id[4bd3e2b9-b26f-4ae3-8df8-11a88807631b]
Traceback (most recent call last):
  File "/home/admin/v2.3/./update_ca.py", line 1740, in <module>
    sys.exit(main())
  File "/home/admin/v2.3/./update_ca.py", line 1737, in main
    args.func(args)
  File "/home/admin/v2.3/./update_ca.py", line 1346, in verify_mgmt_clusters
    if cluster_client.IsClusterClass(cluster["clusterName"]):
  File "/home/admin/v2.3/./update_ca.py", line 501, in IsClusterClass
    clusterCR = self.get_dict(GET_CLUSTER_CMD)
  File "/home/admin/v2.3/./update_ca.py", line 404, in get_dict
    return json.loads(self.get_json(cmd))
  File "/home/admin/v2.3/./update_ca.py", line 393, in get_json
    return self.run_cmd(get_cmd)
  File "/home/admin/v2.3/./update_ca.py", line 387, in run_cmd
    raise RuntimeError("run \"%s\" failed, err: %s" % (kubectl_cmd, error))
RuntimeError: run "kubectl --kubeconfig /opt/vmware/k8s-bootstrapper/4bd3e2b9-b26f-4ae3-8df8-11a88807631b/kubeconfig --request-timeout 30s get cluster -n tkg-system mc2-230 -o json" failed, err: Unable to connect to the server: context deadline exceeded

Note: it is expected during the cluster control plane node rolling update when switching the endpoint IP to the new node.

[root@tca /home/admin/v2.3]# ./update_ca.py verify-mgmtclusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name mc2-230
update_ca[INFO]: ########## verifying management clusters ##########
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: # verifying management cluster[mc2-230], id[4bd3e2b9-b26f-4ae3-8df8-11a88807631b]
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: secret tkg-pkg-tkg-system-values/tkg-system: up to date
update_ca[INFO]: secret [tkr-source-controller-values] in namespace [tkg-system]: up to date
update_ca[INFO]: secret [tkr-vsphere-resolver-values] in namespace [tkg-system]: up to date
update_ca[INFO]: configmap tkr-controller-config: up to date
update_ca[INFO]: cluster [mc2-230] in namespace tkg-system: up to date
update_ca[INFO]: node[10.162.182.222]: up to date
update_ca[ERROR]: node[10.162.178.205]: cannot communicate with airgap repo, return code:255, stdout: , stderr: ssh: connect to host 10.162.178.205 port 22: Connection timed out
update_ca[ERROR]: node[10.162.181.31]: cannot communicate with airgap repo, return code:255, stdout: , stderr: ssh: connect to host 10.162.181.31 port 22: Connection timed out

update_ca[INFO]: node[10.162.179.69]: up to date

[root@tca /home/admin/v2.3]# ./update_ca.py verify-mgmtclusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name mc2-230
update_ca[INFO]: ########## verifying management clusters ##########
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: # verifying management cluster[mc2-230], id[4bd3e2b9-b26f-4ae3-8df8-11a88807631b]
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: secret tkg-pkg-tkg-system-values/tkg-system: up to date
update_ca[INFO]: secret [tkr-source-controller-values] in namespace [tkg-system]: up to date
update_ca[INFO]: secret [tkr-vsphere-resolver-values] in namespace [tkg-system]: up to date
update_ca[INFO]: configmap tkr-controller-config: up to date
update_ca[INFO]: cluster [mc2-230] in namespace tkg-system: up to date
update_ca[INFO]: node[10.162.182.222]: up to date
update_ca[INFO]: node[10.162.179.69]: up to date

[root@tca /home/admin/v2.3]# ./update_ca.py verify-mgmtclusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt
update_ca[INFO]: ########## verifying management clusters ##########
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: # verifying management cluster[mc1-230], id[82b7e000-c260-417d-9ffd-4a03c51e1579]
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: secret tkg-pkg-tkg-system-values/tkg-system: up to date
update_ca[INFO]: secret [tkr-source-controller-values] in namespace [tkg-system]: up to date
update_ca[INFO]: secret [tkr-vsphere-resolver-values] in namespace [tkg-system]: up to date
update_ca[INFO]: configmap tkr-controller-config: up to date
update_ca[INFO]: cluster [mc1-230] in namespace tkg-system: up to date
update_ca[INFO]: node[10.162.177.136]: up to date
update_ca[INFO]: node[10.162.177.7]: up to date
update_ca[INFO]: # verifying management cluster[mc2-230], id[4bd3e2b9-b26f-4ae3-8df8-11a88807631b]
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: secret tkg-pkg-tkg-system-values/tkg-system: up to date
update_ca[INFO]: secret [tkr-source-controller-values] in namespace [tkg-system]: up to date
update_ca[INFO]: secret [tkr-vsphere-resolver-values] in namespace [tkg-system]: up to date
update_ca[INFO]: configmap tkr-controller-config: up to date
update_ca[INFO]: cluster [mc2-230] in namespace tkg-system: up to date
update_ca[INFO]: node[10.162.182.222]: up to date
update_ca[INFO]: node[10.162.179.69]: up to date

[root@tca /home/admin/v2.3]# ./update_ca.py update-mgmtclusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: cluster kapp-controller-config is up to date, skip
update_ca[INFO]: secret tkg-pkg-tkg-system-values/tkg-system is up to date, skip
update_ca[INFO]: secret tkr-source-controller-values/tkg-system is up to date, skip
update_ca[INFO]: secret tkr-vsphere-resolver-values/tkg-system is up to date, skip
update_ca[INFO]: update management cluster tkr-controller-config successfully
update_ca[INFO]: clusterclass mgmt cluster [mc2-230] in namespace tkg-system up to date
update_ca[INFO]: cluster kapp-controller-config is up to date, skip
update_ca[INFO]: secret tkg-pkg-tkg-system-values/tkg-system is up to date, skip
update_ca[INFO]: secret tkr-source-controller-values/tkg-system is up to date, skip
update_ca[INFO]: secret tkr-vsphere-resolver-values/tkg-system is up to date, skip
update_ca[INFO]: update management cluster tkr-controller-config successfully
update_ca[INFO]: clusterclass mgmt cluster [mc1-230] in namespace tkg-system up to date
update_ca[INFO]: start updating legacy management clusters' nodes
update_ca[INFO]: no legacy cluster node to update, skip
update_ca[INFO]: end updating management clusters' nodes successfully
update_ca[INFO]: start updating management clusters' resources: config maps/kcp/kct
update_ca[INFO]: management clusters configuration has been updated successfully!
update_ca[WARNING]: relevant management cluster nodes may be rolling updated, please check the results by running the verify-mgmtclusters command
[root@tca /home/admin/v2.3]# ./update_ca.py verify-mgmtclusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt
update_ca[INFO]: ########## verifying management clusters ##########
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: # verifying management cluster[mc1-230], id[82b7e000-c260-417d-9ffd-4a03c51e1579]
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: secret tkg-pkg-tkg-system-values/tkg-system: up to date
update_ca[INFO]: secret [tkr-source-controller-values] in namespace [tkg-system]: up to date
update_ca[INFO]: secret [tkr-vsphere-resolver-values] in namespace [tkg-system]: up to date
update_ca[INFO]: configmap tkr-controller-config: up to date
update_ca[INFO]: cluster [mc1-230] in namespace tkg-system: up to date
update_ca[INFO]: node[10.162.177.136]: up to date
update_ca[INFO]: node[10.162.177.7]: up to date
update_ca[INFO]: # verifying management cluster[mc2-230], id[4bd3e2b9-b26f-4ae3-8df8-11a88807631b]
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: secret tkg-pkg-tkg-system-values/tkg-system: up to date
update_ca[INFO]: secret [tkr-source-controller-values] in namespace [tkg-system]: up to date
update_ca[INFO]: secret [tkr-vsphere-resolver-values] in namespace [tkg-system]: up to date
update_ca[INFO]: configmap tkr-controller-config: up to date
update_ca[INFO]: cluster [mc2-230] in namespace tkg-system: up to date
update_ca[INFO]: node[10.162.182.222]: up to date
update_ca[INFO]: node[10.162.179.69]: up to date