# Specified cafile is invalid
[root@tca /home/admin/v2.3]# ./update_ca.py verify-v2clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/multi-certs.crt
cafile /home/admin/v2.3/multi-certs.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is invalid: the specified cafile contains more than one certificate, it is expected ONLY ONE Certficate, the root CA certficate in the certficate chain or the self-signed certificate

[root@tca /home/admin/v2.3]# ./update_ca.py verify-v2clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root2.crt
cafile /home/admin/v2.3/root2.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is invalid: SSLError(MaxRetryError("HTTPSConnectionPool(host='lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net', port=443): Max retries exceeded with url: / (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1123)')))"))

# Initial State
[root@tca /home/admin/v2.3]# ./update_ca.py verify-v2clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: ########## verifying minikube ##########
update_ca[ERROR]: tcakubenetescluster cr[mc1-230/mc1-230]: out of date
update_ca[ERROR]: tcakubenetescluster cr[mc2-230/mc2-230]: out of date
update_ca[INFO]: ########## verifying v2 workload clusters ##########
update_ca[INFO]: # verifying v2 workload cluster[wc2-230]
update_ca[ERROR]: tcakubenetescluster cr[wc2-230/wc2-230]: out of date
update_ca[ERROR]: kubecontrolplane cr[wc2-230-control-plane/wc2-230]: out of date
update_ca[ERROR]: secret [wc2-230-kapp-controller-addon] in namespace [wc2-230]: out of date
update_ca[ERROR]: configmap kapp-controller-config/tkg-system: out of date
update_ca[ERROR]: node[10.162.176.91]: out of date, SSL certificate problem
update_ca[INFO]: # verifying v2 workload cluster[wc3-230]
update_ca[ERROR]: tcakubenetescluster cr[wc3-230/wc3-230]: out of date
update_ca[ERROR]: kubecontrolplane cr[wc3-230-control-plane/wc3-230]: out of date
update_ca[ERROR]: secret [wc3-230-kapp-controller-addon] in namespace [wc3-230]: out of date
update_ca[ERROR]: configmap kapp-controller-config/tkg-system: out of date
update_ca[ERROR]: node[10.162.179.208]: out of date, SSL certificate problem

# Update v2cluster wc2-230

[root@tca /home/admin/v2.3]# ./update_ca.py update-v2clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name wc2-230
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: start updating v2 workload clusters kapp-controller-config and nodes
update_ca[INFO]: start updating mgmt clusters' cr in minikube
update_ca[INFO]: updating cluster [mc1-230/mc1-230] tkc airgap annotation repo ca cert
update_ca[INFO]: update cluster mc1-230 tkc airgap ca annotation successfully
update_ca[INFO]: updating cluster [mc2-230/mc2-230] tkc airgap annotation repo ca cert
update_ca[INFO]: update cluster mc2-230 tkc airgap ca annotation successfully
update_ca[INFO]: end updating mgmt clusters' cr in minikube successfully
update_ca[INFO]: cluster wc2-230, node ips: ['10.162.176.91']
update_ca[INFO]: update secret [wc2-230-kapp-controller-addon] in namespace [wc2-230] successfully
update_ca[INFO]: update cluster kapp-controller-config successfully
PLAY [update node airgap repo ca certificate] **********************************************************************************************************************************

TASK [Gathering Facts] *********************************************************************************************************************************************************
[WARNING]: Platform linux on host 10.162.176.91 is using the discovered Python interpreter at /usr/bin/python, but future installation of another Python interpreter could
change this. See https://docs.ansible.com/ansible/2.9/reference_appendices/interpreter_discovery.html for more information.
ok: [10.162.176.91]

TASK [copy new CA to system] ***************************************************************************************************************************************************
changed: [10.162.176.91]
.....some more ansible outputs...
PLAY RECAP *********************************************************************************************************************************************************************
10.162.176.91              : ok=11   changed=8    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0

update_ca[INFO]: cluster nodes are updated successfully
update_ca[INFO]: end updating v2 workload clusters kapp-controller-config and nodes successfully
update_ca[INFO]: start updating workload clusters' crs on mgmt clusters
update_ca[INFO]: checking mgmt cluster [mc2-230] for wcs to update tkc/kcp/kct
update_ca[INFO]: end updating workload clusters' crs on mgmt cluster [mc2-230] successfully
update_ca[INFO]: checking mgmt cluster [mc1-230] for wcs to update tkc/kcp/kct
update_ca[INFO]: updating cluster [wc2-230/wc2-230] tkc spec airgap repo ca cert
update_ca[INFO]: update cluster wc2-230 tkc spec airgap cacert successfully
update_ca[INFO]: update kcp wc2-230-control-plane airgap cacert file content successfully
update_ca[INFO]: update cluster [wc2-230/wc2-230] kcp and kct successfully
update_ca[INFO]: end updating workload clusters' crs on mgmt cluster [mc1-230] successfully
update_ca[INFO]: v2 workload clusters configuration has been updated successfully!
update_ca[WARNING]: relevant cluster control plane nodes may be rolling updated, please check the results by running the verify-v2cluster command


# Monitor wc2-230 updating status
[root@tca /home/admin/v2.3]# ./update_ca.py verify-v2clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name wc2-230
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: ########## verifying minikube ##########
update_ca[INFO]: tcakubenetescluster cr[mc1-230/mc1-230]: up to date
update_ca[INFO]: tcakubenetescluster cr[mc2-230/mc2-230]: up to date
update_ca[INFO]: ########## verifying v2 workload clusters ##########
update_ca[INFO]: # verifying v2 workload cluster[wc2-230]
update_ca[INFO]: tcakubenetescluster cr[wc2-230/wc2-230]: up to date
update_ca[INFO]: kubecontrolplane cr[wc2-230-control-plane/wc2-230]: up to date
update_ca[INFO]: secret [wc2-230-kapp-controller-addon] in namespace [wc2-230]: up to date
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: node[10.162.176.91]: up to date
update_ca[INFO]: node[10.162.178.235]: up to date ========> New Node

[root@tca /home/admin/v2.3]# ./update_ca.py verify-v2clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name wc2-230
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: ########## verifying minikube ##########
update_ca[INFO]: tcakubenetescluster cr[mc1-230/mc1-230]: up to date
update_ca[INFO]: tcakubenetescluster cr[mc2-230/mc2-230]: up to date
update_ca[INFO]: ########## verifying v2 workload clusters ##########
update_ca[INFO]: # verifying v2 workload cluster[wc2-230]
update_ca[INFO]: tcakubenetescluster cr[wc2-230/wc2-230]: up to date
update_ca[INFO]: kubecontrolplane cr[wc2-230-control-plane/wc2-230]: up to date
update_ca[INFO]: secret [wc2-230-kapp-controller-addon] in namespace [wc2-230]: up to date
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: node[10.162.178.235]: up to date


# Check wc3-230 is untouched
[root@tca /home/admin/v2.3]# ./update_ca.py verify-v2clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name wc3-230
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: ########## verifying minikube ##########
update_ca[INFO]: tcakubenetescluster cr[mc1-230/mc1-230]: up to date
update_ca[INFO]: tcakubenetescluster cr[mc2-230/mc2-230]: up to date
update_ca[INFO]: ########## verifying v2 workload clusters ##########
update_ca[INFO]: # verifying v2 workload cluster[wc3-230]
update_ca[ERROR]: tcakubenetescluster cr[wc3-230/wc3-230]: out of date
update_ca[ERROR]: kubecontrolplane cr[wc3-230-control-plane/wc3-230]: out of date
update_ca[ERROR]: secret [wc3-230-kapp-controller-addon] in namespace [wc3-230]: out of date
update_ca[ERROR]: configmap kapp-controller-config/tkg-system: out of date
update_ca[ERROR]: node[10.162.179.208]: out of date, SSL certificate problem

# Update wc3-230
[root@tca /home/admin/v2.3]# ./update_ca.py update-v2clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name wc3-230
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: start updating v2 workload clusters kapp-controller-config and nodes
update_ca[INFO]: start updating mgmt clusters' cr in minikube
update_ca[INFO]: updating cluster [mc1-230/mc1-230] tkc airgap annotation repo ca cert
update_ca[INFO]: update cluster mc1-230 tkc airgap ca annotation successfully
update_ca[INFO]: updating cluster [mc2-230/mc2-230] tkc airgap annotation repo ca cert
update_ca[INFO]: update cluster mc2-230 tkc airgap ca annotation successfully
update_ca[INFO]: end updating mgmt clusters' cr in minikube successfully
update_ca[INFO]: cluster wc3-230, node ips: ['10.162.179.208']
update_ca[INFO]: update secret [wc3-230-kapp-controller-addon] in namespace [wc3-230] successfully
update_ca[INFO]: update cluster kapp-controller-config successfully
PLAY [update node airgap repo ca certificate] **********************************************************************************************************************************

TASK [Gathering Facts] *********************************************************************************************************************************************************
[WARNING]: Platform linux on host 10.162.179.208 is using the discovered Python interpreter at /usr/bin/python, but future installation of another Python interpreter could
change this. See https://docs.ansible.com/ansible/2.9/reference_appendices/interpreter_discovery.html for more information.
ok: [10.162.179.208]

TASK [copy new CA to system] ***************************************************************************************************************************************************
changed: [10.162.179.208]

TASK [rehash ca-bundles] *******************************************************************************************************************************************************
changed: [10.162.179.208]
.....some more ansible outputs...
PLAY RECAP *********************************************************************************************************************************************************************
10.162.179.208             : ok=11   changed=8    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0

update_ca[INFO]: cluster nodes are updated successfully
update_ca[INFO]: end updating v2 workload clusters kapp-controller-config and nodes successfully
update_ca[INFO]: start updating workload clusters' crs on mgmt clusters
update_ca[INFO]: checking mgmt cluster [mc2-230] for wcs to update tkc/kcp/kct
update_ca[INFO]: end updating workload clusters' crs on mgmt cluster [mc2-230] successfully
update_ca[INFO]: checking mgmt cluster [mc1-230] for wcs to update tkc/kcp/kct
update_ca[INFO]: updating cluster [wc3-230/wc3-230] tkc spec airgap repo ca cert
update_ca[INFO]: update cluster wc3-230 tkc spec airgap cacert successfully
update_ca[INFO]: update kcp wc3-230-control-plane airgap cacert file content successfully
update_ca[INFO]: update cluster [wc3-230/wc3-230] kcp and kct successfully
update_ca[INFO]: end updating workload clusters' crs on mgmt cluster [mc1-230] successfully
update_ca[INFO]: v2 workload clusters configuration has been updated successfully!
update_ca[WARNING]: relevant cluster control plane nodes may be rolling updated, please check the results by running the verify-v2cluster command

[root@tca /home/admin/v2.3]# ./update_ca.py verify-v2clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name wc3-230
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: ########## verifying minikube ##########
update_ca[INFO]: tcakubenetescluster cr[mc1-230/mc1-230]: up to date
update_ca[INFO]: tcakubenetescluster cr[mc2-230/mc2-230]: up to date
update_ca[INFO]: ########## verifying v2 workload clusters ##########
update_ca[INFO]: # verifying v2 workload cluster[wc3-230]
update_ca[INFO]: tcakubenetescluster cr[wc3-230/wc3-230]: up to date
update_ca[INFO]: kubecontrolplane cr[wc3-230-control-plane/wc3-230]: up to date
update_ca[INFO]: secret [wc3-230-kapp-controller-addon] in namespace [wc3-230]: up to date
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: node[10.162.179.208]: up to date
update_ca[INFO]: node[10.162.179.35]: up to date  ========> New Node

# It is expected when the old cp machine is being deleted.
[root@tca /home/admin/v2.3]# ./update_ca.py verify-v2clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name wc3-230
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: ########## verifying minikube ##########
update_ca[INFO]: tcakubenetescluster cr[mc1-230/mc1-230]: up to date
update_ca[INFO]: tcakubenetescluster cr[mc2-230/mc2-230]: up to date
update_ca[INFO]: ########## verifying v2 workload clusters ##########
update_ca[INFO]: # verifying v2 workload cluster[wc3-230]
update_ca[INFO]: tcakubenetescluster cr[wc3-230/wc3-230]: up to date
update_ca[INFO]: kubecontrolplane cr[wc3-230-control-plane/wc3-230]: up to date
update_ca[INFO]: secret [wc3-230-kapp-controller-addon] in namespace [wc3-230]: up to date
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[ERROR]: node[10.162.179.208]: cannot communicate with airgap repo, return code:255, stdout: , stderr: ssh: connect to host 10.162.179.208 port 22: Connection timed out

update_ca[INFO]: node[10.162.179.35]: up to date


[root@tca /home/admin/v2.3]# ./update_ca.py verify-v2clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name wc3-230
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: ########## verifying minikube ##########
update_ca[INFO]: tcakubenetescluster cr[mc1-230/mc1-230]: up to date
update_ca[INFO]: tcakubenetescluster cr[mc2-230/mc2-230]: up to date
update_ca[INFO]: ########## verifying v2 workload clusters ##########
update_ca[INFO]: # verifying v2 workload cluster[wc3-230]
update_ca[INFO]: tcakubenetescluster cr[wc3-230/wc3-230]: up to date
update_ca[INFO]: kubecontrolplane cr[wc3-230-control-plane/wc3-230]: up to date
update_ca[INFO]: secret [wc3-230-kapp-controller-addon] in namespace [wc3-230]: up to date
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: node[10.162.179.35]: up to date


# Verify All v2clusters
[root@tca /home/admin/v2.3]# ./update_ca.py verify-v2clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: ########## verifying minikube ##########
update_ca[INFO]: tcakubenetescluster cr[mc1-230/mc1-230]: up to date
update_ca[INFO]: tcakubenetescluster cr[mc2-230/mc2-230]: up to date
update_ca[INFO]: ########## verifying v2 workload clusters ##########
update_ca[INFO]: # verifying v2 workload cluster[wc2-230]
update_ca[INFO]: tcakubenetescluster cr[wc2-230/wc2-230]: up to date
update_ca[INFO]: kubecontrolplane cr[wc2-230-control-plane/wc2-230]: up to date
update_ca[INFO]: secret [wc2-230-kapp-controller-addon] in namespace [wc2-230]: up to date
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: node[10.162.178.235]: up to date
update_ca[INFO]: # verifying v2 workload cluster[wc3-230]
update_ca[INFO]: tcakubenetescluster cr[wc3-230/wc3-230]: up to date
update_ca[INFO]: kubecontrolplane cr[wc3-230-control-plane/wc3-230]: up to date
update_ca[INFO]: secret [wc3-230-kapp-controller-addon] in namespace [wc3-230]: up to date
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: node[10.162.179.35]: up to date

# Alternatively, you can update all the v2clusters, but it may result in all v2clusters rolling update their control plane nodes
[root@tca /home/admin/v2.3]# ./update_ca.py update-v2clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: start updating v2 workload clusters kapp-controller-config and nodes
update_ca[INFO]: start updating mgmt clusters' cr in minikube
update_ca[INFO]: updating cluster [mc1-230/mc1-230] tkc airgap annotation repo ca cert
update_ca[INFO]: update cluster mc1-230 tkc airgap ca annotation successfully
update_ca[INFO]: updating cluster [mc2-230/mc2-230] tkc airgap annotation repo ca cert
update_ca[INFO]: update cluster mc2-230 tkc airgap ca annotation successfully
update_ca[INFO]: end updating mgmt clusters' cr in minikube successfully
update_ca[INFO]: cluster wc2-230, node ips: ['10.162.178.235']
update_ca[INFO]: caCerts in secret [wc2-230-kapp-controller-addon] of namespace [wc2-230] is up to date, skip
update_ca[INFO]: cluster kapp-controller-config is up to date, skip
update_ca[INFO]: cluster wc3-230, node ips: ['10.162.179.35']
update_ca[INFO]: caCerts in secret [wc3-230-kapp-controller-addon] of namespace [wc3-230] is up to date, skip
update_ca[INFO]: cluster kapp-controller-config is up to date, skip
.....some more ansible outputs...
PLAY RECAP *********************************************************************************************************************************************************************
10.162.178.235             : ok=11   changed=7    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0
10.162.179.35              : ok=11   changed=7    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0

update_ca[INFO]: cluster nodes are updated successfully
update_ca[INFO]: end updating v2 workload clusters kapp-controller-config and nodes successfully
update_ca[INFO]: start updating workload clusters' crs on mgmt clusters
update_ca[INFO]: checking mgmt cluster [mc1-230] for wcs to update tkc/kcp/kct
update_ca[INFO]: updating cluster [wc2-230/wc2-230] tkc spec airgap repo ca cert
update_ca[INFO]: update cluster wc2-230 tkc spec airgap cacert successfully
update_ca[INFO]: updating cluster [wc3-230/wc3-230] tkc spec airgap repo ca cert
update_ca[INFO]: update cluster wc3-230 tkc spec airgap cacert successfully
update_ca[INFO]: kubecontrolplane cr[wc2-230-control-plane/wc2-230]: up to date
update_ca[INFO]: update cluster [wc2-230/wc2-230] kcp and kct successfully
update_ca[INFO]: kubecontrolplane cr[wc3-230-control-plane/wc3-230]: up to date
update_ca[INFO]: update cluster [wc3-230/wc3-230] kcp and kct successfully
update_ca[INFO]: end updating workload clusters' crs on mgmt cluster [mc1-230] successfully
update_ca[INFO]: checking mgmt cluster [mc2-230] for wcs to update tkc/kcp/kct
update_ca[INFO]: end updating workload clusters' crs on mgmt cluster [mc2-230] successfully
update_ca[INFO]: v2 workload clusters configuration has been updated successfully!
update_ca[WARNING]: relevant cluster control plane nodes may be rolling updated, please check the results by running the verify-v2cluster command


# Note, as the clusters are updated aleady, so rerun the update-v2clusters command won't result in cp nodes rolling update.

[root@tca /home/admin/v2.3]# ./update_ca.py verify-v2clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: ########## verifying minikube ##########
update_ca[INFO]: tcakubenetescluster cr[mc1-230/mc1-230]: up to date
update_ca[INFO]: tcakubenetescluster cr[mc2-230/mc2-230]: up to date
update_ca[INFO]: ########## verifying v2 workload clusters ##########
update_ca[INFO]: # verifying v2 workload cluster[wc2-230]
update_ca[INFO]: tcakubenetescluster cr[wc2-230/wc2-230]: up to date
update_ca[INFO]: kubecontrolplane cr[wc2-230-control-plane/wc2-230]: up to date
update_ca[INFO]: secret [wc2-230-kapp-controller-addon] in namespace [wc2-230]: up to date
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: node[10.162.178.235]: up to date
update_ca[INFO]: # verifying v2 workload cluster[wc3-230]
update_ca[INFO]: tcakubenetescluster cr[wc3-230/wc3-230]: up to date
update_ca[INFO]: kubecontrolplane cr[wc3-230-control-plane/wc3-230]: up to date
update_ca[INFO]: secret [wc3-230-kapp-controller-addon] in namespace [wc3-230]: up to date
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: node[10.162.179.35]: up to date
