# Specified cafile is invalid
[root@tca /home/admin/v2.3]# ./update_ca.py verify-v1clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/multi-certs.crt
cafile /home/admin/v2.3/multi-certs.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is invalid: the specified cafile contains more than one certificate, it is expected ONLY ONE Certficate, the root CA certficate in the certficate chain or the self-signed certificate

[root@tca /home/admin/v2.3]# ./update_ca.py verify-v1clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root2.crt
cafile /home/admin/v2.3/root2.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is invalid: SSLError(MaxRetryError("HTTPSConnectionPool(host='lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net', port=443): Max retries exceeded with url: / (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1123)')))"))


# Update tkgcontext if you haven't done it before
./update_ca.py update-tkgcontexts --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: updated tkgcontext 279aa222-af53-4af6-a320-329bfc0ecb4a with response <Response [200]>
update_ca[INFO]: updated tkgcontext a421f9ef-d036-4718-a3d2-5fbddd7464bc with response <Response [200]>
update_ca[INFO]: updated tkgcontext 51afe1e3-8a9e-4dc6-999e-9ea3b8e8ec14 with response <Response [200]>
update_ca[INFO]: updated tkgcontext 3f05b6fc-245f-43c8-bb34-9342b7181ebe with response <Response [200]>

# Initial State
[root@tca /home/admin/v2.3]# ./update_ca.py verify-v1clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: ########## verifying v1 workload clusters ##########
update_ca[INFO]: # verifying workload cluster[wc1-230], id[7dbfebcd-4230-4f38-bbc1-8cc1fa57692e]
update_ca[ERROR]: configmap kapp-controller-config/tkg-system: out of date
update_ca[ERROR]: secret [wc1-230-kapp-controller-addon] in namespace [wc1-230]: out of date
update_ca[ERROR]: kubecontrolplane cr[wc1-230-master-control-plane/wc1-230]: out of date
update_ca[ERROR]: kubeadmconfigtemplate cr[wc1-230-np1/wc1-230]: out of date
update_ca[INFO]: query cluster wc1-230 node ips
update_ca[ERROR]: node[10.162.180.68]: out of date, SSL certificate problem
update_ca[ERROR]: node[10.162.182.96]: out of date, SSL certificate problem
update_ca[INFO]: # verifying workload cluster[wc4-230v1], id[5d487a53-dcb2-4d7c-9056-3b682fdf9420]
update_ca[ERROR]: configmap kapp-controller-config/tkg-system: out of date
update_ca[ERROR]: secret [wc4-230v1-kapp-controller-addon] in namespace [wc4-230v1]: out of date
update_ca[ERROR]: kubecontrolplane cr[wc4-230v1-master-control-plane/wc4-230v1]: out of date
update_ca[ERROR]: kubeadmconfigtemplate cr[wc4-230v1-np1/wc4-230v1]: out of date
update_ca[INFO]: query cluster wc4-230v1 node ips
update_ca[ERROR]: node[10.162.177.188]: out of date, SSL certificate problem
update_ca[ERROR]: node[10.162.182.104]: out of date, SSL certificate problem

# Update v1cluster wc1-230

[root@tca /home/admin/v2.3]# ./update_ca.py update-v1clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name wc1-230
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: query cluster wc1-230 node ips
update_ca[INFO]: cluster wc1-230, node ips: ['10.162.180.68', '10.162.182.96']
update_ca[INFO]: start updating v1 workload clusters' nodes
PLAY [update node airgap repo ca certificate] **********************************************************************************************************************************

TASK [Gathering Facts] *********************************************************************************************************************************************************
[WARNING]: Platform linux on host 10.162.180.68 is using the discovered Python interpreter at /usr/bin/python, but future installation of another Python interpreter could
change this. See https://docs.ansible.com/ansible/2.9/reference_appendices/interpreter_discovery.html for more information.
ok: [10.162.180.68]
[WARNING]: Platform linux on host 10.162.182.96 is using the discovered Python interpreter at /usr/bin/python, but future installation of another Python interpreter could
change this. See https://docs.ansible.com/ansible/2.9/reference_appendices/interpreter_discovery.html for more information.
ok: [10.162.182.96]

TASK [copy new CA to system] ***************************************************************************************************************************************************
changed: [10.162.182.96]
changed: [10.162.180.68]

TASK [rehash ca-bundles] *******************************************************************************************************************************************************
changed: [10.162.182.96]
changed: [10.162.180.68]
.....some more ansible outputs...
PLAY RECAP *********************************************************************************************************************************************************************
10.162.180.68              : ok=11   changed=8    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0
10.162.182.96              : ok=11   changed=8    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0

update_ca[INFO]: cluster nodes are updated successfully
update_ca[INFO]: end updating v1 workload clusters' nodes successfully
update_ca[INFO]: start updating workload clusters' resources: kapp-controller-config/kcp/kcts
update_ca[INFO]: update secret [wc1-230-kapp-controller-addon] in namespace [wc1-230] successfully
update_ca[INFO]: update kcp wc1-230-master-control-plane airgap cacert file content successfully
update_ca[INFO]: update kubeadmconfigtemplate wc1-230-np1 airgap cacert file content successfully
update_ca[INFO]: update cluster [wc1-230] kapp, kcp and kcts successfully
update_ca[INFO]: update cluster kapp-controller-config successfully
update_ca[INFO]: end updating v1 workload clusters' resources: kapp-controller-config/kcp/kcts successfully
update_ca[INFO]: v1 workload clusters configuration has been updated successfully!
update_ca[WARNING]: relevant cluster control plane nodes may be rolling updated, please check the results by running the verify-v1cluster command


# Monitor wc1-230 updating status
[root@tca /home/admin/v2.3]# ./update_ca.py verify-v1clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name wc1-230
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: ########## verifying v1 workload clusters ##########
update_ca[INFO]: # verifying workload cluster[wc1-230], id[7dbfebcd-4230-4f38-bbc1-8cc1fa57692e]
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: secret [wc1-230-kapp-controller-addon] in namespace [wc1-230]: up to date
update_ca[INFO]: kubecontrolplane cr[wc1-230-master-control-plane/wc1-230]: up to date
update_ca[INFO]: kubeadmconfigtemplate cr[wc1-230-np1/wc1-230]: up to date
update_ca[INFO]: query cluster wc1-230 node ips
update_ca[INFO]: node[10.162.176.75]: up to date   ===========> New Node
update_ca[INFO]: node[10.162.180.68]: up to date
update_ca[INFO]: node[10.162.182.96]: up to date

[root@tca /home/admin/v2.3]# ./update_ca.py verify-v1clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name wc1-230
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: ########## verifying v1 workload clusters ##########
update_ca[INFO]: # verifying workload cluster[wc1-230], id[7dbfebcd-4230-4f38-bbc1-8cc1fa57692e]
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: secret [wc1-230-kapp-controller-addon] in namespace [wc1-230]: up to date
update_ca[INFO]: kubecontrolplane cr[wc1-230-master-control-plane/wc1-230]: up to date
update_ca[INFO]: kubeadmconfigtemplate cr[wc1-230-np1/wc1-230]: up to date
update_ca[INFO]: query cluster wc1-230 node ips
update_ca[INFO]: node[10.162.176.75]: up to date
update_ca[INFO]: node[10.162.182.96]: up to date


# Update wc4-230
[root@tca /home/admin/v2.3]# ./update_ca.py update-v1clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name wc4-230v1
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: query cluster wc4-230v1 node ips
update_ca[INFO]: cluster wc4-230v1, node ips: ['10.162.181.25', '10.162.182.104']
update_ca[INFO]: start updating v1 workload clusters' nodes
.....some more ansible outputs...
update_ca[INFO]: cluster nodes are updated successfully
update_ca[INFO]: end updating v1 workload clusters' nodes successfully
update_ca[INFO]: start updating workload clusters' resources: kapp-controller-config/kcp/kcts
update_ca[INFO]: update secret [wc4-230v1-kapp-controller-addon] in namespace [wc4-230v1] successfully
update_ca[INFO]: update kcp wc4-230v1-master-control-plane/wc4-230v1 airgap cacert file content successfully
update_ca[INFO]: update kubeadmconfigtemplate wc4-230v1-np1/wc4-230v1 airgap cacert file content successfully
update_ca[INFO]: update cluster [wc4-230v1] kapp, kcp and kcts successfully
update_ca[INFO]: update cluster kapp-controller-config successfully
update_ca[INFO]: end updating v1 workload clusters' resources: kapp-controller-config/kcp/kcts successfully
update_ca[INFO]: v1 workload clusters configuration has been updated successfully!
update_ca[WARNING]: relevant cluster control plane nodes may be rolling updated, please check the results by running the verify-v1cluster command

[root@tca /home/admin/v2.3]# ./update_ca.py verify-v1clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt --name wc4-230v1
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: ########## verifying v1 workload clusters ##########
update_ca[INFO]: # verifying workload cluster[wc4-230v1], id[5d487a53-dcb2-4d7c-9056-3b682fdf9420]
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: secret [wc4-230v1-kapp-controller-addon] in namespace [wc4-230v1]: up to date
update_ca[INFO]: kubecontrolplane cr[wc4-230v1-master-control-plane/wc4-230v1]: up to date
update_ca[INFO]: kubeadmconfigtemplate cr[wc4-230v1-np1/wc4-230v1]: up to date
update_ca[INFO]: query cluster wc4-230v1 node ips
update_ca[INFO]: node[10.162.181.25]: up to date
update_ca[INFO]: node[10.162.182.104]: up to date


# verify all the v1clusters
[root@tca /home/admin/v2.3]# ./update_ca.py verify-v1clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: ########## verifying v1 workload clusters ##########
update_ca[INFO]: # verifying workload cluster[wc1-230], id[7dbfebcd-4230-4f38-bbc1-8cc1fa57692e]
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: secret [wc1-230-kapp-controller-addon] in namespace [wc1-230]: up to date
update_ca[INFO]: kubecontrolplane cr[wc1-230-master-control-plane/wc1-230]: up to date
update_ca[INFO]: kubeadmconfigtemplate cr[wc1-230-np1/wc1-230]: up to date
update_ca[INFO]: query cluster wc1-230 node ips
update_ca[INFO]: node[10.162.176.75]: up to date
update_ca[INFO]: node[10.162.182.96]: up to date
update_ca[INFO]: # verifying workload cluster[wc4-230v1], id[5d487a53-dcb2-4d7c-9056-3b682fdf9420]
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: secret [wc4-230v1-kapp-controller-addon] in namespace [wc4-230v1]: up to date
update_ca[INFO]: kubecontrolplane cr[wc4-230v1-master-control-plane/wc4-230v1]: up to date
update_ca[INFO]: kubeadmconfigtemplate cr[wc4-230v1-np1/wc4-230v1]: up to date
update_ca[INFO]: query cluster wc4-230v1 node ips
update_ca[INFO]: node[10.162.181.25]: up to date
update_ca[INFO]: node[10.162.182.104]: up to date

# Alternatively, you can update all the v1clusters, but it may result in all v1clusters rolling update their control plane nodes

[root@tca /home/admin/v2.3]# ./update_ca.py verify-v1clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: ########## verifying v1 workload clusters ##########
update_ca[INFO]: # verifying workload cluster[wc1-230], id[7dbfebcd-4230-4f38-bbc1-8cc1fa57692e]
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: secret [wc1-230-kapp-controller-addon] in namespace [wc1-230]: up to date
update_ca[INFO]: kubecontrolplane cr[wc1-230-master-control-plane/wc1-230]: up to date
update_ca[INFO]: kubeadmconfigtemplate cr[wc1-230-np1/wc1-230]: up to date
update_ca[INFO]: query cluster wc1-230 node ips
update_ca[INFO]: node[10.162.176.75]: up to date
update_ca[INFO]: node[10.162.182.96]: up to date
update_ca[INFO]: # verifying workload cluster[wc4-230v1], id[5d487a53-dcb2-4d7c-9056-3b682fdf9420]
update_ca[INFO]: configmap kapp-controller-config/tkg-system: up to date
update_ca[INFO]: secret [wc4-230v1-kapp-controller-addon] in namespace [wc4-230v1]: up to date
update_ca[INFO]: kubecontrolplane cr[wc4-230v1-master-control-plane/wc4-230v1]: up to date
update_ca[INFO]: kubeadmconfigtemplate cr[wc4-230v1-np1/wc4-230v1]: up to date
update_ca[INFO]: query cluster wc4-230v1 node ips
update_ca[INFO]: node[10.162.181.25]: up to date
update_ca[INFO]: node[10.162.182.104]: up to date

[root@tca /home/admin/v2.3]# ./update_ca.py update-v1clusters --fqdn lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net --cafile /home/admin/v2.3/root1.crt
update_ca[INFO]: cafile /home/admin/v2.3/root1.crt for airgap repo lvn-dvm-10-162-177-211.dvm.lvn.broadcom.net is valid
update_ca[INFO]: query cluster wc1-230 node ips
update_ca[INFO]: cluster wc1-230, node ips: ['10.162.176.75', '10.162.182.96']
update_ca[INFO]: query cluster wc4-230v1 node ips
update_ca[INFO]: cluster wc4-230v1, node ips: ['10.162.181.25', '10.162.182.104']
update_ca[INFO]: start updating v1 workload clusters' nodes

.....some more ansible outputs...
PLAY RECAP *********************************************************************************************************************************************************************
10.162.176.75              : ok=11   changed=7    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0
10.162.181.25              : ok=11   changed=6    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0
10.162.182.104             : ok=11   changed=6    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0
10.162.182.96              : ok=11   changed=6    unreachable=0    failed=0    skipped=0    rescued=0    ignored=0

update_ca[INFO]: cluster nodes are updated successfully
update_ca[INFO]: end updating v1 workload clusters' nodes successfully
update_ca[INFO]: start updating workload clusters' resources: kapp-controller-config/kcp/kcts
update_ca[INFO]: caCerts in secret [wc1-230-kapp-controller-addon] of namespace [wc1-230] is up to date, skip
update_ca[INFO]: kubecontrolplane cr[wc1-230-master-control-plane/wc1-230]: up to date
update_ca[INFO]: kubeadmconfigtemplate cr[wc1-230-np1/wc1-230]: up to date
update_ca[INFO]: update cluster [wc1-230] kapp, kcp and kcts successfully
update_ca[INFO]: cluster kapp-controller-config is up to date, skip
update_ca[INFO]: caCerts in secret [wc4-230v1-kapp-controller-addon] of namespace [wc4-230v1] is up to date, skip
update_ca[INFO]: kubecontrolplane cr[wc4-230v1-master-control-plane/wc4-230v1]: up to date
update_ca[INFO]: kubeadmconfigtemplate cr[wc4-230v1-np1/wc4-230v1]: up to date
update_ca[INFO]: update cluster [wc4-230v1] kapp, kcp and kcts successfully
update_ca[INFO]: cluster kapp-controller-config is up to date, skip
update_ca[INFO]: end updating v1 workload clusters' resources: kapp-controller-config/kcp/kcts successfully
update_ca[INFO]: v1 workload clusters configuration has been updated successfully!
update_ca[WARNING]: relevant cluster control plane nodes may be rolling updated, please check the results by running the verify-v1cluster command