#!/usr/bin/env python
# ***************************************************************************
# Copyright 2022-2025 VMware, Inc.  All rights reserved. VMware Confidential.
# ***************************************************************************
#
# This script is a drop-in replacement for curl.  The purpose of this script is
# to be a common entry point for the validation of server certs from client
# code making API calls to the NSX Manager.  For security reasons, client code
# should validate the remote (server) cert when making API connections. This
# script validates the certificates as defined by
# https://vmw-confluence.broadcom.net/display/NSBU/X509+Certificate+Validations
# This script acts as a drop-in replacement for curl in the sense that it
# supports the same options as curl, and returns the same error messages and
# exit codes.  Not all curl options are supported. The script doesn't reject
# any options that it doesn't support but silently ignores them.  This script
# supports IPv4 and IPv6.  An example IPv6 curl command is: curl_wrapper -u
# "admin:password" -k -i --thumbprint
# 6D:4B:5C:FE:F6:EA:C3:0D:EC:28:7A:E1:31:3C:F7:59:E4:35:1B:6A:6C:6E:6C:67:64:AE:A5:A8:BF:C0:9F:D8
# https://[fd01:0:106:209:0:a:0:1cc0]/api/v1/node/aaa/providers/vidm
# The thumbprint can be provided in upper or lower case, and with or without
# the colons.


from __future__ import print_function
import sys
import ssl
import importlib
import time
import argparse
import socket
import struct
import json
import subprocess
import traceback
import tempfile
import uuid
import os
import random
import string
import mimetypes
from os.path import exists
from datetime import datetime
from datetime import timedelta
from base64 import b64encode
from xml.dom import minidom

import warnings
warnings.filterwarnings("ignore")  # for CryptographyDeprecationWarning

try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse

try:
    from OpenSSL import crypto  # noqa
    from OpenSSL import SSL     # noqa
except ImportError:
    # In the case that OpenSSL isn't available the script will return
    # an error message, but only in the case that OpenSSL is needed.
    # OpenSSL is needed for HTTPS connections.
    pass

sys.path.append("/usr/lib/vmware/nsx-common/lib/python")
sys.path.append("/opt/vmware/nsx-monitoring/python")
sys.path.append("/opt/vmware/nsx-common/python")
have_nsx = True
try:
    from vmware.nsx.rpc import NsxRpcClient  # noqa
    from vmware.nsx.rpc import NsxRpcConnection  # noqa
    from vmware.nsx.messaging.applproxyinfo_pb2 import ApplProxyInfoService_Stub  # noqa
    from vmware.nsx.messaging.applproxyinfo_pb2 import ApplProxyInfoReqMsg  # noqa
    from vmware.nsx.messaging.applproxyinfo_pb2 import ApplProxyInfoRspMsg  # noqa
    from vmware.nsx.certificate.certificate_service_pb2 import CertificateService_Stub  # noqa
    from vmware.nsx.certificate.certificate_service_pb2 import CheckTrustedRequestMsg  # noqa
    from vmware.nsx.certificate.certificate_service_pb2 import CheckTrustedResponseMsg  # noqa
except ImportError:
    # This code path indicates that either NSX has not yet been installed
    # or we don't have the necessary version of NSX.
    have_nsx = False

have_py3 = sys.version_info >= (3,)
httplib_module = "http.client" if have_py3 else "httplib"
httplib = importlib.import_module(httplib_module)
if have_py3:
    from subprocess import TimeoutExpired
    unicode = str


class LocalLogger:
    """Logging wrapper class so this script works on plain Ubuntu.
    """
    def __init__(self):
        """Constructor checks if nsx_logging library is available.
           If not then don't log to log files.
        """
        self.logger = None
        try:
            import nsx_logging
            self.logger = nsx_logging.getLogger(__name__)
            nsx_logging.basicConfig(syslog=True, subcomp="curl_wrapper")
            self.logger.setLevel(nsx_logging.INFO)
        except ImportError:
            self.logger = None
        if (not self.logger and exists("/etc/issue") and not
                exists(NSX_ISSUE)):
            # This is saying if this script is running on an Ubuntu system
            # (/etc/issue is present) but not on an NSX node (/etc/nsx_issue is
            # not present), then write logs to /var/log/syslog without using
            # NSX's nsx_logger and write logs to /var/log/syslog.log on ESX
            # system.
            try:
                # Copied from /usr/lib/vmware/vsan/bin/vsan-config.py on ESXi
                # 8.0.0 build-21203435
                # Works on ESXi 8.0.0 build-21203435 (EAL4 build).
                # Works on ESXi 7.0.3.
                # Works on ESXi 6.0.7.
                import logging
                import logging.handlers
                self.logger = logging.getLogger('curl_wrapper')
                self.logger.setLevel(logging.INFO)
                fmt = '%(asctime)s %(name)s[%(process)d]: %(message)s'
                datefmt = "%b %d %H:%M:%S"
                formatter = logging.Formatter(fmt=fmt, datefmt=datefmt)
                handler = logging.handlers.SysLogHandler(address='/dev/log')
                handler.setFormatter(formatter)
                self.logger.addHandler(handler)
            except ImportError:
                self.logger = None

    def info(self, msg, *args):
        """Similar signature as info() in Lib/logging/__init__.py
           Don't include kwargs because this script doesn't use kwargs.
        """
        if self.logger:
            self.logger.info(msg, *args)


class NsxZeroize:
    """Wrapper class so that nsx_zeroize code doesn't fail in test environments
       like build servers.
    """
    def __init__(self):
        """Constructor checks if nsx_zeroize library is available.
           If not then don't zeroize the passed variable data.
        """
        self.nsx_zeroize = None
        if not exists(NSX_ISSUE) or not have_nsx:
            # On an Ubuntu dev system with source code, this script will import
            # nsx_zeroize, but it won't work properly because the associated
            # .so library is likely not installed. So don't attempt to import
            # nsx_zeroize in this case.
            return
        try:
            sys.path.append('/opt/vmware/nsx-common/python/nsx_utils')
            import nsx_zeroize
            self.nsx_zeroize = nsx_zeroize
        except ImportError:
            self.nsx_zeroize = None

    def zeroize(self, data):
        """If nsx_zeroize library is available try to
           zeroize the passed data.
        """
        if self.nsx_zeroize:
            self.nsx_zeroize.zeroize(data)


# curl error codes.
# See https://curl.se/libcurl/c/libcurl-errors.html.  The error codes with
# THIS_ preprended are deprecated in libcurl and being overloaded by this
# script.
CURLE_OK = 0
CURLE_FAILED_INIT = 2
CURLE_URL_MALFORMAT = 3
CURLM_INTERNAL_ERROR = 4
CURLE_COULDNT_CONNECT = 7
CURLE_PARTIAL_FILE = 18
CURLE_READ_ERROR = 26
CURLE_OPERATION_TIMEDOUT = 28
CURLE_SSL_CONNECT_ERROR = 35
THIS_WAS_REDIRECTED = 46
CURLE_TOO_MANY_REDIRECTS = 47
THIS_NO_ALTERNATIVE_CERTIFICATE_SUBJECT_NAME = 51
CURLE_GOT_NOTHING = 52
THIS_CRL_CHECK_FAILED = 53
CURLE_PEER_FAILED_VERIFICATION = 60
CURLE_USE_SSL_FAILED = 64
THIS_KEYBOARD_INTERRUPT = 130

# Global variables
INVALID_HTTP_CODE = 0
TYPE_EC = 408
NODE_TYPE_UNKNOWN = "unknown"
_NODE_TYPE = NODE_TYPE_UNKNOWN
DEFAULT_MAX_REDIRECTS = 50
MAX_EMPTY_READS = 3
NSX_ISSUE = "/etc/nsx_issue"
CURLE_URL_MALFORMAT_ERRSTR = ("URL using bad/illegal format or " +
                              "missing URL")
CURLE_READ_ERRSTR = "Failed to open/read local data from file/application"
CONNECTION_TIMED_OUT_MSG = 'Connection timed out'
CURL_WRAPPER_TAG = 'curl_wrapper'
NUM_WRITE_OUT_WORDS = 3
APPLIANCE_INFO = "/etc/vmware/nsx/appliance-info.xml"
EXCEPTION_TIMEDOUT_MSG = 'timedout'
DEFAULT_TIMEOUT = 20
REST_NODE_TYPES = ['nsx-manager', 'global-manager']
ESX_NODE_TYPES = ['nsx-esx', 'nsx-esxio']
VMWARE_CMD = "/bin/vmware"
OBFUSCATE_STR = "*****"
AUTHORIZATION_HDR = "Authorization"
UTF8 = 'utf-8'
ASCII = 'ascii'

lg = LocalLogger()
nz = NsxZeroize()


def get_node_type():
    """Retrieve node type.
       This is a stripped-down version of nsx_utils.node_utils.get_node_type.
       This function was added so that this script is self contained.
    """

    global _NODE_TYPE
    if _NODE_TYPE != NODE_TYPE_UNKNOWN:
        return _NODE_TYPE
    nsx_issue_node_type = NODE_TYPE_UNKNOWN
    try:
        with open(NSX_ISSUE) as fo:
            lines = fo.readlines()
        for line in lines:
            parts = line.split(":", 1)
            if parts[0].strip() == "node-type" and len(parts) > 1:
                nsx_issue_node_type = parts[1].strip()
                break
    except Exception:
        nsx_issue_node_type = NODE_TYPE_UNKNOWN
    _NODE_TYPE = nsx_issue_node_type
    return nsx_issue_node_type


def _is_appliance_info_valid():
    """Function to determine whether this current node is registered with the
       NSX Manager.  Returns True if the current node is registered and False
       otherwise.
    """
    if not exists(APPLIANCE_INFO):
        return False
    try:
        doc = minidom.parse(APPLIANCE_INFO)
        applianceInfo = doc.getElementsByTagName('appliance-proxy')
        if not len(applianceInfo) > 0:
            return False
        return True
    except Exception:
        pass
    return False


class CmdLineOptions:
    """Command line options.
    """
    def __init__(self):
        self.host = None
        self.port = None
        self.url = None
        self.path = None  # Not an option but parsed from url
        self.cacert = None
        self.connect_timeout = None
        self.data = None
        self.form = None
        self.head = None
        self.header = None
        self.include = None
        self.insecure = None
        self.location = None
        self.max_redirs = None
        self.max_time = None
        self.no_hostname_check = None
        self.output = None
        self.remote_name = None
        self.request = None
        self.retry = None
        self.retry_delay = None
        self.retry_max_time = None
        self.silent = None
        self.show_error = None
        self.thumbprint = None
        self.upload_file = None
        self.user = None
        self.verbose = None
        self.write_out = None
        self.trust_store = []       # Not an option. Store for leaf certs.
        self.num_retry_conns = -1   # Not an option. Count of connections.
        self.errcode = CURLE_OK     # Not an option. Store last errcode.
        self.errstr = ""            # Not an option. Store last errstr.
        self.prev_stdout_line = ""  # Not an option. Store last stdout line.
        self.last_curl_fin = False  # Not an option. Has the last curl command
        # finished.


def _get_host_opt(extra_args):
    """Extract host, port, path, url fields from the command line arguments.
       Return the tuple (errcode, errstr, host, port, path, url) where errcode
       and errstr mimic curl's response.
    """
    host = None
    port = None
    path = None
    url = None
    prev_s = None
    errcode = CURLE_URL_MALFORMAT  # assume this error until proven otherwise
    errstr = CURLE_URL_MALFORMAT_ERRSTR
    for s in extra_args:
        if s.lower().startswith('https://') or s.lower().startswith('http://'):
            if prev_s != '-x':
                try:
                    url_parts = urlparse(s)
                    host = url_parts.hostname
                    port = url_parts.port
                    path = url_parts.path
                    url = s
                except ValueError:
                    return (CURLE_URL_MALFORMAT, errstr, None, None, None,
                            None)
        prev_s = s
    if host:
        if not port:
            port = 80 if url_parts.scheme == "http" else 443
        return (CURLE_OK, None, host, port, path, url)
    if url:
        errcode = CURLE_URL_MALFORMAT
        errstr = CURLE_URL_MALFORMAT_ERRSTR
    else:
        errcode = CURLE_FAILED_INIT
        errstr = "no URL specified!"
    return (errcode, errstr, None, None, None, None)


def _get_cacert_opt(parsed_args):
    """Extract the --cacert option from the curl command line arguments.  The
       cacert option is the filename containing a PEM encoded cert.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'cacert') and v:
            return v
    return None


def _get_cert_opt(parsed_args):
    """Extract the --cert option from the curl command line arguments.  The
       cert option is the filename containing a PEM encoded key.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'cert') and v:
            return v
    return None


def _get_connect_timeout_opt(parsed_args):
    """Extract the --connect-timeout option from the curl command line
       arguments.  Note that the code below uses connect_timeout with an
       underscore rather than a hyphen, but that is just how the argparse
       library works.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'connect_timeout') and v:
            return int(v)
    return 0


def _get_data_opt(parsed_args):
    """Extract the --data (-d) option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'd' or k == 'data') and v:
            return v
    return None


def _get_form_opt(parsed_args):
    """Extract the --form (-F) option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'F' or k == 'form') and v:
            # Returns a list
            return v
    return None


def _get_head_opt(parsed_args):
    """Extract the --head option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'I' or k == 'head') and v:
            return True
    return False


def _get_key_opt(parsed_args):
    """Extract the --key option from the curl command line arguments.  The
       key option is the filename containing a PEM encoded key.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'key') and v:
            return v
    return None


def _get_location_opt(parsed_args):
    """Extract the --location option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'L' or k == 'location') and v:
            return True
    return False


def _get_max_redirs_opt(parsed_args):
    """Extract the --max-redirs option from the curl command line arguments.
       Note that the code below uses max_redirs with an underscore rather than
       a hyphen, but that is just how the argparse library works.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'max_redirs') and v:
            return int(v)
    return None


def _get_max_time_opt(parsed_args):
    """Extract the --max-time option from the curl command line arguments.
       Note that the code below uses max_time with an underscore rather than a
       hyphen, but that is just how the argparse library works.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'm' or k == 'max_time') and v:
            return int(v)
    return 0


def _get_method_opt(parsed_args):
    """Extract the --request option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'X' or k == 'request') and v:
            return v
    return None


def _get_no_hostname_check_opt(parsed_args):
    """Extract the --no-hostname-check option from the curl command line
       arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'no_hostname_check') and v:
            return True
    return False


def _get_output_file_opt(parsed_args):
    """Extract the --output option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'o' or k == 'output') and v:
            return v
    return None


def _get_remote_name_opt(parsed_args):
    """Extract the --remote-name option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'O' or k == 'remote_name') and v:
            return True
    return False


def _get_request_headers_opt(parsed_args):
    """Extract the --header option from the curl command line arguments.
    """
    hdrs = {}
    for k1, v1 in parsed_args.__dict__.items():
        if (k1 == 'H' or k1 == 'header') and v1:
            for item in v1:
                k2, v2 = item.split(':')
                hdrs.update({k2.strip(): v2.strip()})
    return hdrs


def _get_response_headers_opt(parsed_args):
    """Extract the --include option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'i' or k == 'include') and v:
            return True
    return False


def _get_retry_opt(parsed_args):
    """Extract the --retry option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if k == 'retry' and v:
            return int(v)
    return 0


def _get_retry_delay_opt(parsed_args):
    """Extract the --retry-delay option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'retry_delay') and v:
            return int(v)
    return 0


def _get_retry_max_time_opt(parsed_args):
    """Extract the --retry-max-time option from the curl command line
       arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'retry_max_time') and v:
            return int(v)
    return 0


def _get_show_error_opt(parsed_args):
    """Extract the --show-error option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'S' or k == 'show_error') and v:
            return True
    return False


def _get_silent_opt(parsed_args):
    """Extract the --silent option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 's' or k == 'silent') and v:
            return True
    return False


def _get_thumbprint_opt(parsed_args):
    """Extract the --thumbprint option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if k == 'thumbprint' and v:
            return v
    return 0


def _get_upload_opt(parsed_args):
    """Extract the --upload-file (-T) from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'T' or k == 'upload_file') and v:
            return v
    return None


def _get_user_passwd_opt(parsed_args):
    """Extract the --user option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'u' or k == 'user') and v:
            return v
    return None


def _get_verbose_opt(parsed_args):
    for k, v in parsed_args.__dict__.items():
        if (k == 'v' or k == 'verbose') and v:
            return True
    return False


def _get_write_out_opt(parsed_args):
    """Extract the --write-out option from the curl command line arguments.
    """
    for k, v in parsed_args.__dict__.items():
        if (k == 'w' or k == 'write_out') and v:
            return v
    return None


def _get_leaf_cert(cert_chain):
    """Return the leaf cert from the chain.  The leaf cert is the first cert in
       the chain.
    """
    return cert_chain[0]


def _log_trying_connection(options, host, port, with_httplib=True):
    """Curl prints the "Trying" message to stderr in verbose mode.
       Trying 10.185.21.158:443...
       On NSX the this script prints to stderr and to syslog.
    """
    with_str = "(with httplib)" if with_httplib else "(with curl)"
    logmsg = ("Trying " + with_str + " " + host + ":"
              + str(port) + "...")
    lg.info(logmsg)
    _print_msg_stderr(options, "*   " + logmsg)


def _log_command(options, cmd_args_in, obfuscate_secrets=False):
    """Log commands invoked by this script.  Typically cmd_args will be an
       array passed to subprocess.Popen().  No secrets should be logged and in
       cases where there are secrets in command arguments, this function should
       be called with obfuscate_secrets=True.
    """
    cmd_args = cmd_args_in
    if obfuscate_secrets:
        cmd_args = list(cmd_args_in)
        # Shallow copy. Therefore don't zeroize items with secrets.
        # Credentials are stored in the -u command line option and in the
        # Authorization header.
        i = 0
        while i < len(cmd_args):
            if cmd_args[i] == '-u' and i < len(cmd_args)-1:
                if ':' in cmd_args[i+1]:
                    parts = cmd_args[i+1].split(':')
                    cmd_args[i+1] = parts[0] + ':' + OBFUSCATE_STR
            elif cmd_args[i] == '-H' and i < len(cmd_args)-1:
                if (AUTHORIZATION_HDR.lower() in cmd_args[i+1].lower() and
                        ':' in cmd_args[i+1]):
                    # The Authorization header looks like:
                    # Authorization: 'Basic %s' % username_passwd
                    parts = cmd_args[i+1].split(':')
                    parts[0] = parts[0].strip()
                    part1 = parts[1]
                    if 'basic' in part1.lower() or 'remote' in part1.lower():
                        words = part1.split()
                        cmd_args[i+1] = (parts[0] + ': ' + words[0] + ' ' +
                                         OBFUSCATE_STR)
                    else:
                        cmd_args[i+1] = parts[0] + ': ' + OBFUSCATE_STR
            i += 1
    cmd_str = str(cmd_args)
    if len(cmd_str) > 0 and cmd_str[0] == '[':
        cmd_str = cmd_str[1:]
    if len(cmd_str) > 0 and cmd_str[-1] == ']':
        cmd_str = cmd_str[:-1]
    if len(cmd_str) > 0:
        lg.info("Calling " + cmd_str)


def _log_exit_code(options, cmd_args, exitcode):
    lg.info(sys.argv[0] + " exit code " + str(exitcode))


def _log_connection_timed_out(options, logmsg):
    """Log curl's "Connection timed out" message
    """
    lg.info(logmsg)
    _print_msg_stderr(options, "* " + logmsg)


def _log_closing_connection(options):
    """Log curl's "Closing connection" stderr message.
    """
    logmsg = ("Closing connection "
              + str(options.num_retry_conns))
    lg.info(logmsg)
    _print_msg_stderr(options, "* " + logmsg)


def _log_transient_problem(options, sleep_time, retry):
    """Log curl's "Warning: Transient problem:" stderr message.
    """
    logmsg = ("Warning: Transient problem:  Will retry in " + str(sleep_time)
              + " seconds. " + str(retry) + " retries left.")
    lg.info(logmsg)
    _print_msg_stderr(options, logmsg)


def _log_backtrace(options, logmsg, backtrace):
    """Log backtrace for debugging purposes.  Curl doesn't log this message so
       this script writes it to the log file, not stderr.
    """
    logmsg = ("Logging backtrace for analysis: " + logmsg + " " + backtrace)
    lg.info(logmsg)


def _log_openssl_timedout(options):
    """Log openssl timed out message.  Curl doesn't log this message so this
       script writes it to the log file, not stderr.
    """
    lg.info("openssl timed out; timeout returned 124")


def _log_cert_verification_result(options, logmsg):
    """Log result of cert verification.  Curl doesn't log this message but it
       is important so we also write it to stderr.
    """
    lg.info(logmsg)
    _print_msg_stderr(options, "* " + logmsg)


def _cert_has_expired(cert):
    """There is a bug in the has_expired() function in the python
       OpenSSL.crypto.X509 library used on ESX versions 6, 7, and 8 and
       therefore we use our own logic in this function.  This function is
       similar to OpenSSL.crypto.X509.has_expired() in that it returns True if
       the cert has expired and False otherwise.  More recently the /bin/vmware
       command has been deprecated so we have removed esx-specific logic.
       Tested on VMware ESXi 6.7.0 build-14320388.
       Tested on VMware ESXi 7.0.3 build-18644231.
       Tested on VMware ESXi 8.0.0 build-21203435.
    """
    if not exists('/usr/bin/curl'):
        # This is an approximate way of determine whether running on ESX.
        not_after = cert.get_notAfter().decode(UTF8)
        # get_notAfter returns time as ASN.1 TIME YYYYMMDDhhmmssZ
        not_after_date = datetime.strptime(not_after, '%Y%m%d%H%M%SZ')
        return not_after_date <= datetime.now()
    else:
        return cert.has_expired()


def _validate_common(cert):
    """Used to validate the leaf cert.  This function is called by both
       _validate_self_signed_cert and _validate_ca_signed_cert and handles all
       common validation logic for these two cert validation functions.
       Returns the tuple (errcode, errstr).  If the cert is validated
       successfully, this function returns (0, None).  Otherwise this function
       returns a non-zero errcode and a non-None errstr.  This function is
       used to validate the leaf cert only.
    """
    if _cert_has_expired(cert):
        return (CURLE_PEER_FAILED_VERIFICATION, "certificate has expired")
    pubkey = cert.get_pubkey()
    if pubkey.type() == crypto.TYPE_RSA:
        if pubkey.bits() < 2048:
            return (CURLE_PEER_FAILED_VERIFICATION,
                    "RSA certificate key length less than 2048")
    elif pubkey.type() == TYPE_EC:
        if pubkey.bits() < 256:
            return (CURLE_PEER_FAILED_VERIFICATION,
                    "EC certificate key length less than 256")
    else:
        return (CURLE_PEER_FAILED_VERIFICATION,
                "certificate neither RSA nor EC")
    for i in range(cert.get_extension_count()):
        ext = cert.get_extension(i)
        # Removed CA:TRUE basic contraint check because the check_trusted
        # API already checks for it.
        if ext.get_short_name() == b'extendedKeyUsage':
            if "Server" not in ext.__str__():
                return (CURLE_PEER_FAILED_VERIFICATION,
                        "certificate is not server certificate")
    return 0, None


def _canonical_thumbprint(thumbprint):
    """Converts to canonical form so that thumbprints can be compared.
    """
    return thumbprint.lower().replace(':', '')


def _validate_trust(cert_chain, options):
    """Establishes trust based on the cert chain received by the server.
       Either a thumbprint is used (the --thumbprint option) or the cacert
       file is used (the --cacert option).  Returns the tuple (errcode,
       errstr).  If trust can be established, this function returns (0,
       None).  Otherwise this function returns a non-zero errcode and an
       non-None errstr.
    """
    if options.thumbprint:
        leaf_cert = _get_leaf_cert(cert_chain)
        digest = leaf_cert.digest("sha256").decode(UTF8)
        if (_canonical_thumbprint(options.thumbprint) !=
                _canonical_thumbprint(digest)):
            return (CURLE_PEER_FAILED_VERIFICATION,
                    ("curl_wrapper failed to verify the legitimacy of the "
                     "server because the given thumbprint " +
                     options.thumbprint + " "
                     "didn't match the certificate's "
                     + _canonical_thumbprint(digest) + "."))
        return 0, None
    elif options.cacert:
        # cacert is the filename containing a certificate
        try:
            cert_chain = _read_cert_chain_from_file(options.cacert)
            thecacert = _get_leaf_cert(cert_chain)
            cacert_digest = thecacert.digest("sha256")
            for cert in cert_chain:
                # Doing equals without digest() doesn't work.
                if cert.digest("sha256") == cacert_digest:
                    return 0, None
        except Exception:
            pass
    return (CURLE_PEER_FAILED_VERIFICATION,
            ("curl_wrapper failed to verify the legitimacy of the server and "
             "therefore could not establish a secure connection to it. "
             "Use the --thumbprint or --cacert option."))


def _validate_self_signed_cert(cert_chain, options):
    """Validate a self-signed cert. The cert_chain parameter is used for
       consistency, but only a single cert is expected (and enforced in this
       function) in the cert chain.  Returns the tuple (errcode, errstr).  If
       cert is validated successfully, this function returns (0, None).
       Otherwise this function returns a non-zero errcode and a non-None
       errstr.
    """
    leaf_cert = _get_leaf_cert(cert_chain)
    (errcode, errstr) = _validate_common(leaf_cert)
    if errcode:
        return (errcode, errstr)
    if len(cert_chain) > 1:
        return (CURLE_PEER_FAILED_VERIFICATION,
                "found self-signed certificate with certificate chain")
    (errcode, errstr) = _validate_trust(cert_chain, options)
    if errcode:
        return (errcode, errstr)
    return (errcode, errstr)


class SimpleParseContext:
    """Used by validate_hostname to validate hostnames. Use a "context" object
       to avoid global variables.
    """
    def __init__(self):
        self.errstr = None
        self.data = None
        self.ptr = 0
        self.parsed_data = []


def _read_octet(context):
    """Used by validate_hostname to validate hostnames.  Returns the next
       token in the ASN.1 string.  For a description of ASN.1 see
       https://luca.ntop.org/Teaching/Appunti/asn1.html
    """
    try:
        octet = context.data[context.ptr]
        # For Python 2 context.data is a string whereas for Python 3 it is a
        # buffer. Ensure that an integer is returned in both cases.
        if not have_py3:
            # Python 2
            octet = ord(octet)
        context.ptr += 1
        return octet
    except IndexError:
        return -1


def _read_length(context):
    """Used by validate_hostname to validate hostnames.  Returns the tuple
       (length of the ASN.1 object, length of the ASN.1 object including
       header).
    """
    octet1 = _read_octet(context)
    if octet1 < 0:
        return (-1, -1)
    if octet1 < 0x80:
        # + 1 because we read 1 octet: octet1
        return (octet1, octet1 + 1)
    # 0x80 exactly: means the length is "indefinite" (not supported here)
    if octet1 == 0x81:  # means the length is stored in one octet
        octet2 = _read_octet(context)
        if octet2 < 0:
            return (-1, -1)
        value = octet2
        # + 2 because we read 2 octets: octet1, octet2
        return (value, value + 2)
    if octet1 == 0x82:  # means the length is stored in two octets
        octet2 = _read_octet(context)
        if octet2 < 0:
            return (-1, -1)
        octet3 = _read_octet(context)
        if octet3 < 0:
            return (-1, -1)
        value = octet2 * 256 + octet3
        # + 3 because we read 3 octets: octet1, octet2, octet3
        return (value, value + 3)
    # unsupported
    return (-1, -1)


def _canonical_ipaddress(ip_addr):
    """Given an IP address as a string, typically an IPv6 address, return the
       canonical form of this IP address.  Note that IPv4 addressses like
       10.04.3.2 (with a leading 0) are not considered valid IPv4 addresses by
       the Python3 library ipaddress or by inet_pton.  For example,
       ipaddress.ip_address("10.04.3.2") will raise a ValueError.
       This function works in Python2 and Python3.
    """
    try:
        _str = socket.inet_pton(socket.AF_INET6, ip_addr)
        a, b = struct.unpack('!2Q', _str)
        return socket.inet_ntop(socket.AF_INET6, struct.pack('!2Q', a, b))
    except socket.error:
        pass
    try:
        _str = socket.inet_pton(socket.AF_INET, ip_addr)
        n = struct.unpack('!I', _str)[0]
        return socket.inet_ntop(socket.AF_INET, struct.pack('!I', n))
    except socket.error:
        pass
    return ip_addr


def _read_dns_name(context):
    """Used by validate_hostname to validate hostnames.
    """
    ALT_DNS = 130
    ALT_IP = 135
    name = ""
    total_length = 0
    type = _read_octet(context)  # read the type
    total_length += 1
    (length, length_with_hdr) = _read_length(context)
    if length < 0:
        context.errstr = ("Unexpected ASN.1 length at position " +
                          str(context.ptr))
        return -1
    total_length += length_with_hdr
    i = length
    while i > 0:
        char = _read_octet(context)
        if char < 0:
            context.errstr = ("Unexpected ASN.1 character at position " +
                              str(context.ptr))
            return -1
        if type == ALT_DNS:
            # build DNS name
            name += chr(char)
        elif type == ALT_IP:
            # build dotted decimal IP address
            if name:
                name += "."
            name += str(char)
        i -= 1
    if type == ALT_DNS or type == ALT_IP:
        if type == ALT_IP and name.count('.') > 4:
            octets = name.split('.')
            name = ""
            count = 0
            for octet in octets:
                if name and count % 2 == 0:
                    name += ":"
                # Remove leading "0x" with [2:]
                name += hex(int(octet))[2:].zfill(2)
                count += 1
            name = _canonical_ipaddress(name)
        context.parsed_data.append(name)
    return total_length


def _parse_subject_alt(context):
    """Used by validate_hostname to validate hostnames.  Writes output to
       context.parsed_data.
    """
    type = _read_octet(context)  # read the type
    if type != 48:
        context.errstr = ("Unexpected ASN.1 type.  Expected 48 but read " +
                          str(type) + " at position " + str(context.ptr))
        return -1
    (length, length_with_hdr) = _read_length(context)
    if length < 0:
        context.errstr = ("Unexpected ASN.1 length at position " +
                          str(context.ptr))
        return -1
    rem_len = length
    while rem_len > 0:
        length = _read_dns_name(context)
        if length < 0:
            context.errstr = ("Unexpected DNS name at position " +
                              str(context.ptr))
            return -1
        rem_len -= length
    return 0


def _is_valid_dns_entry(dnsname):
    """This function filters out incorrectly specified hostname in certs, to
       avoid matching the hostname in cases where the cert is improper.  This
       might not occur in the wild, but just being cautious.  Returns True if
       the DNS name in the cert is proper and False otherwise.
    """
    # See rules in https://en.wikipedia.org/wiki/Wildcard_certificate
    if not dnsname:
        return False
    if '*' not in dnsname:
        return True
    if dnsname.count('*') > 1:
        # A cert with multiple wildcards in a name is not allowed.
        # *.*.domain.com
        return False
    if dnsname.count('.') == 1:
        # A cert with * plus a top-level domain is not allowed.
        # *.com
        return False
    if dnsname == '*':
        # Too general and should not be allowed.
        # *
        return False
    if dnsname[0] != '*':
        # All major browsers have deliberately removed support for
        # partial-wildcard certificates. In other words, the '*' must be the
        # first character if the '*' is present.
        return False
    return True


def _set_err_stderr(options, errcode, errstr):
    """Sets variables to be used by the _print_final_msgs() function.
    """
    options.errcode = errcode
    options.errstr = errstr


def _print_final_msgs(options, stats, without_prefix=False):
    """Used to print final error message to stderr and the write-out message to
       stdout.  Once options.last_curl_fin is True don't print any more
       messages to stderr so that curl has the last say.
    """
    if not options.silent or options.verbose or options.show_error:
        # Final stderr message
        if options.errstr and not options.last_curl_fin:
            if not without_prefix:
                errmsg = ("curl_wrapper: (" + str(options.errcode) + ") " +
                          options.errstr)
            else:
                errmsg = options.errstr
            print(errmsg, file=sys.stderr)
            sys.stderr.flush()
    if stats:
        # Final stdout message
        _print_write_out(options, stats)


def _print_msg_stderr(options, msg):
    """Used to print verbose error message to stderr.
       Once options.last_curl_fin is True don't print any more messages
       to stderr so that curl has the last say.
    """
    # verbose trumps silent
    if options.verbose and not options.last_curl_fin:
        print(msg, file=sys.stderr)
        sys.stderr.flush()


def _hostname_check(dnsname, hostname):
    """Low-level hostname check that takes the 'dnsname' that comes from the
       cert, and 'hostname' which comes from the URL given as input to this
       script.  Returns True if there is a match and False otherwise.
    """
    dnsname = dnsname.lower()
    hostname = hostname.lower()
    if dnsname[0] == '*':
        # The wildcard applies only to one level of the domain name.
        return (dnsname.count('.') == hostname.count('.') and
                hostname.endswith(dnsname[1:]))
    elif dnsname == hostname:
        return True
    elif _is_ipv6_address(hostname):
        return dnsname == _canonical_ipaddress(hostname)
    return False


def _validate_hostname(cert, hostname, options):
    """Perform the hostname check that curl does.  If the hostname in the
       cert's subject doesn't match then iterate through the cert's Subject
       Alternative Name fields.  It may be possible to use the httplib library
       to achieve this so we may be able to remove this code in future.
       Returns the tuple (errcode, errstr).  If the hostname is validated
       successfully, this function returns (0, None).  Otherwise this function
       returns a non-zero errcode and a non-None errstr.
    """
    if options.no_hostname_check:
        return 0, None
    # iterate through common names, typically only one common name
    for comp in cert.get_subject().get_components():
        if comp[0].decode(UTF8) == 'CN':
            cn = comp[1].decode(UTF8)  # common name
            if _is_valid_dns_entry(cn) and _hostname_check(cn, hostname):
                return 0, None
    # iterate through subject alternative names
    ctx = SimpleParseContext()
    dns_names = []
    for i in range(cert.get_extension_count()):
        ext = cert.get_extension(i)
        if ext.get_short_name() == b'subjectAltName':
            # For Python 2 ext.get_data() is a string whereas for Python 3 it
            # is a buffer.  This difference is handled by _read_octet().
            ctx.data = ext.get_data()
            ret = _parse_subject_alt(ctx)
            if ret == -1:
                # Just log parse errors and proceed
                _print_msg_stderr(options, ctx.errstr)
            dns_names = ctx.parsed_data
    for dnsname in dns_names:
        if _is_valid_dns_entry(dnsname) and _hostname_check(dnsname, hostname):
            return 0, None

    errcode = THIS_NO_ALTERNATIVE_CERTIFICATE_SUBJECT_NAME
    ctx.errstr = ("SSL: no alternative certificate subject name matches " +
                  "target host name '" +
                  hostname + "'")
    return (errcode, ctx.errstr)


def _get_int_status(response):
    """Returns integer HTTP status from response object.
       Copied from backup_restore.py.

    Args:
        response: Either http.client.HTTPResponse or webob.response.Response
    Returns:
        webob.response.Response.status_int or http.client.HTTPResponse.status
    """
    if hasattr(response, "status_int"):
        return response.status_int
    return response.status


def _replace_newlines(str):
    """Replace the newline character with the two characters '\' and 'n' so
       that the resultant string can be included in a JSON body.
    """
    return '\\n'.join(str.splitlines()) + '\\n'


def _validate_crl_over_rest(cert_chain, options):
    """This function uses proton to do the actual CRL check.  This function
       makes a REST call to proton and provides the cert chain. If the CRL
       check passes, this function returns (0, None).  Otherwise this function
       returns a non-zero errcode and a non-None errstr.
    """
    errcode = CURLM_INTERNAL_ERROR
    errstr = "unknown"
    conn = None
    try:
        pem_data = ''
        # The check_trusted call below assumes the leaf cert comes first.
        for cert in cert_chain:
            pem_data += _replace_newlines(
                crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode(
                    UTF8))

        timeout = _get_min_timeout(options)
        host = '127.0.0.1'
        port = 7440
        method = 'POST'
        path = ('/nsxapi/api/v1/trust-management/certificates?'
                'action=check_trusted&crl_check=true')
        data = '{"pem_encoded": "' + pem_data + '", "cert_type": "SERVER"}'
        if options.verbose:
            _print_msg_stderr(options, 'Calling ' + path + ' with payload:')
            _print_msg_stderr(options, data)

        hdrs = {}
        hdrs.update({'Content-Type': 'application/json'})
        hdrs.update({'X-NSX-Username': 'admin'})

        conn = httplib.HTTPConnection(host, port, timeout=timeout)
        conn.request(method, path, data, hdrs)
        resp = conn.getresponse()
        resp_status = _get_int_status(resp)
        body = resp.read()
        if resp_status != httplib.OK:
            # Use CURLE_COULDNT_CONNECT to allow a retry
            errcode = CURLE_COULDNT_CONNECT
            errstr = ('Response for /nsxapi/api/v1/trust-management'
                      '/certificates?action=check_trusted&crl_check=true '
                      'was ' + str(resp_status) + ' but ' +
                      str(httplib.OK.value) + ' is expected')
        else:
            resp_obj = json.loads(body)
            app_status = resp_obj.get('status')
            if app_status == 'REJECTED' or app_status == 'ERROR':
                errcode = THIS_CRL_CHECK_FAILED
                errstr = resp_obj.get('error_message')
            else:
                errcode = CURLE_OK
                errstr = None
    except Exception as ex:
        errcode = CURLE_COULDNT_CONNECT
        submsg = str(ex)
        errstr = ("Failed to connect to " + host + " port " +
                  str(port) + ": " + submsg)
        _log_backtrace(options, errstr,
                       traceback.format_exc())
    finally:
        if conn:
            conn.close()
            conn = None
    return errcode, errstr


def _get_nsx_proxy_rpc_connection():
    """This function returns a connection object to the local RPC service.  It
       uses the RPC service tcp://127.0.0.1:9004" for ESX and
       "unix:///var/run/vmware/nsx-proxy/aphinfoservice.sock" for Edge and
       Windows. The function returns a connection object if successful and None
       otherwise.
    """
    if exists("/var/run/vmware/nsx-proxy/aphinfoservice.sock"):
        provider_conn = ("unix:///var/run/vmware/nsx-proxy/"
                         "aphinfoservice.sock")
    else:
        provider_conn = "tcp://127.0.0.1:9004"
    try:
        conn = NsxRpcConnection()
        conn.Connect(provider_conn)
        return conn
    except Exception:
        pass
    return None


def _bypass_crl_check_for_url(options):
    """There are several cases where RPC response will not be available and
       therefore the CRL check can't be performed. Bypass the CRL check for
       these APIs.
    """
    bypass = False
    bypass_list = ['/api/v1/transport-nodes?action=register_node',
                   '/api/v1/cluster/nodes/',
                   '/api/v1/messaging/clients/',
                   '/api/v1/fabric/nodes/']
    path = ''
    try:
        # Shouldn't fail here because this URL parsing has been done earlier.
        # If for some reason there is a URL parsing problem, then don't bypass
        # the CRL check.
        url_parts = urlparse(options.url)
        path = url_parts.path
        if url_parts.query:
            path = path + '?' + url_parts.query
    except ValueError:
        pass
    for api in bypass_list:
        if path.startswith(api):
            if api == '/api/v1/fabric/nodes/':
                if path.endswith('?action=register_node'):
                    bypass = True
            else:
                bypass = True
            break
    if bypass:
        logmsg = ("Skipping CRL check for special API " + path)
        _print_msg_stderr(options, logmsg)
    return bypass


def _validate_crl_over_nsxrpc(cert_chain, options):
    """This function is used when the node is not a Manager. The function
       forwards the CRL check to the registered Manager. This function needs to
       also work in the install use-case on ESX where the NSX vibs are not yet
       installed, and in this case the function returns success to mimic a
       successful CRL check.  If the CRL check passes, this function returns
       (0, None).  Otherwise this function returns a non-zero errcode and a
       non-None errstr.
    """
    if not _is_appliance_info_valid():
        # This node is not yet registered with a Manager and therefore
        # it is not possible to check the CRL. Return success to mimic
        # that the CRL check passed.
        logmsg = ("Skipping CRL check because node isn't "
                  "registered with the manager")
        _print_msg_stderr(options, logmsg)
        return 0, None
    if _bypass_crl_check_for_url(options):
        # this function also calls _print_msg_stderr()
        return 0, None
    pem_data = ''
    # The check_trusted call below assumes the leaf cert comes first.
    for cert in cert_chain:
        pem_data += _replace_newlines(
            crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode(
                UTF8))

    response = None
    if not have_nsx:
        # NSX hasn't been installed yet and therefore it is not possible to
        # check the CRL. Return success to mimic that the CRL check passed.
        logmsg = ("Skipping CRL check because don't have NSX installed or "
                  "don't have suitable NSX version")
        _print_msg_stderr(options, logmsg)
        return 0, None
    appl_info_param = ApplProxyInfoReqMsg()
    conn = None
    try:
        conn = _get_nsx_proxy_rpc_connection()
        if not conn:
            # Use CURLE_COULDNT_CONNECT to allow a retry
            errcode = CURLE_COULDNT_CONNECT
            errstr = "Unable to connect to RPC service"
            return errcode, errstr
        with NsxRpcClient(ApplProxyInfoService_Stub,
                          connection=conn) as nsx_rpc_client:
            get_appl_info = eval("nsx_rpc_client.GetApplProxyInfo")
            response = get_appl_info(appl_info_param)
    except Exception:
        pass
    finally:
        if conn is not None:
            conn.Close()
    if not response:
        # Use CURLE_COULDNT_CONNECT to allow a retry
        errcode = CURLE_COULDNT_CONNECT
        errstr = "Unable to invoke GetApplProxyInfo RPC call"
        return errcode, errstr
    aph_id = 0
    for info in response.applproxy_info:
        if not info.HasField("id"):
            continue
        aph_id = uuid.UUID(int=((info.id.left << 64) + info.id.right))
        if info.master:
            break
    if not aph_id:
        # Use CURLE_COULDNT_CONNECT to allow a retry
        errcode = CURLE_COULDNT_CONNECT
        errstr = "No APH UUID found in CheckTrusted RPC response"
        return errcode, errstr

    check_trusted_param = CheckTrustedRequestMsg()
    check_trusted_param.crl_check = True
    _SERVER = CheckTrustedRequestMsg.CertificateType.Value("SERVER")
    check_trusted_param.cert_type = _SERVER
    s_ok = CheckTrustedResponseMsg.CertificateCheckStatus.Value("OK")
    check_trusted_param.pem_encoded = pem_data
    provider_connection = "tcp://127.0.0.1:4096"
    provider_endpoint = str(aph_id)
    response = None
    conn = None
    try:
        conn = NsxRpcConnection()
        conn.Connect(provider_connection)
        with NsxRpcClient(CertificateService_Stub, connection=conn,
                          destination=provider_endpoint) as nsx_rpc_client:
            check_trusted_rpc = eval("nsx_rpc_client.CheckTrusted")
            response = check_trusted_rpc(check_trusted_param)
    except Exception:
        pass
    finally:
        if conn is not None:
            conn.Close()
    if not response:
        # Use CURLE_COULDNT_CONNECT to allow a retry
        errcode = CURLE_COULDNT_CONNECT
        errstr = "Unable to invoke CheckTrusted RPC call"
        return errcode, errstr

    status = response.status
    s_ok = CheckTrustedResponseMsg.CertificateCheckStatus.Value("OK")
    s_crl_not_ready = CheckTrustedResponseMsg.CertificateCheckStatus.Value(
        "CRL_NOT_READY")
    if status != s_ok and status != s_crl_not_ready:
        errcode = THIS_CRL_CHECK_FAILED
        errstr = response.error_message
        return errcode, errstr
    return 0, None


def _validate_crl(cert_chain, options):
    """Perform a CRL check using the cert chain from the remote server as
       input. If the CRL check passes, this function returns (0, None).
       Otherwise this function returns a non-zero errcode and a non-None
       errstr.
    """
    node_type = get_node_type().split()[0]
    if node_type in REST_NODE_TYPES:
        return _validate_crl_over_rest(cert_chain, options)
    else:
        return _validate_crl_over_nsxrpc(cert_chain, options)


def _validate_ca_signed_cert(cert_chain, hostname, options):
    """Validate a ca-signed cert.  Either the --cacert or --thumbprint option
       must be present otherwise trust will not be established and an error
       will be returned.  Returns the tuple (errcode, errstr).  If cert is
       validated successfully, this function returns (0, None).  Otherwise this
       function returns a non-zero errcode and a non-None errstr.
    """
    leaf_cert = _get_leaf_cert(cert_chain)
    (errcode, errstr) = _validate_common(leaf_cert)
    if errcode:
        return (errcode, errstr)
    (errcode, errstr) = _validate_hostname(leaf_cert, hostname, options)
    if errcode:
        return (errcode, errstr)
    (errcode, errstr) = _validate_trust(cert_chain, options)
    if errcode:
        return (errcode, errstr)
    return _validate_crl(cert_chain, options)


def _validate_cert(cert_chain, hostname, options):
    """Top-level function to validate a cert chain received from the remote
       server. The cert chain could be either a single self-signed cert or a
       CA-signed cert and its cert chain.  Either the --cacert or --thumbprint
       option must be present otherwise trust will not be established and an
       error will be returned.  Returns the tuple (is_self_signed, errcode,
       errstr).  If cert is validated successfully, this function returns
       (is_self_signed, 0, None).  Otherwise this function returns a non-zero
       errcode and a non-None errstr.
    """
    # Assume CA-signed until known otherwise.
    is_self_signed = False
    if len(cert_chain) == 0:
        errcode = CURLE_PEER_FAILED_VERIFICATION
        errstr = "No certificate provided"
        thumbprint = "(no thumbprint)"
    else:
        leaf_cert = _get_leaf_cert(cert_chain)
        is_self_signed = leaf_cert.get_subject() == leaf_cert.get_issuer()
        if is_self_signed:
            (errcode, errstr) = _validate_self_signed_cert(cert_chain,
                                                           options)
        else:
            (errcode, errstr) = _validate_ca_signed_cert(cert_chain, hostname,
                                                         options)
        thumbprint = _canonical_thumbprint(leaf_cert.digest("sha256").
                                           decode(UTF8))
    logmsg = ("certificate verification " + thumbprint + " from " +
              options.host + ":" + str(options.port) + " ")
    if errcode == 0:
        if have_nsx:
            logmsg += "passed"
        else:
            logmsg += "passed (unable to perform full verification)"
    else:
        logmsg += "failed: " + errstr
    _log_cert_verification_result(options, logmsg)
    return is_self_signed, errcode, errstr


def _is_transient_error(errcode, httpcode):
    """Return True if the errorcode and httpcode are considered a transient
       error as documented by the curl man page.  The comment below about
       transient errors comes from the curl man page under the --retry option.
       Transient error means either: a timeout, an FTP 4xx response code or an
       HTTP 408, 429, 500, 502, 503 or 504 response code.
    """
    if errcode in [CURLE_COULDNT_CONNECT, CURLE_OPERATION_TIMEDOUT,
                   CURLE_PARTIAL_FILE]:
        return True
    if httpcode in [408, 429, 500, 502, 503, 504]:
        return True
    if httpcode >= 400 and httpcode < 500:
        return True
    return False


def _is_ipv6_address(host):
    """Return True if host is an IPv6 address.  Don't use ipaddress import
       because that package exists only for Python3.
    """
    return True if ':' in host else False


def _should_try_ipv6(host, port):
    """Check if either the host is an IPv6 address or if there is an IPv6 DNS
       entry. Returns True if host is an IPv6 address or DNS has an IPv6 entry
       for the given host, otherwise return False.  The way we use
       getaddrinfo() only works in Python3, so when the host parameter is a
       domain name this function (_should_try_ipv6) returns False.  The
       calling function then calls _get_peer_cert_chain_with_openssl as a
       fallback.
    """
    # If the host is an IPv4 address and not a hostname then don't use IPv6
    if _is_ipv6_address(host):
        return True
    try:
        entries = socket.getaddrinfo(host, port, family=socket.AF_INET6,
                                     proto=socket.IPPROTO_TCP)
        if len(entries) > 0:
            return True
    except Exception:
        pass
    return False


def _get_peer_cert_chain_with_sock_type(options, host, port, socket_type):
    """Return the cert chain from the host:port and given socket_type (IPv4 or
    IPv6).  Returns the cert chain or throws an Exception on error.  The leaf
    cert is the first cert in cert chain.
    """
    context = SSL.Context(method=SSL.SSLv23_METHOD)
    # When changing the method need to test on Manager and ESXi.
    # Works on VMware ESXi 7.0.3 build-18644231
    # Works on VMware ESXi 6.7.0 build-14320388
    conn = None
    try:
        conn = SSL.Connection(context,
                              socket=socket.socket(socket_type,
                                                   socket.SOCK_STREAM))
        conn.connect((host, port))
        conn.setblocking(1)
        conn.do_handshake()
        return conn.get_peer_cert_chain()
    except Exception as ex:
        raise ex
    finally:
        if conn:
            conn.close()


def _read_cert_chain_from_openssl_output(text_with_pem_data):
    """Read cert chain from openssl output.  The leaf cert is the first cert in
       cert chain.
    """
    cert_chain = []
    text = str(text_with_pem_data)
    start_line = '-----BEGIN CERTIFICATE-----'
    cert_slots = text.split(start_line)
    for single_pem_cert in cert_slots[1:]:
        # The single_pem_cert will typically include the END CERTIFICATE header
        # and it may include other text after that (because of the way openssl
        # displays cert chains), however the load_certificate handles any extra
        # text without complaint.
        cert = crypto.load_certificate(crypto.FILETYPE_PEM,
                                       start_line+single_pem_cert)
        cert_chain.append(cert)
    return cert_chain


def _get_peer_cert_chain_with_openssl(options, host, port):
    """Used as a fallback when _get_peer_cert_chain_with_sock_type() doesn't
       work.  It was found that for some servers,
       _get_peer_cert_chain_with_sock_type() doesn't work on ESXi.  This
       function assumes that the OpenSSL library has been loaded.  The timeout
       in seconds is used as the time for openssl to complete.  The function
       returns the cert chain if successful, otherwise throws an Exception if
       error.  The function throws a Exception(EXCEPTION_TIMEDOUT_MSG) on a
       timeout.
    """
    # openssl doesn't have a parameter to specify the connection timeout.
    # So select the smaller of --max-time and --connect-timeout on the basis
    # that any time spent waiting for openssl to respond will likely be due
    # to the connection time.
    timeout = _get_min_timeout(options)
    output = ""
    cmd_args = ["openssl", "s_client", "-showcerts", "-servername", host,
                "-connect", host + ":443"]
    if have_py3:
        try:
            # MacOS doesn't like DEVNULL for stdin, whereas PIPE appears to
            # work for MacOS and Ubuntu.  The advantage of check_output() over
            # Popen() is that check_output() in Python 3 supports a timeout
            # parameter.
            _log_command(options, cmd_args)
            output = subprocess.check_output(cmd_args, stdin=subprocess.PIPE,
                                             stderr=subprocess.DEVNULL,
                                             timeout=timeout)
        except TimeoutExpired:
            _log_backtrace(options, "openssl timed out",
                           traceback.format_exc())
            # Our special exception used for both Python 2 and Python 3.
            raise Exception(EXCEPTION_TIMEDOUT_MSG)
    else:
        if exists("/usr/bin/timeout"):
            cmd_args = ["/usr/bin/timeout", str(timeout)] + cmd_args
        # TimeoutExpired is not defined in Python 2 so resort to the timeout
        # command.  subprocess.DEVNULL doesn't exist in Python 2.
        # check_output() doesn't work with "timeout=timeout" and with the
        # stdin= parameter when we have connectivity to the remote server.
        # Conversely, check_output() doesn't work without stdin= parameters
        # when we don't have network connectivity. Therefore check_output()
        # doesn't work in all cases.  Fortunately, Popen works in both
        # these cases.
        _log_command(options, cmd_args)
        df = subprocess.Popen(cmd_args, stdin=subprocess.PIPE,
                              stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE)
        output = df.communicate()[0]
        if df.returncode == 124:
            # Assumption is that the 124 return code was returned by timeout
            # because openssl doesn't return 124.
            _log_openssl_timedout(options)
            raise Exception(EXCEPTION_TIMEDOUT_MSG)
        # We don't care about other returns codes because if there is any
        # problem output will be the empty string "". This will cause
        # _read_cert_chain_from_openssl_output() to return an empty cert list
        # and this will result in a "no certificate" error further up the call
        # stack.
    return _read_cert_chain_from_openssl_output(output)


def _get_peer_cert_chain(options, host, port):
    """This function retrieves the cert chain from the remote host using
       a different connection that will be used after this to perform the curl
       operation. This function throws AttributeError when the SSL method is
       not supported the python intepreter.  Note that although AttributeError
       is caught for the first call to _get_peer_cert_chain_with_sock_type(),
       but not the second and that is why this function might throw
       AttributeError.  This function throws TimeoutExpired on timeout. This
       function throws KeyboardInterrupt if user presses CONTROL-C.
    """
    if _should_try_ipv6(host, port):
        socket_type = socket.AF_INET6
        try:
            return _get_peer_cert_chain_with_sock_type(options, host, port,
                                                       socket_type)
        except AttributeError:
            pass
        # Whenever we catch Exception, always catch KeyboardInterrupt prior to
        # catching Exception and throw it.
        except KeyboardInterrupt as ex:
            raise ex
        except Exception:
            pass
    try:
        return _get_peer_cert_chain_with_openssl(options, host, port)
    except KeyboardInterrupt as ex:
        raise ex
    except Exception as ex:
        if str(ex) == EXCEPTION_TIMEDOUT_MSG:
            raise ex
        else:
            pass
    socket_type = socket.AF_INET
    return _get_peer_cert_chain_with_sock_type(options, host, port,
                                               socket_type)


def _submsg_translate(submsg):
    """Translate between httplib errorcodes and curl errorcodes
    """
    if 'timed out' in submsg:
        return CONNECTION_TIMED_OUT_MSG
    elif 'Connection refused' in submsg:
        return 'Connection refused'
    return submsg


def _lookup_cert_in_trust(options, thumbprint):
    """Return the cert matching the thumbprint otherwise return None if not
       found.
    """
    for cert in options.trust_store:
        digest = cert.digest("sha256").decode(UTF8)
        if (_canonical_thumbprint(thumbprint) ==
                _canonical_thumbprint(digest)):
            return cert
    return None


def _add_cert_to_trust(options, leaf_cert):
    """Once the leaf certificate has been validated for a given host and port
       add it to our in-memory trust store. This trust store is then used when
       invoking curl on subsequent retries.  The trust store is located in the
       options structure as a convenience.
    """
    options.trust_store.append(leaf_cert)


def _is_trust_established(options):
    """Check if trust has been established with the current host.
       Return True or False.
    """
    return len(options.trust_store) > 0


def _errcode_from_options_and_exmsg(errcode, options, submsg):
    """Set the error code based on the context of the exception message.
       Previously, curl would return CURLE_COULDNT_CONNECT (7) for connection
       errors unless the --max-time option is set in which case curl returns
       CURLE_OPERATION_TIMEDOUT (28).  However, the more recent behavior is
       curl returns CURLE_OPERATION_TIMEDOUT (28) regardless of options used.
       The submsg parameter is currently not used, but could be used in future
       to decide whether to translate the error code.
    """
    if errcode != CURLE_COULDNT_CONNECT:
        return errcode
    # Translate CURLE_COULDNT_CONNECT to CURLE_OPERATION_TIMEDOUT
    errcode = CURLE_OPERATION_TIMEDOUT
    return errcode


def _validate_peer_cert_chain(options, host, port):
    """Fetch the cert chain from the 'host' and validate it.  Validated leaf
       certs are placed in options.trust_store[].  Returns the tuple
       (is_self_signed, errcode, errstr).  If cert is validated successfully,
       this function returns (is_self_signed, 0, None).  Otherwise this
       function returns a non-zero errcode and a non-None errstr.
    """
    # Assume the peer cert is CA-signed unless told otherwise.
    is_self_signed = False
    try:
        from OpenSSL import crypto  # noqa
        from OpenSSL import SSL     # noqa
    except ImportError:
        errcode = CURLE_USE_SSL_FAILED
        errstr = ("curl_wrapper requires the Python OpenSSL library to " +
                  "validate certificates")
        return is_self_signed, errcode, errstr
    try:
        cert_chain = _get_peer_cert_chain(options, host, port)
        is_self_signed, errcode, errstr = _validate_cert(cert_chain, host,
                                                         options)
        if errcode:
            return is_self_signed, errcode, errstr

        # If _validate_cert() succeeds then the cert is trusted.
        leaf_cert = _get_leaf_cert(cert_chain)
        _add_cert_to_trust(options, leaf_cert)
        return is_self_signed, 0, None
    except AttributeError as ex:
        # _get_peer_cert_chain() may throw an AttributeError exception
        # and when it does, handle it has a non-recoverable error (that is
        # don't retry when the --retry option is used). Specifically, don't
        # return with a CURLE_COULDNT_CONNECT error.
        errcode = CURLE_SSL_CONNECT_ERROR
        submsg = str(ex)
        errstr = ("Failed to retrieve cert chain from " + host + " port " +
                  str(port) + ": " + submsg)
        return is_self_signed, errcode, errstr
    # Whenever we catch Exception, always catch KeyboardInterrupt prior to
    # catching Exception and throw it.
    except KeyboardInterrupt as ex:
        raise ex
    except Exception as ex:
        # Handles timeout and all other exceptions
        if str(ex) == EXCEPTION_TIMEDOUT_MSG:
            submsg = str(ex)
            errcode = _errcode_from_options_and_exmsg(CURLE_COULDNT_CONNECT,
                                                      options, submsg)
            index = submsg.find("timed out after")
            if index >= 0:
                errstr = "Connection " + submsg[index:]
            else:
                errstr = "Connection timed out"
            _log_backtrace(options, errstr,
                           traceback.format_exc())
            return is_self_signed, errcode, errstr
        else:
            submsg = _submsg_translate(str(ex))
            errcode = _errcode_from_options_and_exmsg(CURLE_COULDNT_CONNECT,
                                                      options, submsg)
            errstr = ("Failed to connect to " + host + " port " +
                      str(port) + ": " + submsg)
            _log_backtrace(options, errstr,
                           traceback.format_exc())
            return is_self_signed, errcode, errstr


def _get_min_timeout(options):
    """Helper function to get the smaller of the two timeout options,
       options.max_time and options.connect_timeout. If neither options are
       specified, return 10 seconds.
       The default curl timeout is 2 minutes, but this script uses
       a 10 second timeout.
    """
    timeout = options.max_time if options.max_time > 0 else DEFAULT_TIMEOUT
    if options.connect_timeout > 0 and options.connect_timeout < timeout:
        timeout = options.connect_timeout
    return timeout


def read_name(context):
    """ Used by parse_form_option() to read the name in name=value strings in
        curl's --form option.  Returns '' if no name is present.
    """
    name = ''
    while context.ptr < len(context.data):
        ch = context.data[context.ptr]
        if ch == '=':
            break
        name += ch
        context.ptr += 1
    return name


def read_equals(context):
    """Used by parse_form_option() to read the = character in name=value
       strings in curl's --form option.  Returns None if no name is present.
    """
    if context.ptr < len(context.data):
        ch = context.data[context.ptr]
        if ch == '=':
            context.ptr += 1
            return ch
    return None


def read_string(context, quote_char):
    """Helper function used by parse_form_option() to read strings where the
       string starts with the given quote_char. The function handles escaped
       characters such that if the quote_char is escaped with '\' then the
       function continues.  Returns '' if no value is present.
    """
    value = ''
    if len(quote_char) == 1:
        if context.ptr >= len(context.data):
            return None
        if context.data[context.ptr] != quote_char:
            return None
        ch = context.data[context.ptr]
        context.ptr += 1
        value += ch
        while context.ptr < len(context.data):
            ch = context.data[context.ptr]
            context.ptr += 1
            value += ch
            if ch == quote_char:
                return value
    elif len(quote_char) == 2:
        if context.ptr + 1 >= len(context.data):
            return None
        if quote_char[0] != '\\':
            return None
        if context.data[context.ptr] != quote_char[0]:
            return None
        if context.data[context.ptr + 1] != quote_char[1]:
            return None
        context.ptr += 2
        found_escape = False
        while context.ptr < len(context.data):
            ch = context.data[context.ptr]
            context.ptr += 1
            value += ch
            if found_escape:
                if context.ptr < len(context.data):
                    ch = context.data[context.ptr]
                    context.ptr += 1
                    value += ch
                    if ch == quote_char[1]:
                        return value
                    found_escape = False
                else:
                    return None
            else:
                if context.ptr < len(context.data):
                    ch = context.data[context.ptr]
                    context.ptr += 1
                    value += ch
                    if ch == quote_char[0]:
                        found_escape = True
                else:
                    return None
    return None


def read_value(context):
    """ Used by parse_form_option() to read the value in name=value strings in
        curl's --form option.  Returns '' if no value is present.
        Read up to first unescaped ; character.
        Returns None on error.
    """
    value = ''
    while context.ptr < len(context.data):
        ch = context.data[context.ptr]
        if ch == ';':
            context.ptr += 1
            break
        elif ch in ['"', "'"]:
            value += read_string(context, ch)
        elif ch == '\\':
            if context.ptr + 1 < len(context.data):
                if context.data[context.ptr + 1] in ['"', "'"]:
                    value += read_string(context, ch +
                                         context.data[context.ptr + 1])
                else:
                    return None
            else:
                return None
        else:
            value += ch
            context.ptr += 1
    return value


def parse_form_option(context):
    """Parse curl's --form option and output the parsed strings in
       context.parsed_data list.
    """
    # The string will be empty for -F =@a.out
    name = read_name(context)
    ch = read_equals(context)
    if ch:
        value = read_value(context)
        context.parsed_data.append((name, value))
        # Ignore the remaining part of the option after the first semi-colon.
        # For example ";type=image/jpeg"
    else:
        context.parsed_data.append((name, ''))


def copy_file_in_chunks(filename, out_file):
    """To avoid reading large files into memory, read the large files in
       chunks and append to an already open out_file.  The out_file is then
       passed directly to httplib's request function.  The parameter filename
       is the name of the input file, and out_file is a handle to an already
       open file.  Returns (0, None) on success, otherwise returns
       (CURLE_READ_ERROR, errstr) if unable to read filename.
    """
    if not exists(filename):
        return CURLE_READ_ERROR, CURLE_READ_ERRSTR
    try:
        with open(filename, "rb") as f:
            while True:
                chunk = f.read(8192)  # Read in chunks
                if not chunk:
                    break
                out_file.write(chunk)
    # FileNotFoundError in Python3:
    # IOError in Python2:
    except Exception:
        return CURLE_READ_ERROR, CURLE_READ_ERRSTR
    return 0, None


def gather_upload_data(form_option, file):
    """Process curl's --data option and output content to the already open file
       handle. Returns (errcode, errstr).
    """
    crlf = '\r\n'.encode(UTF8)
    value = form_option
    if value.startswith('@'):
        value = value[1:]
        errcode, errstr = copy_file_in_chunks(value, file)
        if errcode:
            return errcode, errstr
        file.write(crlf)
    else:
        file.write(value.encode(UTF8) + crlf)
    return 0, None


def generate_boundary():
    """Generate "boundary" string for curl's --data and --form options.  In
       one PUT or POST API, one boundary string is used to separate multiple
       attachments.
    """
    length = 20
    characters = string.ascii_letters
    random_string = ''.join(random.choice(characters) for _ in range(length))
    return "------------------------" + random_string


def get_content_type(filename):
    """Guess the mime type based on the filename extension.
    """
    mime_type, _ = mimetypes.guess_type(filename)
    if not mime_type:
        mime_type = "application/octet-stream"
    return mime_type


def gather_upload_form(options, file):
    """Handle use-cases like:
         curl -F profile=@portrait.jpg https://example.com/upload.cgi
         curl -F =@portrait.jpg https://example.com/upload.cgi
         curl -F "story=<hugefile.txt" https://example.com/
         curl -F "web=@index.html;type=text/html" example.com
         curl -F "file=@\"local,file\";filename=\"name;in;post\""
           https://example.com
         curl -F "image=@file1.gif" -F "image2=@file2.gif"
       Because large files can be uploaded with -F and because multiple -F
       options can be provided in a single curl call, we write intermediate
       results to file, and then file is passed directly to httplib's request
       function.  This method updates options.header, specifically, the
       Content-Type header may be appended with a boundary string.
    """
    crlf = '\r\n'.encode(UTF8)
    for form_option in options.form:
        # option.form is a list where each item is a single -F option.
        # In this for loop, do one -F option at a time.
        ctx = SimpleParseContext()
        ctx.data = form_option
        parse_form_option(ctx)
        boundary = None
        for (name, value) in ctx.parsed_data:
            # Interestingly, curl allows an empty name and empty value.
            if name and not value:
                # curl gives either -F or --form depending on which was used
                errstr = "option -F: is badly used here"
                return CURLE_FAILED_INIT, errstr
            do_read_file = False
            if value.startswith('@') or value.startswith('<'):
                do_read_file = True
                value = value[1:]
                # Remove quotes if present
                if value.startswith("'"):
                    if not value.endswith("'"):
                        return None
                    value = value[1:-1]
                if value.startswith('"'):
                    if not value.endswith('"'):
                        return None
                    value = value[1:-1]
            if not boundary:
                # don't encode() boundary because it is used later
                boundary = generate_boundary()
            file.write(boundary.encode(UTF8) + crlf)
            file.write(('Content-Disposition: form-data').encode(UTF8))
            if name:
                file.write(('; name="' + name + '"').encode(UTF8))
            if do_read_file:
                file.write(('; filename="' + value + '"').encode(UTF8))
            file.write(crlf)
            if do_read_file:
                file.write(("Content-Type: " + get_content_type(value))
                           .encode(UTF8))
            file.write(crlf + crlf)
            if do_read_file:
                errcode, errstr = copy_file_in_chunks(value, file)
                if errcode:
                    return errcode, errstr
            else:
                file.write(value.encode(UTF8))
            file.write(crlf)
    if boundary:
        # We added at least one attachment above.  The headers are encoded at
        # the time of sending so don't encode here.
        file.write(boundary.encode(UTF8) + crlf)
        if 'Content-Type' in options.header:
            prev_content_type = options.header['Content-Type']
            words = prev_content_type.split(';')
            content_type = words[0] + '; boundary=' + boundary
        else:
            content_type = 'multipart/form-data; boundary=' + boundary
        options.header.update({'Content-Type': content_type})
    return 0, None


def _get_http_method(options):
    """ Get the HTTP method name from options.
    """
    if options.request:
        return options.request
    if options.form:
        return 'POST'
    if options.upload_file:
        return 'PUT'
    return 'GET'


def _httplib_no_follow(url, was_redirected, options, output_file):
    """Use httplib to simulate curl functionality and write the REST response
       to an output_file.  The output_file may be stdout or an actual file.
       Tested on VMware ESXi 7.0.3 build-18644231.  Tested on VMware ESXi 6.7.0
       build-14320388.
       Return the tuple (errcode, httpcode).
    """
    try:
        # We use the url function parameter rather than options.url because we
        # may have been redirected by the server.
        url_parts = urlparse(url)
        host = url_parts.hostname
        port = url_parts.port
        path = url_parts.path
        if url_parts.query:
            path = path + '?' + url_parts.query
        if not port:
            port = 80 if url_parts.scheme == "http" else 443
    except ValueError:
        errcode = CURLE_URL_MALFORMAT
        if was_redirected:
            errstr = ("Found bad URL in Location header: " + url)
        else:
            errstr = CURLE_URL_MALFORMAT_ERRSTR
        _set_err_stderr(options, errcode, errstr)
        return errcode, INVALID_HTTP_CODE

    _log_trying_connection(options, host, port, True)

    # EAL4_Zeroize_Sensitive_Data_By_Caller
    # username_passwd zeroized by the calling call_curl() function.
    username_passwd = options.user
    hdrs = {}
    if username_passwd:
        username_passwd = b64encode(
            username_passwd.encode(UTF8)).decode(ASCII)
        # EAL4_Zeroize_Sensitive_Data
        # hdrs[AUTHORIZATION_HDR] zeroized in finally block
        hdrs.update({AUTHORIZATION_HDR: 'Basic %s' % username_passwd})

    conn = None
    errcode = CURLE_OK
    httpcode = -1

    # timeout is connection timeout and is set to the smaller of max_time and
    # connect_timeout.  The max_time is used for each retry. In other words, if
    # max_time is 30 seconds, and the retry is 6, then we will retry 6 times
    # waiting for 30 seconds on each try. The total wait time will be 6 * 30
    # seconds plus the sleep time between each retry.
    timeout = _get_min_timeout(options)
    upload_filename = None
    tmp_upload_filename = None
    if options.data or options.form:
        _, tmp_upload_filename = tempfile.mkstemp()
        upload_filename = tmp_upload_filename
        with open(tmp_upload_filename, 'wb') as upload_data:
            if options.data:
                errcode, errstr = gather_upload_data(options.data, upload_data)
            else:
                # This function typically updates options.header
                errcode, errstr = gather_upload_form(options, upload_data)
            if errcode:
                _set_err_stderr(options, errcode, errstr)
                return errcode, INVALID_HTTP_CODE
        options.header.update({'Content-Length':
                               os.path.getsize(tmp_upload_filename)})
    elif options.upload_file:
        upload_filename = options.upload_file[0]
        if not exists(upload_filename):
            errcode = CURLE_READ_ERROR
            _set_err_stderr(options, errcode, CURLE_READ_ERRSTR)
            return errcode, INVALID_HTTP_CODE
        options.header.update({'Content-Length':
                               os.path.getsize(upload_filename)})

    # Note that the Authorization header in the -H option overrides the
    # Authorization header constructed by the -u username/password option (as
    # it does in curl).
    hdrs.update(options.header)
    if 'User-Agent' not in hdrs:
        hdrs.update({'User-Agent': CURL_WRAPPER_TAG})

    try:
        if url.lower().startswith("https:"):
            _, errcode, errstr = \
                _validate_peer_cert_chain(options, host, port)
            if errcode:
                # Either unable to fetch cert or cert failed validation.
                _set_err_stderr(options, errcode, errstr)
                return errcode, INVALID_HTTP_CODE
            # In case of CA-signed certs it would be preferable to avoid
            # setting create_unverified_context(), however on ESX, Python
            # complains when validating some CA-signed certs: [SSL:
            # CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed
            # certificate in certificate chain (_ssl.c:1125)
            ctx = ssl._create_unverified_context()
            if options.key and options.cert:
                conn = httplib.HTTPSConnection(host, port, context=ctx,
                                               timeout=timeout,
                                               key_file=options.key,
                                               cert_file=options.cert)
            else:
                conn = httplib.HTTPSConnection(host, port, context=ctx,
                                               timeout=timeout)
        else:
            conn = httplib.HTTPConnection(host, port,
                                          timeout=timeout)

        http_method = _get_http_method(options)
        if upload_filename:
            with open(upload_filename, 'rb') as upload_data:
                conn.request(http_method, path, upload_data, hdrs)
        else:
            conn.request(http_method, path, None, hdrs)
        resp = conn.getresponse()

        content_length = -1  # unknown
        for key, value in resp.getheaders():
            if key == 'content-length':
                content_length = int(value.encode(UTF8))
                break
        if options.include or options.head:
            version = str(resp.version)
            version = "HTTP/%s.%s" % (version[0], version[1])
            output_file.write(("%s %s %s\n" % (version, resp.status,
                                               resp.reason)).encode(UTF8))
            for key, value in resp.getheaders():
                output_file.write(("%s: %s\n" % (key, value)).encode(UTF8))
        if options.location:
            # Auto-redirection
            resp_headers = dict(resp.getheaders())
            if 'Location' in resp_headers and resp_headers['Location'] != url:
                return THIS_WAS_REDIRECTED, resp_headers['Location']
        if not options.head:
            recv_length = 0
            empty_reads_consecutive = 0
            while True:
                buffer = resp.read(65536)
                recv_length += len(buffer)
                if len(buffer) > 0:
                    empty_reads_consecutive = 0
                    output_file.write(buffer)
                else:
                    empty_reads_consecutive += 1
                    # content_length is -1 if unknown
                    if content_length == recv_length:
                        break
                    # resp.closed defined only in Python 3.
                    if ((hasattr(resp, 'closed') and resp.closed)
                            or empty_reads_consecutive >= MAX_EMPTY_READS):
                        if content_length > 0:
                            errcode = CURLE_PARTIAL_FILE
                        break
                    time.sleep(1)
        httpcode = resp.status
        return errcode, httpcode
    except AttributeError as ex:
        # _get_peer_cert_chain() may throw an AttributeError exception
        # and when it does, handle it has a non-recoverable error (that is
        # don't retry when the --retry option is used). Specifically, don't
        # return with a CURLE_COULDNT_CONNECT error.
        submsg = str(ex)
        errcode = CURLE_SSL_CONNECT_ERROR
        errstr = ("Failed to retrieve cert chain from " + host + " port " +
                  str(port) + ": " + submsg)
        _set_err_stderr(options, errcode, errstr)
        return (errcode, INVALID_HTTP_CODE)
    # Whenever we catch Exception, always catch KeyboardInterrupt prior to
    # catching Exception and throw it.
    except KeyboardInterrupt as ex:
        errcode = THIS_KEYBOARD_INTERRUPT
        raise ex
    except Exception as ex:
        submsg = _submsg_translate(str(ex))
        errcode = _errcode_from_options_and_exmsg(CURLE_COULDNT_CONNECT,
                                                  options, submsg)
        errstr = ("Failed to connect to " + host + " port " +
                  str(port) + ": " + submsg)
        _log_backtrace(options, errstr,
                       traceback.format_exc())
        _set_err_stderr(options, errcode, errstr)
        return errcode, INVALID_HTTP_CODE
    finally:
        if tmp_upload_filename:
            os.remove(tmp_upload_filename)
        if errcode != THIS_KEYBOARD_INTERRUPT:
            _log_closing_connection(options)
        if conn:
            conn.close()
        if AUTHORIZATION_HDR in hdrs:
            nz.zeroize(hdrs[AUTHORIZATION_HDR])


class Stats:
    def __init__(self):
        self.url_effective = ''
        self.http_code = INVALID_HTTP_CODE


class WriteOutParseContext:
    """Used by _print_write_out to print the --write-out format string.
    """
    def __init__(self):
        self.errstr = ''
        self.input = ''
        self.ptr = 0


def _read_char(context):
    """Used by _print_write_out to print the --write-out format string.
    """
    if context.ptr >= len(context.input):
        return None
    char = context.input[context.ptr]
    context.ptr += 1
    return char


def _print_write_out(options, stats):
    """Print the --write-out format string to stdout.  Returns no error code.
       curl doesn't return an error code if there are formatting errors in the
       format string, nor are the error messages silenced with the -s option.
    """
    if options.errcode == THIS_KEYBOARD_INTERRUPT:
        return
    output = ''
    context = WriteOutParseContext()
    errstrfmt = "curl_wrapper: unknown --write-out variable: '{0}'"
    context.input = options.write_out

    if not context.input:
        # Nothing to write out.
        return
    TEXT_MODE = 0
    VARIABLE_MODE = 1
    mode = TEXT_MODE
    variable = ''
    while True:
        char = _read_char(context)
        if not char:
            break
        if mode == TEXT_MODE:
            if char == '%':
                char = _read_char(context)
                if not char:
                    break
                if char == '{':
                    mode = VARIABLE_MODE
                    variable = ''
                elif char == '%':
                    output += char
                else:
                    output += '%' + char
            elif char == '\\':
                char = _read_char(context)
                if not char:
                    break
                elif char == 'n':
                    output += '\n'
                elif char == 'r':
                    output += '\r'
                elif char == 't':
                    output += '\t'
                else:
                    output += char
            else:
                output += char
        else:
            # Parsing a variable in this block, eg %{url_effective}
            if char == '}':
                if variable == 'url_effective':
                    output += stats.url_effective
                elif variable == 'http_code':
                    if stats.http_code == INVALID_HTTP_CODE:
                        output += '000'
                    else:
                        output += str(stats.http_code)
                else:
                    errstr = errstrfmt.format(variable)
                    print(errstr, file=sys.stderr)
                    return
                mode = TEXT_MODE
                variable = ''
            else:
                variable += char
    if variable:
        errstr = errstrfmt.format(variable)
        print(errstr, file=sys.stderr)
        return
    print(output, end='')
    return


def _httplib_to_file(options, output_file, stats):
    """Use httplib to simulate curl functionality and write the REST response
       to an output_file.  The output_file may be stdout or an actual file.
       Tested on VMware ESXi 7.0.3 build-18644231.  Tested on VMware ESXi 6.7.0
       build-14320388.
       Return the tuple (errcode, httpcode).
       """

    url = options.url
    stats.url_effective = options.url

    if not options.location:
        errcode, httpcode = _httplib_no_follow(options.url, False, options,
                                               output_file)
        stats.http_code = httpcode
        return errcode, httpcode

    if options.max_redirs is None or options.max_redirs < 0:
        redirections = DEFAULT_MAX_REDIRECTS
    else:
        redirections = options.max_redirs

    was_redirected = False
    for _ in range(redirections + 1):
        errcode, httpcode = _httplib_no_follow(url, was_redirected,
                                               options, output_file)
        stats.http_code = httpcode
        if errcode != THIS_WAS_REDIRECTED:
            return errcode, httpcode
        # When errcode is THIS_WAS_REDIRECTED then httpcode is the new url.
        url = httpcode
        stats.url_effective = options.url
        was_redirected = True
    errcode = CURLE_TOO_MANY_REDIRECTS
    errstr = ("Maximum (" + str(redirections) + ") redirects followed")
    _set_err_stderr(options, errcode, errstr)
    return errcode, INVALID_HTTP_CODE


def _append_curl_headers(cmd, headers):
    """ Append -H headers for curl command.
    """
    for key, value in headers.items():
        cmd.append('-H')
        cmd.append(key + ": " + value)


def build_curl_cmd(options, last_try, output_file):
    """Build curl command line options in a list and return the list.
    """
    cmd = []
    cmd.append('/usr/bin/curl')
    if options.cacert:
        cmd.append('--cacert')
        cmd.append(options.cacert)
    elif options.url.lower().startswith("https:"):
        leaf_cert = _lookup_cert_in_trust(options, options.thumbprint)
        # If we can't find a trusted cert that matches the given thumbprint
        # then we may not use the -k curl option.
        if leaf_cert:
            is_self_signed = leaf_cert.get_subject() == leaf_cert.get_issuer()
            if is_self_signed:
                # The -k option is used in the case that (1) the peer is using
                # a self-signed cert and (2) a thumbprint has been provided and
                # the peer cert has already been validated against that
                # thumbprint.  Under this case the -k must be used because
                # otherwise curl will fail the hostname check with the
                # self-signed cert.
                cmd.append('-k')
            else:
                tmp_file, tmp_filename = tempfile.mkstemp()
                pem_data = crypto.dump_certificate(crypto.FILETYPE_PEM,
                                                   leaf_cert)
                if not have_py3:
                    pem_data = pem_data.decode(UTF8)
                # os.write takes a buffer on Python 3 and a string on Python 2.
                os.write(tmp_file, pem_data)
                os.close(tmp_file)
                cmd.append('--cacert')
                cmd.append(tmp_filename)
    if options.connect_timeout:
        cmd.append('--connect-timeout')
        cmd.append(str(options.connect_timeout))
    if options.data:
        cmd.append('-d')
        cmd.append(options.data)
    if options.form:
        for option in options.form:
            cmd.append('-F')
            cmd.append(option)
    if options.head:
        cmd.append('-I')
    if len(options.header) > 0:
        _append_curl_headers(cmd, options.header)
    if options.include:
        cmd.append('-i')
    # options.insecure (-k) is handled above
    if options.location:
        cmd.append('-L')
    if options.max_redirs:
        cmd.append('--max-redirs')
        cmd.append(str(options.max_redirs))
    if options.max_time > 0:
        cmd.append('-m')
        cmd.append(str(options.max_time))
    # skip output because we must use output_file instead
    if output_file:
        cmd.append('-o')
        cmd.append(output_file)
    # skip remote_name because we must use output_file instead
    if options.upload_file:
        for option in options.upload_file:
            cmd.append('-T')
            cmd.append(option)
    if options.request:
        cmd.append('-X')
        cmd.append(options.request)
    # skip retry because it is handled by _call_http()
    # skip retry_delay because it is handled by _call_http()
    # skip retry_max_time because it is handled by _call_http()
    # always use silent mode
    cmd.append('-s')
    if options.show_error or not options.silent:
        if last_try:
            cmd.append('-S')
    # skip thumbprint because curl doesn't support this parameter
    if options.user:
        cmd.append('-u')
        cmd.append(options.user)
    if options.verbose:
        cmd.append('-v')
    # curl to write out all variables supported by this script
    if options.write_out:
        cmd.append('-w')
        # The number of words in this string must match NUM_WRITE_OUT_WORDS
        cmd.append(CURL_WRAPPER_TAG + " %{http_code} %{url_effective}")
    cmd.append(options.url)
    return cmd


def _collect_stats(stats, line):
    """Extract the --write-out values from the stdout output emitted by curl.
       This function is only called if line has NUM_WRITE_OUT_WORDS words.
    """
    words = line.split()
    stats.http_code = int(words[1])
    stats.url_effective = words[2].decode(UTF8)


def _remove_tmp_cacert_file(cmd_args):
    """Remove the temporary cacert file when invoking curl. Swallows any
       exception thrown while attempting to delete the temporary file.
       Returns no value or exception.
    """
    cacert = None
    prev_s = None
    for s in cmd_args:
        if prev_s == '--cacert':
            cacert = s
            break
        prev_s = s
    if cacert and exists(cacert):
        try:
            os.remove(cacert)
        except Exception:
            pass
    return


def _write_to_stdout(options, stats, tag, line):
    """Function to delay the writing out of curl's stdout so we can detect the
       last line having curl's --write-out results
    """
    if options.prev_stdout_line:
        if (not line and options.write_out and
                tag in options.prev_stdout_line and
                len(options.prev_stdout_line.split()) == NUM_WRITE_OUT_WORDS):
            # last line has been read and the special tag is present
            _collect_stats(stats, options.prev_stdout_line)
            # remove tag and write-out results
            options.prev_stdout_line = \
                options.prev_stdout_line[0: options.prev_stdout_line.find(tag)]
        if hasattr(sys.stdout, "buffer"):
            # Python 3
            sys.stdout.buffer.write(options.prev_stdout_line)
        else:
            # Python 2
            sys.stdout.write(options.prev_stdout_line)
    options.prev_stdout_line = line


def _curl_to_file(options, last_try, output_file, stats):
    """Call curl to write the REST response to an output_file.
       Return the tuple (errcode, httpcode).
    """
    cmd_args = build_curl_cmd(options, last_try, output_file)
    df = None
    _ex = None
    try:
        host = options.host
        port = options.port
        _log_trying_connection(options, host, port, False)
        # Important to call sys.stderr.flush() before invoking a subprocess. In
        # our case, the log_trying_connection calls the flush() function.
        _log_command(options, cmd_args, True)
        df = subprocess.Popen(cmd_args, stdout=subprocess.PIPE)
        if have_py3:
            tag = bytes(CURL_WRAPPER_TAG, UTF8)
        else:
            tag = CURL_WRAPPER_TAG
        for line in df.stdout:
            _write_to_stdout(options, stats, tag, line)
        _write_to_stdout(options, stats, tag, None)
        df.communicate()[0]
    except Exception as ex:
        _ex = ex
        _log_backtrace(options, "curl error",
                       traceback.format_exc())
    finally:
        if last_try:
            options.last_curl_fin = True
        _remove_tmp_cacert_file(cmd_args)
        errcode = df.returncode if df else CURLM_INTERNAL_ERROR
        if not _ex or str(_ex) != EXCEPTION_TIMEDOUT_MSG:
            _log_closing_connection(options)
        # The cmd_args includes secrets but because it includes a shallow copy
        # of secrets in 'options' we don't zeroize the cmd_args.
        return errcode, stats.http_code


def _http_to_file(first_try, last_try, options, output_filename, stats):
    """On first attempt use httplib to simulate curl functionality and write
       the REST response to an output_file.  On subsequent attempts use curl if
       it is available.  Return the tuple (errcode, httpcode).
    """
    # curl_to_file doesn't validate the peer cert so until we have completed
    # the step of validating the peer cert we continue using _httplib_to_file.
    if (first_try or not exists('/usr/bin/curl') or not
            _is_trust_established(options)):
        if not output_filename:
            if hasattr(sys.stdout, "buffer"):
                # Python 3
                file = sys.stdout.buffer
            else:
                # Python 2
                file = sys.stdout
            return _httplib_to_file(options, file, stats)
        else:
            with open(output_filename, 'wb') as file:
                return _httplib_to_file(options, file, stats)
    else:
        # output_filename may be None which means stdout
        return _curl_to_file(options, last_try, output_filename, stats)


def _call_http(options):
    """Use httplib to simulate curl functionality and write the REST response
       to an output_file.  The function simulates curl's retry options.
       This function returns a single error code, where 0 indicates success.
    """
    stats = Stats()
    output_filename = None
    if options.remote_name:
        output_filename = options.url.split('/')[-1]
    else:
        output_filename = options.output

    if options.retry_max_time > 0:
        stop_time = datetime.now() + timedelta(seconds=options.retry_max_time)
    sleep_time = 1 if options.retry_delay <= 0 else options.retry_delay
    errcode = CURLM_INTERNAL_ERROR
    httpcode = -1

    retry = options.retry
    first_try = True
    try:
        while retry >= 0:
            # options.num_retry_conns is initialized to -1 and here
            # we increment to 0.
            options.num_retry_conns += 1
            if (output_filename and exists(output_filename) and
                    output_filename != '/dev/null'):
                os.remove(output_filename)
            last_try = (retry == 0)
            (errcode, httpcode) = _http_to_file(first_try, last_try, options,
                                                output_filename, stats)
            # Repeat if transient error.
            if not _is_transient_error(errcode, httpcode):
                break
            retry -= 1
            first_try = False
            if retry >= 0:
                # mimic curl's --retry
                if options.retry_max_time > 0:
                    now = datetime.now()
                    if now > stop_time:
                        errcode = CURLE_COULDNT_CONNECT  # timeout
                        break
                    if now + timedelta(seconds=sleep_time) > stop_time:
                        time.sleep((stop_time - now).total_seconds())
                        errcode = CURLE_COULDNT_CONNECT  # timeout
                        break
                _log_transient_problem(options, sleep_time, retry+1)
                time.sleep(sleep_time)
                if options.retry_delay <= 0:
                    # curl man page says maximum sleep time is 10 minutes,
                    # which is 600 seconds.
                    sleep_time = min(sleep_time * 2, 600)
                else:
                    sleep_time = options.retry_delay
    except KeyboardInterrupt:
        errcode = THIS_KEYBOARD_INTERRUPT
        _set_err_stderr(options, errcode, "")
    finally:
        # when writing to stdout output_filename will be None
        if errcode and output_filename and output_filename != '/dev/null':
            if exists(output_filename):
                os.remove(output_filename)
        if not errcode:
            # It is possible for options.errcode to have been set earlier but
            # for errcode to be zero at this point.  If this is the case
            # errcode should be used, and we must set options.errcode
            # accordingly before calling _print_final_msgs().
            _set_err_stderr(options, errcode, "")
        _print_final_msgs(options, stats)
    return errcode


def check_mutually_exclusive_options(options):
    """Check if mutually exclusive options have been provided.
       Return the tuple (errcode, errstr) where errcode and errstr mimic curl's
       response.
    """
    cnt = 0
    elist = []
    if options.data:
        cnt += 1
        elist.append("POST (-d, --data)")
    if options.form:
        cnt += 1
        elist.append("multipart formpost (-F, --form)")
    if options.upload_file:
        cnt += 1
        elist.append("PUT (-T, --upload-file)")
    if options.head:
        cnt += 1
        elist.append("HEAD (-I, --head)")
    if cnt <= 1:
        return 0, None
    # curl treats -d as POST even when user gives a different method for -X
    # curl treats -T as PUT even when user gives a different method for -X
    first_line = elist[0]
    second_line = elist[1]
    search_words = ["PUT", "POST"]

    if ((not any(word in first_line for word in search_words))
            and any(word in second_line for word in search_words)):
        # The first line doesn't have either PUT or POST but the the
        # second line does, so swap them to mimic curl.
        first_line = elist[1]
        second_line = elist[0]

    for method in search_words:
        if method in first_line:
            first_line = first_line.replace(method, method + "\nWarning:")
    errstr = ("Warning: You can only select one HTTP request method! "
              "You asked for both " + first_line + " and " + second_line + ".")
    return CURLE_FAILED_INIT, errstr


def call_curl(args):
    """Top-level function that can be called from another python program.
       The function name is a bit misleading because this function used to
       invoke the curl binary but the curl binary was dropped because curl
       doesn't support checking trust with a thumbprint.  Instead this script
       uses the python library httplib to simulate curl.
       This function returns a single error code, where 0 indicates success.

       The call_curl function can be called as follows:
       import imp
       curl_wrapper = imp.load_source('curl_wrapper',
           '/opt/vmware/nsx-common/python/nsx_utils/curl_wrapper')
       args = ("-u admin:password -i "
               "https://10.192.193.86/api/v1/node/aaa/providers/vidm").split()
       sys.exit(curl_wrapper.call_curl(args))
    """
    # Step 1: Use argparse to parse arguments into a list
    parser = argparse.ArgumentParser()
    parser.add_argument('--cacert', required=False)
    parser.add_argument('--cert', required=False)
    parser.add_argument('--connect-timeout', required=False)
    parser.add_argument('-d', '--data', required=False)
    parser.add_argument('-F', '--form', action='append', required=False)
    parser.add_argument('-H', '--header', action='append', required=False)
    parser.add_argument('-i', '--include', action='store_true', required=False)
    parser.add_argument('-I', '--head', action='store_true', required=False)
    parser.add_argument('-k', '--insecure', action='store_true',
                        required=False)
    parser.add_argument('--key', required=False)
    parser.add_argument('-L', '--location', action='store_true',
                        required=False)
    parser.add_argument('--max-redirs', required=False)
    parser.add_argument('-m', '--max-time', required=False)
    parser.add_argument('--no-hostname-check', action='store_true',
                        required=False)
    parser.add_argument('-o', '--output', required=False)
    parser.add_argument('-O', '--remote-name', action='store_true',
                        required=False)
    parser.add_argument('--retry', required=False)
    parser.add_argument('--retry-delay', required=False)
    parser.add_argument('--retry-max-time', required=False)
    # This script does --silent always
    parser.add_argument('-s', '--silent', action='store_true', required=False)
    parser.add_argument('-S', '--show-error', action='store_true',
                        required=False)
    parser.add_argument('-T', '--upload-file', action='append', required=False)
    parser.add_argument('--thumbprint', required=False)
    parser.add_argument('-u', '--user', required=False)
    parser.add_argument('-v', '--verbose', action='store_true',
                        required=False)
    parser.add_argument('-w', '--write-out', required=False)
    parser.add_argument('-X', '--request', required=False)
    parsed_args, extra_args = parser.parse_known_args(args)

    # Step 2: Convert argparse list to a compact structure
    options = CmdLineOptions()
    # Process the silent and show_error first because we need these if there
    # are problems parsing the other arguments
    try:
        options.silent = _get_silent_opt(parsed_args)
        options.show_error = _get_show_error_opt(parsed_args)

        (errcode, errstr, options.host, options.port, options.path,
            options.url) = _get_host_opt(extra_args)
        if errcode:
            _set_err_stderr(options, errcode, errstr)
            _print_final_msgs(options, None)
            return errcode

        options.cacert = _get_cacert_opt(parsed_args)
        options.cert = _get_cert_opt(parsed_args)
        options.connect_timeout = _get_connect_timeout_opt(parsed_args)
        options.data = _get_data_opt(parsed_args)
        options.form = _get_form_opt(parsed_args)
        options.head = _get_head_opt(parsed_args)
        options.header = _get_request_headers_opt(parsed_args)
        options.include = _get_response_headers_opt(parsed_args)
        # options.insecure is not used
        options.key = _get_key_opt(parsed_args)
        options.location = _get_location_opt(parsed_args)
        options.max_redirs = _get_max_redirs_opt(parsed_args)
        options.max_time = _get_max_time_opt(parsed_args)
        options.no_hostname_check = _get_no_hostname_check_opt(parsed_args)
        options.output = _get_output_file_opt(parsed_args)
        options.remote_name = _get_remote_name_opt(parsed_args)
        options.request = _get_method_opt(parsed_args)
        options.retry = _get_retry_opt(parsed_args)
        options.retry_delay = _get_retry_delay_opt(parsed_args)
        options.retry_max_time = _get_retry_max_time_opt(parsed_args)
        # options.silent is handled above
        # options.show_error is handled above
        options.thumbprint = _get_thumbprint_opt(parsed_args)
        options.upload_file = _get_upload_opt(parsed_args)
        # EAL4_Zeroize_Sensitive_Data
        # options.user zeroized in finally block
        options.user = _get_user_passwd_opt(parsed_args)
        options.verbose = _get_verbose_opt(parsed_args)
        options.write_out = _get_write_out_opt(parsed_args)
        errcode, errstr = check_mutually_exclusive_options(options)
        if errcode:
            _set_err_stderr(options, errcode, errstr)
            _print_final_msgs(options, None, True)
            return errcode
        if options.upload_file and len(options.upload_file) > 1:
            errcode = CURLE_FAILED_INIT
            errstr = "curl_wrapper supports at most one -T option"
            # whereas curl supports multiple -T options
            _set_err_stderr(options, errcode, errstr)
            _print_final_msgs(options, None)
            return errcode

        # Step 3: Do curl equivalent using compact option structure
        return _call_http(options)
    finally:
        nz.zeroize(options.user)


def _read_cert_chain_from_file(filename):
    """Test function to read a cert chain from a PEM encoded file.
    """
    with open(filename, "r") as f:
        return _read_cert_chain_from_openssl_output(f.read())


def validate_server_cert_from_file(filename):
    """Test function to read a cert from a file and check it passes validation.
    """
    cert_chain = _read_cert_chain_from_file(filename)

    options = CmdLineOptions()
    options.silent = False
    options.show_error = True
    options.max_time = DEFAULT_TIMEOUT
    options.connect_timeout = DEFAULT_TIMEOUT
    options.host = "localhost"

    # For testing purposes get the thumbprint from the cert.
    leaf_cert = _get_leaf_cert(cert_chain)
    options.thumbprint = leaf_cert.digest("sha256").decode(UTF8)
    # TODO: The CN may have a * as the first character, in which case we
    # should replace it with a fake name (in this test function).
    hostname = leaf_cert.get_subject().CN

    _, errcode, errstr = _validate_cert(cert_chain, hostname, options)
    if errcode:
        # No check for silent and show_error here because this is a test
        # function.
        print(errstr, file=sys.stderr)
        print("Failed certificate validation")
    else:
        print("Passed certificate validation")
    return errcode


if __name__ == '__main__':
    """Main entry point when called as a script from another other program.
       The other entry point call_curl is more efficient when called from
       another python program.
    """
    _log_command(None, sys.argv, True)
    errcode = call_curl(sys.argv[1:])
    _log_exit_code(None, sys.argv, errcode)
    sys.exit(errcode)
