This adds the PubKeyPlugin directive and associated code from
SunSSH, allowing an in-process shared library to be called
into to check public keys for authentication.

--- hpn-ssh-hpn-18.4.2/auth2-pubkey.c.orig
+++ hpn-ssh-hpn-18.4.2/auth2-pubkey.c
@@ -23,6 +23,11 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
+/*
+ * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright 2015 Joyent, Inc.
+ * Use is subject to license terms.
+ */
 
 #include "includes.h"
 
@@ -41,11 +46,13 @@
 #include <time.h>
 #include <unistd.h>
 #include <limits.h>
+#include <dlfcn.h>
 
 #include "xmalloc.h"
 #include "ssh.h"
 #include "ssh2.h"
 #include "packet.h"
+#include "digest.h"
 #include "kex.h"
 #include "sshbuf.h"
 #include "log.h"
@@ -84,6 +91,15 @@
 	return ret;
 }
 
+static const char *RSA_SYM_NAME = "sshd_user_rsa_key_allowed";
+static const char *ECDSA_SYM_NAME = "sshd_user_ecdsa_key_allowed";
+typedef int (*RSA_SYM)(struct passwd *, RSA *, const char *);
+typedef int (*ECDSA_SYM)(struct passwd *, EC_KEY *, const char *);
+
+static const char *UNIV_SYM_NAME = "sshd_user_key_allowed";
+typedef int (*UNIV_SYM)(struct passwd *, const char *,
+    const u_char *, size_t);
+
 static int
 userauth_pubkey(struct ssh *ssh, const char *method)
 {
@@ -745,6 +761,124 @@
 	return found_key;
 }
 
+/**
+ * Checks whether or not access is allowed based on a plugin specified
+ * in sshd_config (PubKeyPlugin).
+ *
+ * Note that this expects a symbol in the loaded library that takes
+ * the current user (pwd entry), the current RSA key and it's fingerprint.
+ * The symbol is expected to return 1 on success and 0 on failure.
+ *
+ * While we could optimize this code to dlopen once in the process' lifetime,
+ * sshd is already a slow beast, so this is really not a concern.
+ * The overhead is basically a rounding error compared to everything else, and
+ * it keeps this code minimally invasive.
+ */
+static int
+user_key_allowed_from_plugin(struct passwd *pw, struct sshkey *key)
+{
+	RSA_SYM rsa_sym = NULL;
+	ECDSA_SYM ecdsa_sym = NULL;
+	UNIV_SYM univ_sym = NULL;
+	char *fp = NULL;
+	char *argfp = NULL;
+	void *handle = NULL;
+	int success = 0;
+
+	if (options.pubkey_plugin == NULL || pw == NULL || key == NULL ||
+	    (key->type != KEY_RSA &&
+	    key->type != KEY_DSA && key->type != KEY_ECDSA))
+		return success;
+
+	handle = dlopen(options.pubkey_plugin, RTLD_NOW);
+	if (handle == NULL) {
+		debug("Unable to open library %s: %s", options.pubkey_plugin,
+		dlerror());
+		goto out;
+	}
+
+	/*
+	 * If we have the new-style universal symbol for checking keys, use
+	 * that instead of the old-style per-key-type symbols.
+	 */
+	univ_sym = (UNIV_SYM)dlsym(handle, UNIV_SYM_NAME);
+	if (univ_sym != NULL) {
+		u_char *blob;
+		const char *type = sshkey_type(key);
+		size_t len = 0;
+		if (sshkey_to_blob(key, &blob, &len) != 0) {
+			debug("failed to convert key to rfc4253 format");
+			goto out;
+		}
+		debug("Invoking %s from %s", UNIV_SYM_NAME,
+		    options.pubkey_plugin);
+		success = (*univ_sym)(pw, type, blob, len);
+		debug("pubkeyplugin returned: %d", success);
+		goto out;
+	}
+
+	/* Otherwise, continue with the old-style fingerprint symbols. */
+	fp = sshkey_fingerprint(key, SSH_DIGEST_MD5, SSH_FP_HEX);
+	if (fp == NULL) {
+		debug("failed to generate fingerprint");
+		goto out;
+	}
+	if (strncmp(fp, "MD5:", 4) != 0) {
+		debug("fingerprint not in MD5:hex format");
+		goto out;
+	}
+	/* give the plugin the string without leading MD5: */
+	argfp = fp + 4;
+
+	switch (key->type) {
+	case KEY_RSA:
+		rsa_sym = (RSA_SYM)dlsym(handle, RSA_SYM_NAME);
+		if (rsa_sym == NULL) {
+			debug("Unable to resolve symbol %s: %s", RSA_SYM_NAME,
+			dlerror());
+			goto out;
+		}
+		debug2("Invoking %s from %s", RSA_SYM_NAME,
+		    options.pubkey_plugin);
+		success = (*rsa_sym)(pw, key->rsa, argfp);
+		break;
+	case KEY_ECDSA:
+		ecdsa_sym = (ECDSA_SYM)dlsym(handle, ECDSA_SYM_NAME);
+		if (ecdsa_sym == NULL) {
+			debug("Unable to resolve symbol %s: %s", ECDSA_SYM_NAME,
+			dlerror());
+			goto out;
+		}
+		debug2("Invoking %s from %s", ECDSA_SYM_NAME,
+		    options.pubkey_plugin);
+		success = (*ecdsa_sym)(pw, key->ecdsa, argfp);
+		break;
+	default:
+		debug2("user_key_plugins only support RSA and ECDSA keys");
+	}
+
+	debug("pubkeyplugin returned: %d", success);
+
+out:
+	if (handle != NULL) {
+		dlclose(handle);
+		ecdsa_sym = NULL;
+		rsa_sym = NULL;
+		univ_sym = NULL;
+		handle = NULL;
+	}
+
+	if (success)
+		verbose("Found matching %s key: %s", sshkey_type(key), fp);
+
+	if (fp != NULL) {
+		free(fp);
+		fp = NULL;
+	}
+
+	return success;
+}
+
 /*
  * Check whether key authenticates and authorises the user.
  */
@@ -796,6 +930,10 @@
 	sshauthopt_free(opts);
 	opts = NULL;
 
+	success = user_key_allowed_from_plugin(pw, key);
+	if (success > 0)
+		goto out;
+
 	if ((success = user_key_command_allowed2(pw, key, remote_ip,
 	    remote_host, conn_id, rdomain, &opts)) != 0)
 		goto out;
--- hpn-ssh-hpn-18.4.2/servconf.c.orig
+++ hpn-ssh-hpn-18.4.2/servconf.c
@@ -214,6 +214,7 @@
 	 */
 	options->pam_service_per_authmethod = 1;
 #endif
+	options->pubkey_plugin = NULL;
 }
 
 /* Returns 1 if a string option is unset or set to "none" or 0 otherwise. */
@@ -573,6 +574,7 @@
 	sAllowStreamLocalForwarding, sFingerprintHash, sDisableForwarding,
 	sExposeAuthInfo, sRDomain, sPubkeyAuthOptions, sSecurityKeyProvider,
 	sRequiredRSASize, sChannelTimeout, sUnusedConnectionTimeout,
+	sPubKeyPlugin,
 	sDeprecated, sIgnore, sUnsupported
 } ServerOpCodes;
 
@@ -743,6 +745,7 @@
 	{ "exposeauthinfo", sExposeAuthInfo, SSHCFG_ALL },
 	{ "rdomain", sRDomain, SSHCFG_ALL },
 	{ "casignaturealgorithms", sCASignatureAlgorithms, SSHCFG_ALL },
+	{ "pubkeyplugin", sPubKeyPlugin, SSHCFG_ALL },
 	{ "securitykeyprovider", sSecurityKeyProvider, SSHCFG_GLOBAL },
 	{ "requiredrsasize", sRequiredRSASize, SSHCFG_ALL },
 	{ "channeltimeout", sChannelTimeout, SSHCFG_ALL },
@@ -2705,6 +2708,18 @@
 		}
 		break;
 
+	case sPubKeyPlugin:
+		/*
+		 * Can't use parse_filename, as we need to support plain
+		 * names which dlopen will find on our lib path.
+		 */
+		arg = strdelim(&str);
+		if (!arg || *arg == '\0')
+			fatal("%s line %d: missing file name.",
+			    filename, linenum);
+		options->pubkey_plugin = xstrdup(arg);
+		break;
+
 	case sDeprecated:
 	case sIgnore:
 	case sUnsupported:
--- hpn-ssh-hpn-18.4.2/servconf.h.orig
+++ hpn-ssh-hpn-18.4.2/servconf.h
@@ -239,6 +239,7 @@
 
 	int	fingerprint_hash;
 	int	expose_userauth_info;
+	char	*pubkey_plugin;
 	u_int64_t timing_secret;
 	char   *sk_provider;
 	int	required_rsa_size;	/* minimum size of RSA keys */