/*	$OpenBSD: aesni.c,v 1.53 2021/10/24 10:26:22 patrick Exp $	*/
/*-
 * Copyright (c) 2003 Jason Wright
 * Copyright (c) 2003, 2004 Theo de Raadt
 * Copyright (c) 2010, Thordur I. Bjornsson
 * Copyright (c) 2010, Mike Belopuhov
 * All rights reserved.
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/atomic.h>
#include <sys/malloc.h>
#include <sys/pool.h>
#include <sys/mbuf.h>
#include <sys/smr.h>

#include <crypto/cryptodev.h>
#include <crypto/aes.h>
#include <crypto/gmac.h>
#include <crypto/xform.h>
#include <crypto/cryptosoft.h>

#include <machine/fpu.h>

struct aesni_aes_ctx {
	uint32_t		 aes_ekey[4 * (AES_MAXROUNDS + 1)];
	uint32_t		 aes_dkey[4 * (AES_MAXROUNDS + 1)];
	uint32_t		 aes_klen;
	uint32_t		 aes_pad[3];
};

struct aesni_xts_ctx {
	struct aesni_aes_ctx	 xts_keys[2];
};

struct aesni_session {
	uint32_t		 ses_ekey[4 * (AES_MAXROUNDS + 1)];
	uint32_t		 ses_dkey[4 * (AES_MAXROUNDS + 1)];
	uint32_t		 ses_klen;
	uint8_t			 ses_nonce[AESCTR_NONCESIZE];
	int			 ses_sid;
	GHASH_CTX		*ses_ghash;
	struct aesni_xts_ctx	*ses_xts;
	struct swcr_data	*ses_swd;
	SMR_LIST_ENTRY(aesni_session)
				 ses_entries;
	uint8_t			*ses_buf;
	size_t			 ses_buflen;
	struct smr_entry	 ses_smr;
};

struct aesni_softc {
	int32_t			 sc_cid;
	uint32_t		 sc_sid;
	struct mutex		 sc_mtx;
	SMR_LIST_HEAD(, aesni_session)
				 sc_sessions;
} *aesni_sc;

struct pool aesnipl;

uint32_t aesni_ops;

/* assembler-assisted key setup */
extern void aesni_set_key(struct aesni_session *ses, uint8_t *key, size_t len);

/* aes encryption/decryption */
extern void aesni_enc(struct aesni_session *ses, uint8_t *dst, uint8_t *src);
extern void aesni_dec(struct aesni_session *ses, uint8_t *dst, uint8_t *src);

/* assembler-assisted CBC mode */
extern void aesni_cbc_enc(struct aesni_session *ses, uint8_t *dst,
	    uint8_t *src, size_t len, uint8_t *iv);
extern void aesni_cbc_dec(struct aesni_session *ses, uint8_t *dst,
	    uint8_t *src, size_t len, uint8_t *iv);

/* assembler-assisted CTR mode */
extern void aesni_ctr_enc(struct aesni_session *ses, uint8_t *dst,
	    uint8_t *src, size_t len, uint8_t *icb);

/* assembler-assisted XTS mode */
extern void aesni_xts_enc(struct aesni_xts_ctx *xts, uint8_t *dst,
	    uint8_t *src, size_t len, uint8_t *tweak);
extern void aesni_xts_dec(struct aesni_xts_ctx *xts, uint8_t *dst,
	    uint8_t *src, size_t len, uint8_t *tweak);

/* assembler-assisted GMAC */
extern void aesni_gmac_update(GHASH_CTX *ghash, uint8_t *src, size_t len);
extern void aesni_gmac_final(struct aesni_session *ses, uint8_t *tag,
    uint8_t *icb, uint8_t *hashstate);

void	aesni_setup(void);
int	aesni_newsession(u_int32_t *, struct cryptoini *);
int	aesni_freesession(u_int64_t);
int	aesni_process(struct cryptop *);

struct aesni_session *
	aesni_get(uint32_t);
void	aesni_free(struct aesni_session *);
void	aesni_free_smr(void *);

int	aesni_swauth(struct cryptop *, struct cryptodesc *, struct swcr_data *,
	    caddr_t);

int	aesni_encdec(struct cryptop *, struct cryptodesc *,
	    struct cryptodesc *, struct aesni_session *);

void	pclmul_setup(void);
void	ghash_update_pclmul(GHASH_CTX *, uint8_t *, size_t);

void
aesni_setup(void)
{
	int algs[CRYPTO_ALGORITHM_MAX + 1];

	aesni_sc = malloc(sizeof(*aesni_sc), M_DEVBUF, M_NOWAIT|M_ZERO);
	if (aesni_sc == NULL)
		return;

	bzero(algs, sizeof(algs));

	/* Encryption algorithms. */
	algs[CRYPTO_AES_CBC] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_AES_CTR] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_AES_GCM_16] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_AES_XTS] = CRYPTO_ALG_FLAG_SUPPORTED;

	/* Authenticated encryption algorithms. */
	algs[CRYPTO_AES_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_AES_128_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_AES_192_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_AES_256_GMAC] = CRYPTO_ALG_FLAG_SUPPORTED;

	/* HMACs needed for IPsec, uses software crypto. */
	algs[CRYPTO_MD5_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_SHA1_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_RIPEMD160_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_SHA2_256_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_SHA2_384_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;
	algs[CRYPTO_SHA2_512_HMAC] = CRYPTO_ALG_FLAG_SUPPORTED;

	/* IPsec Extended Sequence Numbers. */
	algs[CRYPTO_ESN] = CRYPTO_ALG_FLAG_SUPPORTED;

	aesni_sc->sc_cid = crypto_get_driverid(CRYPTOCAP_F_MPSAFE);
	if (aesni_sc->sc_cid < 0) {
		free(aesni_sc, M_DEVBUF, sizeof(*aesni_sc));
		aesni_sc = NULL;
		return;
	}

	pool_init(&aesnipl, sizeof(struct aesni_session), 16, IPL_VM, 0,
	    "aesni", NULL);
	pool_setlowat(&aesnipl, 2);

	mtx_init(&aesni_sc->sc_mtx, IPL_VM);

	crypto_register(aesni_sc->sc_cid, algs, aesni_newsession,
	    aesni_freesession, aesni_process);
}

int
aesni_newsession(u_int32_t *sidp, struct cryptoini *cri)
{
	struct aesni_session *ses = NULL;
	struct aesni_aes_ctx *aes1, *aes2;
	struct cryptoini *c;
	const struct auth_hash *axf;
	struct swcr_data *swd;
	int i;

	if (sidp == NULL || cri == NULL)
		return (EINVAL);

	ses = pool_get(&aesnipl, PR_NOWAIT | PR_ZERO);
	if (!ses)
		return (ENOMEM);
	smr_init(&ses->ses_smr);

	ses->ses_buf = malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT|M_ZERO);
	if (ses->ses_buf != NULL)
		ses->ses_buflen = PAGE_SIZE;

	for (c = cri; c != NULL; c = c->cri_next) {
		switch (c->cri_alg) {
		case CRYPTO_AES_CBC:
			ses->ses_klen = c->cri_klen / 8;
			fpu_kernel_enter();
			aesni_set_key(ses, c->cri_key, ses->ses_klen);
			fpu_kernel_exit();
			break;

		case CRYPTO_AES_CTR:
		case CRYPTO_AES_GCM_16:
		case CRYPTO_AES_GMAC:
			ses->ses_klen = c->cri_klen / 8 - AESCTR_NONCESIZE;
			memcpy(ses->ses_nonce, c->cri_key + ses->ses_klen,
			    AESCTR_NONCESIZE);
			fpu_kernel_enter();
			aesni_set_key(ses, c->cri_key, ses->ses_klen);
			fpu_kernel_exit();
			break;

		case CRYPTO_AES_XTS:
			ses->ses_xts = malloc(sizeof(struct aesni_xts_ctx),
			    M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
			if (ses->ses_xts == NULL) {
				aesni_free(ses);
				return (ENOMEM);
			}

			ses->ses_klen = c->cri_klen / 16;
			aes1 = &ses->ses_xts->xts_keys[0];
			aes1->aes_klen = ses->ses_klen;
			aes2 = &ses->ses_xts->xts_keys[1];
			aes2->aes_klen = ses->ses_klen;

			fpu_kernel_enter();
			aesni_set_key((struct aesni_session *)aes1,
			    c->cri_key, aes1->aes_klen);
			aesni_set_key((struct aesni_session *)aes2,
			    c->cri_key + ses->ses_klen, aes2->aes_klen);
			fpu_kernel_exit();
			break;

		case CRYPTO_AES_128_GMAC:
		case CRYPTO_AES_192_GMAC:
		case CRYPTO_AES_256_GMAC:
			ses->ses_ghash = malloc(sizeof(GHASH_CTX),
			    M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
			if (ses->ses_ghash == NULL) {
				aesni_free(ses);
				return (ENOMEM);
			}

			/* prepare a hash subkey */
			fpu_kernel_enter();
			aesni_enc(ses, ses->ses_ghash->H, ses->ses_ghash->H);
			fpu_kernel_exit();
			break;

		case CRYPTO_MD5_HMAC:
			axf = &auth_hash_hmac_md5_96;
			goto authcommon;
		case CRYPTO_SHA1_HMAC:
			axf = &auth_hash_hmac_sha1_96;
			goto authcommon;
		case CRYPTO_RIPEMD160_HMAC:
			axf = &auth_hash_hmac_ripemd_160_96;
			goto authcommon;
		case CRYPTO_SHA2_256_HMAC:
			axf = &auth_hash_hmac_sha2_256_128;
			goto authcommon;
		case CRYPTO_SHA2_384_HMAC:
			axf = &auth_hash_hmac_sha2_384_192;
			goto authcommon;
		case CRYPTO_SHA2_512_HMAC:
			axf = &auth_hash_hmac_sha2_512_256;
		authcommon:
			swd = malloc(sizeof(struct swcr_data), M_CRYPTO_DATA,
			    M_NOWAIT|M_ZERO);
			if (swd == NULL) {
				aesni_free(ses);
				return (ENOMEM);
			}
			ses->ses_swd = swd;

			swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
			    M_NOWAIT);
			if (swd->sw_ictx == NULL) {
				aesni_free(ses);
				return (ENOMEM);
			}

			swd->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA,
			    M_NOWAIT);
			if (swd->sw_octx == NULL) {
				aesni_free(ses);
				return (ENOMEM);
			}

			for (i = 0; i < c->cri_klen / 8; i++)
				c->cri_key[i] ^= HMAC_IPAD_VAL;

			axf->Init(swd->sw_ictx);
			axf->Update(swd->sw_ictx, c->cri_key, c->cri_klen / 8);
			axf->Update(swd->sw_ictx, hmac_ipad_buffer,
			    axf->blocksize - (c->cri_klen / 8));

			for (i = 0; i < c->cri_klen / 8; i++)
				c->cri_key[i] ^= (HMAC_IPAD_VAL ^
				    HMAC_OPAD_VAL);

			axf->Init(swd->sw_octx);
			axf->Update(swd->sw_octx, c->cri_key, c->cri_klen / 8);
			axf->Update(swd->sw_octx, hmac_opad_buffer,
			    axf->blocksize - (c->cri_klen / 8));

			for (i = 0; i < c->cri_klen / 8; i++)
				c->cri_key[i] ^= HMAC_OPAD_VAL;

			swd->sw_axf = axf;
			swd->sw_alg = c->cri_alg;

			break;

		case CRYPTO_ESN:
			/* nothing to do */
			break;

		default:
			aesni_free(ses);
			return (EINVAL);
		}
	}

	mtx_enter(&aesni_sc->sc_mtx);
	ses->ses_sid = ++aesni_sc->sc_sid;
	SMR_LIST_INSERT_HEAD_LOCKED(&aesni_sc->sc_sessions, ses, ses_entries);
	mtx_leave(&aesni_sc->sc_mtx);

	*sidp = ses->ses_sid;
	return (0);
}

int
aesni_freesession(u_int64_t tid)
{
	struct aesni_session *ses;
	u_int32_t sid = (u_int32_t)tid;

	mtx_enter(&aesni_sc->sc_mtx);
	SMR_LIST_FOREACH_LOCKED(ses, &aesni_sc->sc_sessions, ses_entries) {
		if (ses->ses_sid == sid) {
			SMR_LIST_REMOVE_LOCKED(ses, ses_entries);
			break;
		}
	}
	mtx_leave(&aesni_sc->sc_mtx);

	if (ses == NULL)
		return (EINVAL);

	smr_call(&ses->ses_smr, aesni_free_smr, ses);

	return (0);
}

void
aesni_free(struct aesni_session *ses)
{
	struct swcr_data *swd;
	const struct auth_hash *axf;

	if (ses->ses_ghash) {
		explicit_bzero(ses->ses_ghash, sizeof(GHASH_CTX));
		free(ses->ses_ghash, M_CRYPTO_DATA, sizeof(GHASH_CTX));
	}

	if (ses->ses_xts) {
		explicit_bzero(ses->ses_xts, sizeof(struct aesni_xts_ctx));
		free(ses->ses_xts, M_CRYPTO_DATA, sizeof(struct aesni_xts_ctx));
	}

	if (ses->ses_swd) {
		swd = ses->ses_swd;
		axf = swd->sw_axf;

		if (swd->sw_ictx) {
			explicit_bzero(swd->sw_ictx, axf->ctxsize);
			free(swd->sw_ictx, M_CRYPTO_DATA, axf->ctxsize);
		}
		if (swd->sw_octx) {
			explicit_bzero(swd->sw_octx, axf->ctxsize);
			free(swd->sw_octx, M_CRYPTO_DATA, axf->ctxsize);
		}
		free(swd, M_CRYPTO_DATA, sizeof(*swd));
	}

	if (ses->ses_buf) {
		explicit_bzero(ses->ses_buf, ses->ses_buflen);
		free(ses->ses_buf, M_DEVBUF, ses->ses_buflen);
	}

	explicit_bzero(ses, sizeof (*ses));
	pool_put(&aesnipl, ses);
}

void
aesni_free_smr(void *arg)
{
	struct aesni_session *ses = arg;

	aesni_free(ses);
}

struct aesni_session *
aesni_get(uint32_t sid)
{
	struct aesni_session *ses = NULL;

	SMR_ASSERT_CRITICAL();
	SMR_LIST_FOREACH(ses, &aesni_sc->sc_sessions, ses_entries) {
		if (ses->ses_sid == sid)
			break;
	}
	return (ses);
}

int
aesni_swauth(struct cryptop *crp, struct cryptodesc *crd,
    struct swcr_data *sw, caddr_t buf)
{
	int type;

	if (crp->crp_flags & CRYPTO_F_IMBUF)
		type = CRYPTO_BUF_MBUF;
	else
		type = CRYPTO_BUF_IOV;

	return (swcr_authcompute(crp, crd, sw, buf, type));
}

int
aesni_encdec(struct cryptop *crp, struct cryptodesc *crd,
    struct cryptodesc *crda, struct aesni_session *ses)
{
	int aadlen, err, ivlen, iskip, oskip, rlen;
	uint8_t iv[EALG_MAX_BLOCK_LEN];
	uint8_t icb[AESCTR_BLOCKSIZE];
	uint8_t tag[GMAC_DIGEST_LEN];
	uint8_t *buf = ses->ses_buf;
	uint32_t *dw;

	aadlen = rlen = err = iskip = oskip = 0;

	if (crd->crd_len > ses->ses_buflen) {
		if (buf != NULL) {
			explicit_bzero(buf, ses->ses_buflen);
			free(buf, M_DEVBUF, ses->ses_buflen);
		}

		ses->ses_buflen = 0;
		rlen = roundup(crd->crd_len, EALG_MAX_BLOCK_LEN);
		ses->ses_buf = buf = malloc(rlen, M_DEVBUF, M_NOWAIT |
		    M_ZERO);
		if (buf == NULL)
			return (ENOMEM);
		ses->ses_buflen = rlen;
	}

	/* CBC uses 16, CTR/XTS only 8. */
	ivlen = (crd->crd_alg == CRYPTO_AES_CBC) ? 16 : 8;

	/* Initialize the IV */
	if (crd->crd_flags & CRD_F_ENCRYPT) {
		if (crd->crd_flags & CRD_F_IV_EXPLICIT)
			memcpy(iv, crd->crd_iv, ivlen);
		else
			arc4random_buf(iv, ivlen);

		/* Do we need to write the IV */
		if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) {
			if (crp->crp_flags & CRYPTO_F_IMBUF) {
				if (m_copyback((struct mbuf *)crp->crp_buf,
				    crd->crd_inject, ivlen, iv, M_NOWAIT)) {
					err = ENOMEM;
					goto out;
				}
			} else
				cuio_copyback((struct uio *)crp->crp_buf,
				    crd->crd_inject, ivlen, iv);
		}
	} else {
		if (crd->crd_flags & CRD_F_IV_EXPLICIT)
			memcpy(iv, crd->crd_iv, ivlen);
		else {
			if (crp->crp_flags & CRYPTO_F_IMBUF)
				m_copydata((struct mbuf *)crp->crp_buf,
				    crd->crd_inject, ivlen, iv);
			else
				cuio_copydata((struct uio *)crp->crp_buf,
				    crd->crd_inject, ivlen, iv);
		}
	}

	if (crda) {
		/* Supply GMAC with AAD */
		aadlen = crda->crd_len;
		if (crda->crd_flags & CRD_F_ESN) {
			aadlen += 4;
			/* SPI */
			if (crp->crp_flags & CRYPTO_F_IMBUF)
				m_copydata((struct mbuf *)crp->crp_buf,
				    crda->crd_skip, 4, buf);
			else
				cuio_copydata((struct uio *)crp->crp_buf,
				    crda->crd_skip, 4, buf);
			iskip = 4; /* additional input offset */
			/* ESN */
			memcpy(buf + 4, crda->crd_esn, 4);
			oskip = iskip + 4; /* offset output buffer by 8 */
		}
		rlen = roundup(aadlen, GMAC_BLOCK_LEN);
		if (crp->crp_flags & CRYPTO_F_IMBUF)
			m_copydata((struct mbuf *)crp->crp_buf,
			    crda->crd_skip + iskip, crda->crd_len - iskip,
			    buf + oskip);
		else
			cuio_copydata((struct uio *)crp->crp_buf,
			    crda->crd_skip + iskip, crda->crd_len - iskip,
			    buf + oskip);
		fpu_kernel_enter();
		aesni_gmac_update(ses->ses_ghash, buf, rlen);
		fpu_kernel_exit();
		bzero(buf, aadlen);
	}

	/* Copy data to be processed to the buffer */
	if (crp->crp_flags & CRYPTO_F_IMBUF)
		m_copydata((struct mbuf *)crp->crp_buf, crd->crd_skip,
		    crd->crd_len, buf);
	else
		cuio_copydata((struct uio *)crp->crp_buf, crd->crd_skip,
		    crd->crd_len, buf);

	if (crd->crd_alg == CRYPTO_AES_CTR ||
	    crd->crd_alg == CRYPTO_AES_GCM_16 ||
	    crd->crd_alg == CRYPTO_AES_GMAC) {
		bzero(icb, AESCTR_BLOCKSIZE);
		memcpy(icb, ses->ses_nonce, AESCTR_NONCESIZE);
		memcpy(icb + AESCTR_NONCESIZE, iv, AESCTR_IVSIZE);
		/* rlen is for gcm and gmac only */
		rlen = roundup(crd->crd_len, AESCTR_BLOCKSIZE);
	}

	/* Apply cipher */
	fpu_kernel_enter();
	switch (crd->crd_alg) {
	case CRYPTO_AES_CBC:
		if (crd->crd_flags & CRD_F_ENCRYPT)
			aesni_cbc_enc(ses, buf, buf, crd->crd_len, iv);
		else
			aesni_cbc_dec(ses, buf, buf, crd->crd_len, iv);
		break;
	case CRYPTO_AES_CTR:
		aesni_ctr_enc(ses, buf, buf, crd->crd_len, icb);
		break;
	case CRYPTO_AES_GCM_16:
		icb[AESCTR_BLOCKSIZE - 1] = 1;
		if (crd->crd_flags & CRD_F_ENCRYPT) {
			/* encrypt padded data */
			aesni_ctr_enc(ses, buf, buf, rlen, icb);
			/* zero out padding bytes */
			bzero(buf + crd->crd_len, rlen - crd->crd_len);
			/* hash encrypted data padded with zeroes */
			aesni_gmac_update(ses->ses_ghash, buf, rlen);
		} else {
			aesni_gmac_update(ses->ses_ghash, buf, rlen);
			aesni_ctr_enc(ses, buf, buf, rlen, icb);
		}
		goto gcmcommon;
	case CRYPTO_AES_GMAC:
		icb[AESCTR_BLOCKSIZE - 1] = 1;
		aesni_gmac_update(ses->ses_ghash, buf, rlen);
	gcmcommon:
		/* lengths block */
		bzero(tag, GMAC_BLOCK_LEN);
		dw = (uint32_t *)tag + 1;
		*dw = htobe32(aadlen * 8);
		dw = (uint32_t *)tag + 3;
		*dw = htobe32(crd->crd_len * 8);
		aesni_gmac_update(ses->ses_ghash, tag, GMAC_BLOCK_LEN);
		/* finalization */
		aesni_gmac_final(ses, tag, icb, ses->ses_ghash->S);
		break;
	case CRYPTO_AES_XTS:
		if (crd->crd_flags & CRD_F_ENCRYPT)
			aesni_xts_enc(ses->ses_xts, buf, buf, crd->crd_len, iv);
		else
			aesni_xts_dec(ses->ses_xts, buf, buf, crd->crd_len, iv);
		break;
	}
	fpu_kernel_exit();

	aesni_ops++;

	/* Copy back the result */
	if (crp->crp_flags & CRYPTO_F_IMBUF) {
		if (m_copyback((struct mbuf *)crp->crp_buf, crd->crd_skip,
		    crd->crd_len, buf, M_NOWAIT)) {
			err = ENOMEM;
			goto out;
		}
	} else
		cuio_copyback((struct uio *)crp->crp_buf, crd->crd_skip,
		    crd->crd_len, buf);

	/* Copy back the authentication tag */
	if (crda) {
		if (crp->crp_flags & CRYPTO_F_IMBUF) {
			if (m_copyback((struct mbuf *)crp->crp_buf,
			    crda->crd_inject, GMAC_DIGEST_LEN, tag,
			    M_NOWAIT)) {
				err = ENOMEM;
				goto out;
			}
		} else
			memcpy(crp->crp_mac, tag, GMAC_BLOCK_LEN);

		/* clean up GHASH state */
		bzero(ses->ses_ghash->S, GMAC_BLOCK_LEN);
		bzero(ses->ses_ghash->Z, GMAC_BLOCK_LEN);
	}

out:
	explicit_bzero(buf, roundup(crd->crd_len, EALG_MAX_BLOCK_LEN));
	return (err);
}

int
aesni_process(struct cryptop *crp)
{
	struct aesni_session *ses;
	struct cryptodesc *crd, *crda, *crde;
	int err = 0;
	int i;

	KASSERT(crp->crp_ndesc >= 1);

	smr_read_enter();
	ses = aesni_get(crp->crp_sid & 0xffffffff);
	if (!ses) {
		err = EINVAL;
		goto out;
	}

	crda = crde = NULL;
	for (i = 0; i < crp->crp_ndesc; i++) {
		crd = &crp->crp_desc[i];
		switch (crd->crd_alg) {
		case CRYPTO_AES_CBC:
		case CRYPTO_AES_CTR:
		case CRYPTO_AES_XTS:
			err = aesni_encdec(crp, crd, NULL, ses);
			if (err != 0)
				goto out;
			break;

		case CRYPTO_AES_GCM_16:
		case CRYPTO_AES_GMAC:
			crde = crd;
			if (!crda)
				continue;
			goto gcmcommon;
		case CRYPTO_AES_128_GMAC:
		case CRYPTO_AES_192_GMAC:
		case CRYPTO_AES_256_GMAC:
			crda = crd;
			if (!crde)
				continue;
		gcmcommon:
			err = aesni_encdec(crp, crde, crda, ses);
			if (err != 0)
				goto out;
			break;

		case CRYPTO_MD5_HMAC:
		case CRYPTO_SHA1_HMAC:
		case CRYPTO_RIPEMD160_HMAC:
		case CRYPTO_SHA2_256_HMAC:
		case CRYPTO_SHA2_384_HMAC:
		case CRYPTO_SHA2_512_HMAC:
			err = aesni_swauth(crp, crd, ses->ses_swd,
			    crp->crp_buf);
			if (err != 0)
				goto out;
			break;

		default:
			err = EINVAL;
			goto out;
		}
	}

out:
	smr_read_leave();
	return (err);
}

void
pclmul_setup(void)
{
	ghash_update = ghash_update_pclmul;
}

void
ghash_update_pclmul(GHASH_CTX *ghash, uint8_t *src, size_t len)
{
	fpu_kernel_enter();
	aesni_gmac_update(ghash, src, len);
	fpu_kernel_exit();
}