/*	$OpenBSD: mds.S,v 1.5 2024/04/14 09:59:04 kettenis Exp $	*/
/*
 * Copyright (c) 2019 Philip Guenther <guenther@openbsd.org>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */
/*
 * ASM sequences for mitigating MDS on different Intel CPU models, taken from
 *	https://software.intel.com/security-software-guidance/insights/deep-dive-intel-analysis-microarchitectural-data-sampling
 * and adjusted to fit OpenBSD style and kernel usage.
 * Some naming inspired by FreeBSD's usage of these sequences.
 */

#include "assym.h"

#include <machine/asm.h>
#include <machine/specialreg.h>

ENTRY(mds_handler_ivb)
	RETGUARD_SETUP(mds_handler_ivb, r11)
	movq	CPUVAR(MDS_BUF),%rax
	movdqa	%xmm0,CPUVAR(MDS_TMP)
	pxor	%xmm0,%xmm0

	lfence
	orpd	(%rax),%xmm0
	orpd	(%rax),%xmm0
	mfence
	movl	$40,%ecx
	addq	$16,%rax
1:	movntdq	%xmm0,(%rax)
	addq	$16,%rax
	decl	%ecx
	jnz	1b
	mfence

	movdqa	CPUVAR(MDS_TMP),%xmm0
	RETGUARD_CHECK(mds_handler_ivb, r11)
	retq
	lfence
END(mds_handler_ivb)

ENTRY(mds_handler_bdw)
	RETGUARD_SETUP(mds_handler_bdw, r11)
	movq	CPUVAR(MDS_BUF),%rax
	movdqa	%xmm0,CPUVAR(MDS_TMP)
	pxor	%xmm0,%xmm0

	movq	%rax,%rdi
	movq	%rax,%rsi
	movl	$40,%ecx
1:	movntdq	%xmm0,(%rax)
	addq	$16,%rax
	decl	%ecx
	jnz	1b
	mfence
	movl	$1536,%ecx
	rep movsb
	lfence

	movdqa	CPUVAR(MDS_TMP),%xmm0
	RETGUARD_CHECK(mds_handler_bdw, r11)
	retq
	lfence
END(mds_handler_bdw)

ENTRY(mds_handler_skl)
	xorl	%ecx,%ecx
	xgetbv
	testb	$XFEATURE_AVX512,%al
	jne	mds_handler_skl_avx512
	testb	$XFEATURE_AVX,%al
	jne	mds_handler_skl_avx
	jmp	mds_handler_skl_sse
END(mds_handler_skl)

ENTRY(mds_handler_skl_sse)
	RETGUARD_SETUP(mds_handler_skl_sse, r11)
	movq	CPUVAR(MDS_BUF),%rax
	leaq	64(%rax),%rdi
	movdqa	%xmm0,CPUVAR(MDS_TMP)
	pxor	%xmm0,%xmm0

	lfence
	orpd	(%rax),%xmm0
	orpd	(%rax),%xmm0
	xorl	%eax,%eax
1:	clflushopt	5376(%rdi,%rax,8)
	addl	$8,%eax
	cmpl	$8*12,%eax
	jb	1b
	sfence
	movl	$6144,%ecx
	xorl	%eax,%eax
	rep stosb
	mfence

	movdqa	CPUVAR(MDS_TMP),%xmm0
	RETGUARD_CHECK(mds_handler_skl_sse, r11)
	retq
	lfence
END(mds_handler_skl_sse)

ENTRY(mds_handler_skl_avx)
	RETGUARD_SETUP(mds_handler_skl_avx, r11)
	movq	CPUVAR(MDS_BUF),%rax
	leaq	64(%rax),%rdi
	vmovdqa	%ymm0,CPUVAR(MDS_TMP)
	vpxor	%ymm0,%ymm0,%ymm0

	lfence
	vorpd	(%rax),%ymm0,%ymm0
	vorpd	(%rax),%ymm0,%ymm0
	xorl	%eax,%eax
1:	clflushopt	5376(%rdi,%rax,8)
	addl	$8,%eax
	cmpl	$8*12,%eax
	jb	1b
	sfence
	movl	$6144,%ecx
	xorl	%eax,%eax
	rep stosb
	mfence

	vmovdqa	CPUVAR(MDS_TMP),%ymm0
	RETGUARD_CHECK(mds_handler_skl_avx, r11)
	retq
	lfence
END(mds_handler_skl_avx)

ENTRY(mds_handler_skl_avx512)
	RETGUARD_SETUP(mds_handler_skl_avx512, r11)
	movq	CPUVAR(MDS_BUF),%rax
	leaq	64(%rax),%rdi
	vmovdqa64	%zmm0,CPUVAR(MDS_TMP)
	vpxorq	%zmm0,%zmm0,%zmm0

	lfence
	vorpd	(%rax),%zmm0,%zmm0
	vorpd	(%rax),%zmm0,%zmm0
	xorl	%eax,%eax
1:	clflushopt	5376(%rdi,%rax,8)
	addl	$8,%eax
	cmpl	$8*12,%eax
	jb	1b
	sfence
	movl	$6144,%ecx
	xorl	%eax,%eax
	rep stosb
	mfence

	vmovdqa64	CPUVAR(MDS_TMP),%zmm0
	RETGUARD_CHECK(mds_handler_skl_avx512, r11)
	retq
	lfence
END(mds_handler_skl_avx512)

ENTRY(mds_handler_silvermont)
	RETGUARD_SETUP(mds_handler_silvermont, r11)
	movq	CPUVAR(MDS_BUF),%rax
	movdqa	%xmm0,CPUVAR(MDS_TMP)
	pxor	%xmm0,%xmm0

	movl	$16,%ecx
1:	movntdq	%xmm0,(%rax)
	addq	$16,%rax
	decl	%ecx
	jnz	1b
	mfence

	movdqa	CPUVAR(MDS_TMP),%xmm0
	RETGUARD_CHECK(mds_handler_silvermont, r11)
	retq
	lfence
END(mds_handler_silvermont)

ENTRY(mds_handler_knights)
	RETGUARD_SETUP(mds_handler_knights, r11)
	movq	CPUVAR(MDS_BUF),%rdi
	xorl	%eax,%eax
	movl	$16,%ecx
	rep stosq
	movl	$128,%ecx
	rep stosq
	mfence
	RETGUARD_CHECK(mds_handler_knights, r11)
	retq
	lfence
END(mds_handler_knights)