/*	$OpenBSD: locore.S,v 1.64 2023/12/12 07:37:21 deraadt Exp $	*/
/*	$NetBSD: locore.S,v 1.2 1996/10/16 19:33:09 ws Exp $	*/

/*
 * Copyright (C) 1995, 1996 Wolfgang Solfrank.
 * Copyright (C) 1995, 1996 TooLs GmbH.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *	This product includes software developed by TooLs GmbH.
 * 4. The name of TooLs GmbH may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "assym.h"

#include <sys/syscall.h>

#include <machine/asm.h>
#include <machine/param.h>
#include <machine/pmap.h>
#include <machine/psl.h>
#include <machine/trap.h>

#define GET_CPUINFO(r)	mfsprg r,0

#define	INTSTK		(8*1024)	/* 8K interrupt stack */
#define	SPILLSTK	(1*1024)
#define	DDBSTK		(7*1024)

	.text

#ifdef MULTIPROCESSOR
_ENTRY(cpu_spinup_trampoline)
	lis	%r3,cpu_hatch_stack@ha
	lwz	%r1,cpu_hatch_stack@l(%r3)

	b	cpu_hatch
	/* NOTREACHED */
#endif

/*
 * void cpu_switchto(struct proc *old, struct proc *new)
 * Switch from "old" proc to "new".
 */
_ENTRY(cpu_switchto_asm)
	mflr	%r0			/* save lr */
	stw	%r0,4(%r1)
	stwu	%r1,(-SFRAMELEN - 16)(%r1)
	stw	%r31,(SFRAMELEN + 12)(%r1)
	stw	%r30,(SFRAMELEN + 8)(%r1)

/*
 * r3 - old proc
 * r4 -	new proc
 * r5 - cpuinfo
 */
	GET_CPUINFO(%r5)

	li	%r31,SONPROC
	stb	%r31,P_STAT(%r4)

	or.	%r3,%r3,%r3		/* old process was exiting? */
	beq	switch_exited

	mfsr	%r10,PPC_USER_SR	/* save PPC_USER_SR for copyin/copyout*/
	mfcr	%r11			/* save cr */
	mr	%r12,%r2		/* save r2 */
	stmw	%r10,8(%r1)
	lwz	%r31,P_ADDR(%r3)
	stw	%r1,PCB_SP(%r31)	/* save SP */

switch_exited:
	/* disable interrupts while actually switching */
	mfmsr	%r30
	andi.	%r30,%r30,~PSL_EE@l
	mtmsr	%r30
	isync

	stw	%r4,CI_CURPROC(%r5)	/* record new process */

#ifdef MULTIPROCESSOR
	stw	%r5,P_CPU(%r4)
#endif

	lwz	%r31,P_ADDR(%r4)
	stw	%r31,CI_CURPCB(%r5)	/* indicate new pcb */

	lwz	%r6,PCB_PMR(%r31)

	/* save real pmap pointer for spill fill */
	stwu	%r6,CI_CURPM(%r5)
	stwcx.	%r6,%r0,%r5		/* clear possible reservation */

	addic.	%r6,%r6,64
	li	%r5,0

	lwz	%r1,PCB_SP(%r31)	/* get new procs SP */
	lwz	%r0,(SFRAMELEN + 16 + 4)(%r1)
	RETGUARD_SETUP_LATE(cpu_switchto_asm, %r9, %r0)

	ori	%r30,%r30,PSL_EE	/* interrupts are okay again */
	mtmsr	%r30

	lmw	%r10,8(%r1)		/* get other regs */
	mr	%r2,%r12		/* get saved r2 */
	mtcr	%r11			/* get saved cr */
	isync
	mtsr	PPC_USER_SR,%r10	/* get saved PPC_USER_SR */
	isync

	lwz	%r31,(SFRAMELEN + 12)(%r1)
	lwz	%r30,(SFRAMELEN + 8)(%r1)
	addi	%r1,%r1,SFRAMELEN + 16
	mtlr	%r0
	RETGUARD_CHECK(cpu_switchto_asm, %r9, %r0)
	blr

_ENTRY(cpu_idle_enter)
	RETGUARD_SETUP(cpu_idle_enter, %r11, %r12)
	lis	%r4, ppc_cpuidle@ha
	lwz	%r4, ppc_cpuidle@l(%r4)
	cmpwi	%r4, 0
	beq	1f
	/* must disable external interrupts during idle queue checking */
	mfmsr	%r3
	andi.	%r3,%r3,~PSL_EE@l
	mtmsr	%r3
	isync
1:
	RETGUARD_CHECK(cpu_idle_enter, %r11, %r12)
	blr

_ENTRY(cpu_idle_cycle)
	RETGUARD_SETUP(cpu_idle_cycle, %r11, %r12)
	lis	%r4, ppc_cpuidle@ha
	lwz	%r4, ppc_cpuidle@l(%r4)
	cmpwi	%r4, 0
	beq	idledone

	/*
	 * Interrupts were disabled in cpu_idle_enter, but must be
	 * enabled for sleeping, but before that indicate that we
	 * are entering sleep mode.
	 */
	GET_CPUINFO(%r4)
	lwz	%r6,CI_FLAGS(%r4)
	ori	%r6,%r6,CI_FLAGS_SLEEPING@l
	stw	%r6,CI_FLAGS(%r4)

	/* enable interrupts, required before setting POW */
	mfmsr	%r3
	ori	%r5,%r3,PSL_EE@l
	mtmsr	%r5
	oris	%r5, %r5, PSL_POW@h
	sync
	/* low power mode */
	mtmsr	%r5
	sync
	isync

	/*
	 * restore interrupts to disabled, so CI_FLAGS is only modified
	 * with interrupts disabled.
	 */
	mtmsr %r3
	isync

	/* clear CI_FLAGS_SLEEPING since sleeping is over */
	andi.	%r6,%r6,~CI_FLAGS_SLEEPING@l
	stw	%r6,CI_FLAGS(%r4)
idledone:
	RETGUARD_CHECK(cpu_idle_cycle, %r11, %r12)
	blr

_ENTRY(cpu_idle_leave)
	RETGUARD_SETUP(cpu_idle_leave, %r11, %r12)
	lis	%r4, ppc_cpuidle@ha
	lwz	%r4, ppc_cpuidle@l(%r4)
	cmpwi	%r4, 0
	beq	1f
	/* enable interrupts disabled in cpu_idle_enter. */
	mfmsr	%r3
	ori	%r3,%r3,PSL_EE@l
	mtmsr	%r3
1:
	RETGUARD_CHECK(cpu_idle_leave, %r11, %r12)
	blr

/*
 * This code gets copied to all the trap vectors
 * except ISI/DSI, ALI, and the interrupts
 */
	.text
	.globl	trapcode,trapsize
	.type	trapcode,@function
	.type	trapsize,@object
trapcode:
	mtsprg	1,%r1			/* save SP */
nop32_1s:
	mfmsr	%r1
	clrldi	%r1,%r1,1
	mtmsrd	%r1
nop32_1e:
	GET_CPUINFO(%r1)
	stmw	%r28,CI_TEMPSAVE(%r1)	/* free r28-r31 */
	mflr	%r28			/* save LR */
	mfcr	%r29			/* save CR */
	mfsprg	%r1,1			/* restore SP */

	/* Test whether we already had PR set */
	mfsrr1	%r31
	mtcr	%r31
	bc	4,17,1f			/* branch if PSL_PR is clear */
	GET_CPUINFO(%r1)
	lwz	%r1,CI_CURPCB(%r1)
	addi	%r1,%r1,USPACE		/* stack is top of user struct */
1:
	bla	s_trap
trapsize =		.-trapcode

/*
 * For ALI: has to save DSISR and DAR
 */
	.globl	alitrap,alisize
alitrap:
	mtsprg	1,%r1			/* save SP */
nop32_2s:
	mfmsr	%r1
	clrldi	%r1,%r1,1
	mtmsrd	%r1
nop32_2e:
	GET_CPUINFO(%r1)
	stmw	%r28,CI_TEMPSAVE(%r1)	/* free r28-r31 */
	mfdar	%r30
	mfdsisr	%r31
	stmw	%r30,CI_TEMPSAVE+16(%r1)
	mflr	%r28			/* save LR */
	mfcr	%r29			/* save CR */
	mfsprg	%r1,1			/* restore SP */

	/* Test whether we already had PR set */
	mfsrr1	%r31
	mtcr	%r31
	bc	4,17,1f			/* branch if PSL_PR is clear */
	GET_CPUINFO(%r1)
	lwz	%r1,CI_CURPCB(%r1)
	addi	%r1,%r1,USPACE		/* stack is top of user struct */
1:
	bla	s_trap
alisize =		.-alitrap

/*
 * Similar to the above for DSI
 * Has to handle BAT spills
 * and standard pagetable spills
 */
	.globl	dsitrap,dsisize
	.type	dsitrap,@function
	.type	dsisize,@object
dsitrap:
	mtsprg	1,%r1
	GET_CPUINFO(%r1)
	stmw	%r28,CI_DISISAVE(%r1)	/* free r28-r31 */
nop32_3s:
	mfmsr	%r28
	clrldi	%r28,%r28,1
	mtmsrd	%r28
nop32_3e:
	mfsprg	%r1,1
	mfcr	%r29			/* save CR */
	mfxer	%r30			/* save XER */
	mtsprg	2,%r30			/* in SPRG2 */
	mfsrr1	%r31			/* test kernel mode */
nopbat_1s:
	mtcr	%r31
	bc	12,17,1f		/* branch if PSL_PR is set */
	mfdar	%r31			/* get fault address */
	rlwinm	%r31,%r31,7,25,28	/* get segment * 8 */
	addis	%r31,%r31,battable@ha
	lwz	%r30,battable@l(%r31)		/* get batu */
	mtcr	%r30
	bc	4,30,1f			/* branch if supervisor valid is false */
	lwz	%r31,battable+4@l(%r31)			/* get batl */
/* We randomly use the highest two bat registers here */
	mftb	%r28
	andi.	%r28,%r28,1
	bne	2f
	mtdbatu	2,%r30
	mtdbatl	2,%r31
	b	3f
2:
	mtdbatu	3,%r30
	mtdbatl	3,%r31
3:
	mfsprg	%r30,2			/* restore XER */
	mtxer	%r30
	mtcr	%r29			/* restore CR */
	mtsprg	1,%r1
	GET_CPUINFO(%r1)
	lmw	%r28,CI_DISISAVE(%r1)	/* restore r28-r31 */
	mfsprg	1,%r1
	rfi				/* return to trapped code */
1:
nopbat_1e:
	mflr	%r28			/* save LR */
	bla	s_dsitrap
dsisize =		.-dsitrap

/*
 * Similar to the above for ISI
 */
	.globl	isitrap,isisize
	.type	isitrap,@function
	.type	isisize,@object
isitrap:
	mtsprg	1,%r1                   /* save SP */
nop32_4s:
	mfmsr	%r1
	clrldi	%r1,%r1,1
	mtmsrd	%r1
nop32_4e:
	GET_CPUINFO(%r1)
	stmw	%r28,CI_DISISAVE(%r1)	/* free r28-r31 */
	mflr	%r28			/* save LR */
	mfcr	%r29			/* save CR */
	mfsrr1	%r31			/* test kernel mode */
	mfsprg	%r1,1                   /* restore SP */
	bla	s_isitrap
isisize =		.-isitrap

/*
 * This one for the external interrupt handler.
 */
	.globl	extint,extsize
	.type	extint,@function
	.type	extsize,@object
extint:
	mtsprg	1,%r1			/* save SP */
nop32_5s:
	mfmsr	%r1
	clrldi	%r1,%r1,1
	mtmsrd	%r1
nop32_5e:
	GET_CPUINFO(%r1)
	stmw	%r28,CI_TEMPSAVE(%r1)	/* free r28-r31 */
	mflr	%r28			/* save LR */
	mfcr	%r29			/* save CR */
	mfxer	%r30			/* save XER */
	lwz	%r31,CI_INTRDEPTH(%r1)	/* were we already running on intstk? */
	addic.	%r31,%r31,1
	stw	%r31,CI_INTRDEPTH(%r1)
	lwz	%r1,CI_INTSTK(%r1)	/* get interrupt stack */
	beq	1f
	mfsprg	%r1,1			/* yes, get old SP */
1:
	ba	extintr
extsize =		.-extint

/*
 * And this one for the decrementer interrupt handler.
 */
	.globl	decrint,decrsize
	.type	decrint,@function
	.type	decrsize,@object
decrint:
	mtsprg	1,%r1			/* save SP */
nop32_6s:
	mfmsr	%r1
	clrldi	%r1,%r1,1
	mtmsrd	%r1
nop32_6e:
	GET_CPUINFO(%r1)
	stmw	%r28,CI_TEMPSAVE(%r1)	/* free r28-r31 */
	mflr	%r28			/* save LR */
	mfcr	%r29			/* save CR */
	mfxer	%r30			/* save XER */
	lwz	%r31,CI_INTRDEPTH(%r1)	/* were we already running on intstk? */
	addic.	%r31,%r31,1
	stw	%r31,CI_INTRDEPTH(%r1)
	lwz	%r1,CI_INTSTK(%r1)	/* get interrupt stack */
	beq	1f
	mfsprg	%r1,1			/* yes, get old SP */
1:
	ba	decrintr
decrsize =		.-decrint

/*
 * Now the tlb software load for 603 processors:
 * (Code essentially from the 603e User Manual, Chapter 5)
 */
#define	DMISS	976
#define	DCMP	977
#define	HASH1	978
#define	HASH2	979
#define	IMISS	980
#define	ICMP	981
#define	RPA	982

#define	bdneq	bdnzf 2,
#define	tlbli	.long	0x7c0007e4+0x800*
#define	tlbld	.long	0x7c0007a4+0x800*

	.globl	tlbimiss,tlbimsize
	.type	tlbimiss,@function
	.type	tlbimsize,@object
tlbimiss:
	mfspr	%r2,HASH1		/* get first pointer */
	li	%r1,8
	mfctr	%r0			/* save counter */
	mfspr	%r3,ICMP		/* get first compare value */
	addi	%r2,%r2,-8		/* predec pointer */
1:
	mtctr	%r1			/* load counter */
2:
	lwzu	%r1,8(%r2)		/* get next pte */
	cmplw	0,%r1,%r3		/* see if found pte */
	bdneq	2b			/* loop if not eq */
	bne	3f			/* not found */
	lwz	%r1,4(%r2)		/* load tlb entry lower word */
	andi.	%r3,%r1,8		/* check G-bit */
	bne	4f			/* if guarded, take ISI */
	mtctr	%r0			/* restore counter */
	mfspr	%r0,IMISS		/* get the miss address for the tlbli */
	mfsrr1	%r3			/* get the saved cr0 bits */
	mtcrf	0x80,%r3		/* and restore */
	ori	%r1,%r1,0x100		/* set the reference bit */
	mtspr	RPA,%r1			/* set the pte */
	srwi	%r1,%r1,8		/* get byte 7 of pte */
	tlbli	0			/* load the itlb */
	stb	%r1,6(%r2)		/* update page table */
	rfi

3:	/* not found in pteg */
	andi.	%r1,%r3,0x40		/* have we already done second hash? */
	bne	5f
	mfspr	%r2,HASH2		/* get the second pointer */
	ori	%r3,%r3,0x40		/* change the compare value */
	li	%r1,8
	addi	%r2,%r2,-8		/* predec pointer */
	b	1b
4:	/* guarded */
	mfsrr1	%r3
	andi.	%r2,%r3,0xffff		/* clean upper srr1 */
	addis	%r2,%r2,0x800		/* set srr<4> to flag prot violation */
	b	6f
5:	/* not found anywhere */
	mfsrr1	%r3
	andi.	%r2,%r3,0xffff		/* clean upper srr1 */
	addis	%r2,%r2,0x4000		/* set srr1<1> to flag pte not found */
6:
	mtctr	%r0			/* restore counter */
	mtsrr1	%r2
	mfmsr	%r0
	xoris	%r0,%r0,2		/* flip the msr<tgpr> bit */
	mtcrf	0x80,%r3		/* restore cr0 */
	mtmsr	%r0			/* now with native gprs */
	isync
	ba	EXC_ISI
tlbimsize =		.-tlbimiss

	.globl	tlbdlmiss,tlbdlmsize
	.type	tlbdlmiss,@function
	.type	tlbdlmsize,@object
tlbdlmiss:
	mfspr	%r2,HASH1		/* get first pointer */
	li	%r1,8
	mfctr	%r0			/* save counter */
	mfspr	%r3,DCMP		/* get first compare value */
	addi	%r2,%r2,-8		/* predec pointer */
1:
	mtctr	%r1			/* load counter */
2:
	lwzu	%r1,8(%r2)		/* get next pte */
	cmplw	0,%r1,%r3		/* see if found pte */
	bdneq	2b			/* loop if not eq */
	bne	3f			/* not found */
	lwz	%r1,4(%r2)		/* load tlb entry lower word */
	mtctr	%r0			/* restore counter */
	mfspr	%r0,DMISS		/* get the miss address for the tlbld */
	mfsrr1	%r3			/* get the saved cr0 bits */
	mtcrf	0x80,%r3		/* and restore */
	ori	%r1,%r1,0x100		/* set the reference bit */
	mtspr	RPA,%r1			/* set the pte */
	srwi	%r1,%r1,8		/* get byte 7 of pte */
	tlbld	0			/* load the dtlb */
	stb	%r1,6(%r2)		/* update page table */
	rfi

3:	/* not found in pteg */
	andi.	%r1,%r3,0x40		/* have we already done second hash? */
	bne	5f
	mfspr	%r2,HASH2		/* get the second pointer */
	ori	%r3,%r3,0x40		/* change the compare value */
	li	%r1,8
	addi	%r2,%r2,-8		/* predec pointer */
	b	1b
5:	/* not found anywhere */
	mfsrr1	%r3
	lis	%r1,0x4000		/* set dsisr<1> to flag pte not found */
	mtctr	%r0			/* restore counter */
	andi.	%r2,%r3,0xffff		/* clean upper srr1 */
	mtsrr1	%r2
	mtdsisr	%r1			/* load the dsisr */
	mfspr	%r1,DMISS		/* get the miss address */
	mtdar	%r1			/* put in dar */
	mfmsr	%r0
	xoris	%r0,%r0,2		/* flip the msr<tgpr> bit */
	mtcrf	0x80,%r3		/* restore cr0 */
	mtmsr	%r0			/* now with native gprs */
	isync
	ba	EXC_DSI
tlbdlmsize =		.-tlbdlmiss

	.globl	tlbdsmiss,tlbdsmsize
	.type	tlbdsmiss,@function
	.type	tlbdsmsize,@object
tlbdsmiss:
	mfspr	%r2,HASH1		/* get first pointer */
	li	%r1,8
	mfctr	%r0			/* save counter */
	mfspr	%r3,DCMP		/* get first compare value */
	addi	%r2,%r2,-8		/* predec pointer */
1:
	mtctr	%r1			/* load counter */
2:
	lwzu	%r1,8(%r2)		/* get next pte */
	cmplw	0,%r1,%r3		/* see if found pte */
	bdneq	2b			/* loop if not eq */
	bne	3f			/* not found */
	lwz	%r1,4(%r2)		/* load tlb entry lower word */
	andi.	%r3,%r1,0x80		/* check the C-bit */
	beq	4f
5:
	mtctr	%r0			/* restore counter */
	mfspr	%r0,DMISS		/* get the miss address for the tlbld */
	mfsrr1	%r3			/* get the saved cr0 bits */
	mtcrf	0x80,%r3		/* and restore */
	mtspr	RPA,%r1			/* set the pte */
	tlbld	0			/* load the dtlb */
	rfi

3:	/* not found in pteg */
	andi.	%r1,%r3,0x40		/* have we already done second hash? */
	bne	5f
	mfspr	%r2,HASH2		/* get the second pointer */
	ori	%r3,%r3,0x40		/* change the compare value */
	li	%r1,8
	addi	%r2,%r2,-8		/* predec pointer */
	b	1b
4:	/* found, but C-bit = 0 */
	rlwinm.	%r3,%r1,30,0,1		/* test PP */
	bge-	7f
	andi.	%r3,%r1,1
	beq+	8f
9:	/* found, but protection violation (PP==00)*/
	mfsrr1	%r3
	lis	%r1,0xa00	/* indicate protection violation on store */
	b	1f
7:	/* found, PP=1x */
	mfspr	%r3,DMISS		/* get the miss address */
	mfsrin	%r1,%r3			/* get the segment register */
	mfsrr1	%r3
	rlwinm	%r3,%r3,18,31,31	/* get PR-bit */
	rlwnm.	%r1,%r1,%r3,1,1		/* get the key */
	bne-	9b			/* protection violation */
8:	/* found, set reference/change bits */
	lwz	%r1,4(%r2)		/* reload tlb entry */
	ori	%r1,%r1,0x180
	sth	%r1,6(%r2)
	b	5b
5:	/* not found anywhere */
	mfsrr1	%r3
	lis	%r1,0x4200		/* set dsisr<1> to flag pte not found */
					/* dsisr<6> to flag store */
1:
	mtctr	%r0			/* restore counter */
	andi.	%r2,%r3,0xffff		/* clean upper srr1 */
	mtsrr1	%r2
	mtdsisr	%r1			/* load the dsisr */
	mfspr	%r1,DMISS		/* get the miss address */
	mtdar	%r1			/* put in dar */
	mfmsr	%r0
	xoris	%r0,%r0,2		/* flip the msr<tgpr> bit */
	mtcrf	0x80,%r3		/* restore cr0 */
	mtmsr	%r0			/* now with native gprs */
	isync
	ba	EXC_DSI
tlbdsmsize =		.-tlbdsmiss

#ifdef DDB
/*
 * In case of DDB we want a separate trap catcher for it
 */
	.globl	ddblow,ddbsize
ddblow:
	mtsprg	1,%r1			/* save SP */
nop32_7s:
	mfmsr	%r1
	clrldi	%r1,%r1,1
	mtmsrd	%r1
nop32_7e:
	GET_CPUINFO(%r1)
	stmw	%r28,CI_DDBSAVE(%r1)	/* free r28-r31 */
	mflr	%r28			/* save LR */
	mfcr	%r29			/* save CR */
	GET_CPUINFO(%r30)
	lwz	%r30,CI_INTSTK(%r30)	/* get interrupt stack */
	addi	%r1,%r30,(SPILLSTK+DDBSTK)
	bla	ddbtrap
ddbsize =		.-ddblow
#endif  /* DDB */

/*
 * If this interrupt occurred between the runqueue check and the setting
 * of the POW bit, do no enter to sleep.  We do that in order to process
 * the result of this interrupt directly and not when coming back from
 * sleep, when the next clock tick or interrupt will fire.
 */
#define CPU_IDLE_CHECK(sr1,sr2,sr3,rSRR0,flag)				\
	GET_CPUINFO(sr1);						\
	lwz	sr2,CI_FLAGS(sr1);					\
	andi.	sr3,sr2,flag@l;						\
	beq	1f;							\
	andi.	sr2,sr2,~flag@l;					\
	stw	sr2,CI_FLAGS(sr1);					\
	lis	rSRR0,idledone@ha;					\
	addi	rSRR0,rSRR0,idledone@l;					\
1:

/*
 * FRAME_SETUP assumes:
 *	SPRG1		SP (1)
 *	savearea	r28-r31,DAR,DSISR	(DAR & DSISR only for DSI traps)
 *	28		LR
 *	29		CR
 *	1		kernel stack
 *	LR		trap type
 *	SRR0/1		as at start of trap
 */
#define FRAME_SETUP(savearea)	FRAME_SETUP_FLAG(savearea, CI_FLAGS_SLEEPING)

#define	FRAME_SETUP_FLAG(savearea, flag)				\
/* Have to enable translation to allow access of kernel stack: */	\
	GET_CPUINFO(%r31);						\
	mfsrr0	%r30;							\
	stw	%r30,savearea+24(%r31);					\
	mfsrr1	%r30;							\
	stw	%r30,savearea+28(%r31);					\
	/* load all kernel segment registers. */			\
	lis	%r31,kernel_pmap_@ha;					\
	addi	%r31,%r31,kernel_pmap_@l;				\
	lwz	%r30,0(%r31);	mtsr	0,%r30;				\
	lwz	%r30,4(%r31);	mtsr	1,%r30;				\
	lwz	%r30,8(%r31);	mtsr	2,%r30;				\
	lwz	%r30,12(%r31);	mtsr	3,%r30;				\
	lwz	%r30,16(%r31);	mtsr	4,%r30;				\
	lwz	%r30,20(%r31);	mtsr	5,%r30;				\
	lwz	%r30,24(%r31);	mtsr	6,%r30;				\
	lwz	%r30,28(%r31);	mtsr	7,%r30;				\
	lwz	%r30,32(%r31);	mtsr	8,%r30;				\
	lwz	%r30,36(%r31);	mtsr	9,%r30;				\
	lwz	%r30,40(%r31);	mtsr	10,%r30;			\
	lwz	%r30,44(%r31);	mtsr	11,%r30;			\
	lwz	%r30,48(%r31);	mtsr	12,%r30;			\
/*	lwz	%r30,52(%r31);	mtsr 13,%r30; - dont load user SR - XXX? */ \
	lwz	%r30,56(%r31);	mtsr	14,%r30;			\
	lwz	%r30,60(%r31);	mtsr	15,%r30;			\
	mfmsr	%r30;							\
	ori	%r30,%r30,(PSL_DR|PSL_IR);				\
	mtmsr	%r30;							\
	isync;								\
	mfsprg	%r31,1;							\
	stwu	%r31,-FRAMELEN(%r1);					\
	stw	%r0,FRAME_0+8(%r1);					\
	stw	%r31,FRAME_1+8(%r1);					\
	stw	%r2,FRAME_2+8(%r1);					\
	stw	%r28,FRAME_LR+8(%r1);					\
	stw	%r29,FRAME_CR+8(%r1);					\
	GET_CPUINFO(%r2);						\
	lmw	%r28,savearea(%r2);					\
	stmw	%r3,FRAME_3+8(%r1);					\
	lmw	%r28,savearea+16(%r2);					\
	mfxer	%r3;							\
	mfctr	%r4;							\
	mflr	%r5;							\
	andi.	%r5,%r5,0xff00;						\
	stw	%r3,FRAME_XER+8(%r1);					\
	stw	%r4,FRAME_CTR+8(%r1);					\
	stw	%r5,FRAME_EXC+8(%r1);					\
	stw	%r28,FRAME_DAR+8(%r1);					\
	stw	%r29,FRAME_DSISR+8(%r1);				\
	CPU_IDLE_CHECK(%r5,%r6,%r0,%r30,flag)				\
	stw	%r30,FRAME_SRR0+8(%r1);					\
	stw	%r31,FRAME_SRR1+8(%r1)

#define	FRAME_LEAVE(savearea)						\
/* Now restore regs: */							\
	lwz	%r2,FRAME_SRR0+8(%r1);					\
	lwz	%r3,FRAME_SRR1+8(%r1);					\
	lwz	%r4,FRAME_CTR+8(%r1);					\
	lwz	%r5,FRAME_XER+8(%r1);					\
	lwz	%r6,FRAME_LR+8(%r1);					\
	GET_CPUINFO(%r7);						\
	stw	%r2,savearea(%r7);					\
	stw	%r3,savearea+4(%r7);					\
	lwz	%r7,FRAME_CR+8(%r1);					\
	mtctr	%r4;							\
	mtxer	%r5;							\
	mtlr	%r6;							\
	mtsprg	1,%r7;			/* save cr */			\
	lmw	%r2,FRAME_2+8(%r1);					\
	lwz	%r0,FRAME_0+8(%r1);					\
	lwz	%r1,FRAME_1+8(%r1);					\
	mtsprg	2,%r2;			/* save r2 & r3 */		\
	mtsprg	3,%r3;							\
/* Disable translation, machine check and recoverability: */		\
	mfmsr	%r2;							\
	lis	%r3,(PSL_DR|PSL_IR|PSL_ME|PSL_RI)@ha;			\
	addi	%r3,%r3,(PSL_DR|PSL_IR|PSL_ME|PSL_RI)@l;		\
	andc	%r2,%r2,%r3;						\
	mtmsr	%r2;							\
	isync;								\
/* Decide whether we return to user mode: */				\
	GET_CPUINFO(%r2);						\
	lwz	%r3,savearea+4(%r2);					\
	mtcr	%r3;							\
	bc	4,17,1f;		/* branch if PSL_PR is false */	\
/* Restore user & kernel access SR: */					\
	lwz	%r2,CI_CURPM(%r2);	/* get real address of pmap */	\
	lwz	%r3,0(%r2);	mtsr	0,%r3;				\
	lwz	%r3,4(%r2);	mtsr	1,%r3;				\
	lwz	%r3,8(%r2);	mtsr	2,%r3;				\
	lwz	%r3,12(%r2);	mtsr	3,%r3;				\
	lwz	%r3,16(%r2);	mtsr	4,%r3;				\
	lwz	%r3,20(%r2);	mtsr	5,%r3;				\
	lwz	%r3,24(%r2);	mtsr	6,%r3;				\
	lwz	%r3,28(%r2);	mtsr	7,%r3;				\
	lwz	%r3,32(%r2);	mtsr	8,%r3;				\
	lwz	%r3,36(%r2);	mtsr	9,%r3;				\
	lwz	%r3,40(%r2);	mtsr	10,%r3;				\
	lwz	%r3,44(%r2);	mtsr	11,%r3;				\
	lwz	%r3,48(%r2);	mtsr	12,%r3;				\
	lwz	%r3,52(%r2);	mtsr	13,%r3;				\
	lwz	%r3,56(%r2);	mtsr	14,%r3;				\
	lwz	%r3,60(%r2);	mtsr	15,%r3;				\
1:	mfsprg	%r2,1;			/* restore cr */		\
	mtcr	%r2;							\
	GET_CPUINFO(%r2);						\
	lwz	%r3,savearea(%r2);					\
	mtsrr0	%r3;							\
	lwz	%r3,savearea+4(%r2);					\
	mtsrr1	%r3;							\
	mfsprg	%r2,2;			/* restore r2 & r3 */		\
	mfsprg	%r3,3

/*
 * Preamble code for DSI/ISI traps
 */
disitrap:
	GET_CPUINFO(%r1)
	lmw	%r30,CI_DISISAVE(%r1)
	stmw	%r30,CI_TEMPSAVE(%r1)
	lmw	%r30,CI_DISISAVE+8(%r1)
	stmw	%r30,CI_TEMPSAVE+8(%r1)
	mfdar	%r30
	mfdsisr	%r31
	stmw	%r30,CI_TEMPSAVE+16(%r1)
realtrap:
	/* Test whether we already had PR set */
	mfsrr1	%r1
	mtcr	%r1
	/* restore SP (might have been overwritten) */
	mfsprg	%r1,1
	bc	4,17,s_trap		/* branch if PSL_PR is false */
	GET_CPUINFO(%r1)
	lwz	%r1,CI_CURPCB(%r1)
	addi	%r1,%r1,USPACE		/* stack is top of user struct */
/*
 * Now the common trap catching code.
 */
	.globl	s_trap
s_trap:
	FRAME_SETUP(CI_TEMPSAVE)
/* Now we can recover interrupts again: */
	mfmsr	%r7
	mfsrr1	%r31
	andi.	%r31,%r31,PSL_EE	/* restore EE from previous context */
	or	%r7,%r7,%r31
	ori	%r7,%r7,(PSL_ME|PSL_RI)
	mtmsr	%r7
	isync
/* Call C trap code: */
trapagain:
	addi	%r3,%r1,8
	bl	trap

	.globl	trapexit
trapexit:
/* Disable interrupts: */
	mfmsr	%r3
	andi.	%r3,%r3,~PSL_EE@l
	mtmsr	%r3
	isync
/* Test AST pending: */
	lwz	%r5,FRAME_SRR1+8(%r1)
	mtcr	%r5
	bc	4,17,1f			/* branch if PSL_PR is false */
	GET_CPUINFO(%r3)
	lwz	%r4,CI_CURPROC(%r3)
	lwz	%r4,P_MD_ASTPENDING(%r4)
	andi.	%r4,%r4,1
	beq	1f
	li	%r6,EXC_AST
	stw	%r6,FRAME_EXC+8(%r1)
	b	trapagain
1:
	FRAME_LEAVE(CI_TEMPSAVE)
rfi1:	rfi

/*
 * Child comes here at the end of a fork.
 * Mostly similar to the above.
 */
	.globl	proc_trampoline
	.type	proc_trampoline,@function
proc_trampoline:
	bl	proc_trampoline_mi
	mtlr	%r31
	mr	%r3,%r30
	blrl				/* jump indirect to r31 */
	b	trapexit

/*
 * DSI second stage fault handler
 */
s_dsitrap:
	mfdsisr	%r31			/* test if this is spill fault */
	mtcr	%r31
	mtsprg	1,%r1			/* save SP */
	bc	4,1,disitrap		/* branch if table miss is false */
	GET_CPUINFO(%r30)
	lwz	%r30,CI_INTSTK(%r30)	/* get interrupt stack */
	addi	%r1,%r30,SPILLSTK
	stwu	%r1,-52(%r1)
	stw	%r0,48(%r1)		/* save non-volatile registers */
	stw	%r3,44(%r1)
	stw	%r4,40(%r1)
	stw	%r5,36(%r1)
	stw	%r6,32(%r1)
	stw	%r7,28(%r1)
	stw	%r8,24(%r1)
	stw	%r9,20(%r1)
	stw	%r10,16(%r1)
	stw	%r11,12(%r1)
	stw	%r12,8(%r1)
	mfxer	%r30			/* save XER */
	mtsprg	2,%r30
	mflr	%r30			/* save trap type */
	mfctr	%r31			/* & CTR */
	mfdar	%r7
	mfsrr1	%r4
	mfdsisr	%r5
	li	%r6, 0
s_pte_spill:
	andi.	%r0,%r4,PSL_PR
	li	%r3,0
	bne	1f
	mr	%r3,%r7
	bl	pte_spill_r		/* try a spill */
1:
	cmpwi	0,%r3,0
	mtctr	%r31			/* restore CTR */
	mtlr	%r30			/* and trap type */
	mfsprg	%r31,2			/* get saved XER */
	mtxer	%r31			/* restore XER */
	lwz	%r12,8(%r1)		/* restore non-volatile registers */
	lwz	%r11,12(%r1)
	lwz	%r10,16(%r1)
	lwz	%r9,20(%r1)
	lwz	%r8,24(%r1)
	lwz	%r7,28(%r1)
	lwz	%r6,32(%r1)
	lwz	%r5,36(%r1)
	lwz	%r4,40(%r1)
	lwz	%r3,44(%r1)
	lwz	%r0,48(%r1)
	beq	disitrap
	mtcr	%r29			/* restore CR */
	mtlr	%r28			/* restore LR */
	GET_CPUINFO(%r1)
	lmw	%r28,CI_DISISAVE(%r1)	/* restore r28-r31 */
	mfsprg	%r1,1			/* restore SP */
rfi2:	rfi				/* return to trapped code */

/*
 * ISI second stage fault handler
 */
s_isitrap:
	mfsrr1	%r31			/* test if this may be a spill fault */
	mtcr	%r31
	mtsprg	1,%r1			/* save SP */
	bc	4,1,disitrap		/* branch if table miss is false */
	GET_CPUINFO(%r30)
	lwz	%r30,CI_INTSTK(%r30)	/* get interrupt stack */
	addi	%r1,%r30,SPILLSTK
	stwu	%r1,-52(%r1)
	stw	%r0,48(%r1)		/* save non-volatile registers */
	stw	%r3,44(%r1)
	stw	%r4,40(%r1)
	stw	%r5,36(%r1)
	stw	%r6,32(%r1)
	stw	%r7,28(%r1)
	stw	%r8,24(%r1)
	stw	%r9,20(%r1)
	stw	%r10,16(%r1)
	stw	%r11,12(%r1)
	stw	%r12,8(%r1)
	mfxer	%r30			/* save XER */
	mtsprg	2,%r30
	mflr	%r30			/* save trap type */
	mfctr	%r31			/* & ctr */
	mfsrr0	%r7
	mfsrr1	%r4
	li	%r5, 0
	li	%r6, 1
	b	s_pte_spill		/* above */

/*
 * External interrupt second level handler
 */
#define	INTRENTER							\
/* Save non-volatile registers: */					\
	stwu	%r1,-88(%r1);		/* temporarily */		\
	stw	%r0,84(%r1);						\
	mfsprg	%r0,1;			/* get original SP */		\
	stw	%r0,0(%r1);		/* and store it */		\
	stw	%r3,80(%r1);						\
	stw	%r4,76(%r1);						\
	stw	%r5,72(%r1);						\
	stw	%r6,68(%r1);						\
	stw	%r7,64(%r1);						\
	stw	%r8,60(%r1);						\
	stw	%r9,56(%r1);						\
	stw	%r10,52(%r1);						\
	stw	%r11,48(%r1);						\
	stw	%r12,44(%r1);						\
	stw	%r28,40(%r1);		/* saved LR */			\
	stw	%r29,36(%r1);		/* saved CR */			\
	stw	%r30,32(%r1);		/* saved XER */			\
	GET_CPUINFO(%r4);						\
	lmw	%r28,CI_TEMPSAVE(%r4);	/* restore r28-r31 */		\
	mfctr	%r6;							\
	lwz	%r5,CI_INTRDEPTH(%r4);			\
	mfsrr0	%r4;							\
	mfsrr1	%r3;							\
	stw	%r6,28(%r1);						\
	stw	%r5,20(%r1);						\
	stw	%r4,12(%r1);						\
	stw	%r3,8(%r1);						\
/* load all kernel segment registers. */				\
	lis	3,kernel_pmap_@ha;					\
	addi	3,3,kernel_pmap_@l;					\
	lwz	%r5,0(%r3);	mtsr 0,%r5;				\
	lwz	%r5,4(%r3);	mtsr 1,%r5;				\
	lwz	%r5,8(%r3);	mtsr 2,%r5;				\
	lwz	%r5,12(%r3);	mtsr 3,%r5;				\
	lwz	%r5,16(%r3);	mtsr 4,%r5;				\
	lwz	%r5,20(%r3);	mtsr 5,%r5;				\
	lwz	%r5,24(%r3);	mtsr 6,%r5;				\
	lwz	%r5,28(%r3);	mtsr 7,%r5;				\
	lwz	%r5,32(%r3);	mtsr 8,%r5;				\
	lwz	%r5,36(%r3);	mtsr 9,%r5;				\
	lwz	%r5,40(%r3);	mtsr 10,%r5;				\
	lwz	%r5,44(%r3);	mtsr 11,%r5;				\
	lwz	%r5,48(%r3);	mtsr 12,%r5;				\
/*	lwz	%r5,52(%r3);	mtsr 13,%r5;	- dont load user SR - XXX? */ \
	lwz	%r5,56(%r3);	mtsr 14,%r5;				\
	lwz	%r5,60(%r3);	mtsr 15,%r5;				\
/* interrupts are recoverable here, and enable translation */		\
	mfmsr	%r5;							\
	ori	%r5,%r5,(PSL_IR|PSL_DR|PSL_RI);				\
	mtmsr	%r5;							\
	isync

	.globl	extint_call
	.type	extint_call,@function
extintr:
	INTRENTER
extint_call:
	bl	extint_call			/* to be filled in later */
intr_exit:
/* Disable interrupts (should already be disabled) and MMU here: */
	mfmsr	%r3
	andi.	%r3,%r3,~(PSL_EE|PSL_ME|PSL_RI|PSL_DR|PSL_IR)@l
	mtmsr	%r3
	isync
/* restore possibly overwritten registers: */
	lwz	%r12,44(%r1)
	lwz	%r11,48(%r1)
	lwz	%r10,52(%r1)
	lwz	%r9,56(%r1)
	lwz	%r8,60(%r1)
	lwz	%r7,64(%r1)
	lwz	%r6,8(%r1)
	lwz	%r5,12(%r1)
	lwz	%r4,28(%r1)
	lwz	%r3,32(%r1)
	mtsrr1	%r6
	mtsrr0	%r5
	mtctr	%r4
	mtxer	%r3

	GET_CPUINFO(%r5)
	lwz	%r4,CI_INTRDEPTH(%r5)
	addi	%r4,%r4,-1		/* adjust reentrancy count */
	stw	%r4,CI_INTRDEPTH(%r5)

/* Returning to user mode? */
	mtcr	%r6			/* saved SRR1 */
	bc	4,17,1f			/* branch if PSL_PR is false */
	lwz	%r3,CI_CURPM(%r5)	/* get current pmap real address */
	/* reload all segment registers. */
	lwz	%r4,0(3);	mtsr 0,%r4;
	lwz	%r4,4(3);	mtsr 1,%r4;
	lwz	%r4,8(3);	mtsr 2,%r4;
	lwz	%r4,12(3);	mtsr 3,%r4;
	lwz	%r4,16(3);	mtsr 4,%r4;
	lwz	%r4,20(3);	mtsr 5,%r4;
	lwz	%r4,24(3);	mtsr 6,%r4;
	lwz	%r4,28(3);	mtsr 7,%r4;
	lwz	%r4,32(3);	mtsr 8,%r4;
	lwz	%r4,36(3);	mtsr 9,%r4;
	lwz	%r4,40(3);	mtsr 10,%r4;
	lwz	%r4,44(3);	mtsr 11,%r4;
	lwz	%r4,48(3);	mtsr 12,%r4;
	lwz	%r4,52(3);	mtsr 13,%r4;
	lwz	%r4,56(3);	mtsr 14,%r4;
	lwz	%r4,60(3);	mtsr 15,%r4;
	lwz	%r4,CI_CURPROC(%r5)
	lwz	%r4,P_MD_ASTPENDING(%r4)	/* Test AST pending */
	andi.	%r4,%r4,1
	beq	1f
/* Setup for entry to realtrap: */
	lwz	%r3,0(%r1)		/* get saved SP */
	mtsprg	1,%r3
	li	%r6,EXC_AST
	stmw	%r28,CI_TEMPSAVE(%r5)	/* establish tempsave again */
	mtlr	%r6
	lwz	%r28,40(%r1)		/* saved LR */
	lwz	%r29,36(%r1)		/* saved CR */
	lwz	%r6,68(%r1)
	lwz	%r5,72(%r1)
	lwz	%r4,76(%r1)
	lwz	%r3,80(%r1)
	lwz	%r0,84(%r1)
	b	realtrap
1:
/* Here is the normal exit of extintr: */
	lwz	%r5,36(%r1)
	lwz	%r6,40(%r1)
	mtcr	%r5
	mtlr	%r6
	lwz	%r6,68(%r1)
	lwz	%r5,72(%r1)
	lwz	%r4,76(%r1)
	lwz	%r3,80(%r1)
	lwz	%r0,84(%r1)
	lwz	%r1,0(%r1)
rfi3:	rfi

/*
 * Decrementer interrupt second level handler
 */
	.globl	decrintr
decrintr:
	INTRENTER
	addi	%r3,%r1,8			/* intr frame */
	bl	decr_intr
	b	intr_exit


/*
 * int setfault()
 *
 * Similar to setjmp to setup for handling faults on accesses to user memory.
 * Any routine using this may only call bcopy, either the form below,
 * or the (currently used) C code optimized, so it doesn't use any non-volatile
 * registers.
 */
	.globl	setfault
	.type	setfault,@function
setfault:
	mflr	%r0
	RETGUARD_SETUP_LATE(setfault, %r11, %r0)
	mfcr	%r12
	GET_CPUINFO(%r4)
	lwz	%r4,CI_CURPCB(%r4)
	stw	%r3,PCB_FAULT(%r4)
	stw	%r0,0(%r3)
	stw	%r1,4(%r3)
	stmw	%r12,8(%r3)
	li	%r3,0
	RETGUARD_CHECK(setfault, %r11, %r0)
	blr

/*
 * The following code gets copied to the top of the user stack on process
 * execution.  It does signal trampolining on signal delivery.
 *
 * On entry r1 points to a struct sigframe at bottom of current stack.
 * All other registers are unchanged.
 */
	.section .rodata
	.globl	sigcode,esigcode
	.type	sigcode,@function
	.type	esigcode,@function
sigcode:
	addi	%r1,%r1,-((16+FPSIG_SIZEOF+15)& ~0xf)		/* reserved space for callee */
	addi	%r6,%r1,8
	stfd	%f0,0(%r6)
	stfd	%f1,8(%r6)
	stfd	%f2,16(%r6)
	stfd	%f3,24(%r6)
	stfd	%f4,32(%r6)
	stfd	%f5,40(%r6)
	stfd	%f6,48(%r6)
	stfd	%f7,56(%r6)
	stfd	%f8,64(%r6)
	stfd	%f9,72(%r6)
	stfd	%f10,80(%r6)
	stfd	%f11,88(%r6)
	stfd	%f12,96(%r6)
	stfd	%f13,104(%r6)
	mffs	%f0
	stfd	%f0,112(%r6)
	lfd	%f0,0(%r6)	/* restore the clobbered register */
	blrl
	addi	%r6,%r1,8
	lfd	%f0,112(%r6)
	mtfsf	0xff,%f0
	lfd	%f0,0(%r6)
	lfd	%f1,8(%r6)
	lfd	%f2,16(%r6)
	lfd	%f3,24(%r6)
	lfd	%f4,32(%r6)
	lfd	%f5,40(%r6)
	lfd	%f6,48(%r6)
	lfd	%f7,56(%r6)
	lfd	%f8,64(%r6)
	lfd	%f9,72(%r6)
	lfd	%f10,80(%r6)
	lfd	%f11,88(%r6)
	lfd	%f12,96(%r6)
	lfd	%f13,104(%r6)
	addi	%r3,%r1,((16+FPSIG_SIZEOF+15)&~0xf)+SF_SC	/* compute &sf_sc */
	li	%r0,SYS_sigreturn
	.globl	sigcodecall
sigcodecall:
	sc				/* sigreturn(scp) */
	.globl	sigcoderet
sigcoderet:
esigcode:
	/* FALLTHROUGH */
	.globl	sigfill
sigfill:
	.long	0		# illegal
esigfill:
	.align	4
	.globl	sigfillsiz
sigfillsiz:
	.long	esigfill - sigfill

	.text

#ifdef DDB
/*
 * Deliberate entry to ddbtrap
 */
	.globl	ddb_trap
ddb_trap:
	mtsprg	1,%r1
	mfmsr	%r3
	mtsrr1	%r3
	andi.	%r3,%r3,~(PSL_EE|PSL_ME)@l
	mtmsr	%r3				/* disable interrupts */
	isync
	GET_CPUINFO(%r3)
	stmw	%r28,CI_DDBSAVE(%r3)

	mflr	%r28
	li	%r29,EXC_BPT
	mtlr	%r29
	mfcr	%r29
	mtsrr0	%r28

/*
 * Now the ddb trap catching code.
 */
ddbtrap:
	/*
	 * Do not let FRAME_SETUP() change the return address of, and
	 * corrupt, this frame.
	 */
	FRAME_SETUP_FLAG(CI_DDBSAVE, 0)
/* Call C trap code: */
	addi	%r3,%r1,8
	bl	db_trap_glue
	or.	%r3,%r3,%r3
	bne	ddbleave
/* This wasn't for DDB, so switch to real trap: */
	lwz	%r3,FRAME_EXC+8(%r1)	/* save exception */
	GET_CPUINFO(%r4)
	stw	%r3,CI_DDBSAVE+8(%r4)
	FRAME_LEAVE(CI_DDBSAVE)
	mtsprg	1,%r1			/* prepare for entrance to realtrap */
	GET_CPUINFO(%r1)
	stmw	%r28,CI_TEMPSAVE(%r1)
	mflr	%r28
	mfcr	%r29
	lwz	%r31,CI_DDBSAVE+8(%r1)
	mtlr	%r31
	b	realtrap
ddbleave:
	FRAME_LEAVE(CI_DDBSAVE)
rfi4:	rfi
#endif /* DDB */

	.globl	rfi_inst
rfi_inst:
	rfi
	.globl	rfid_inst
rfid_inst:
	rfid
	.globl	nop_inst
 nop_inst:
	nop

	.globl	rfi_start
rfi_start:
	.long	rfi1, rfi1 + 4
	.long	rfi2, rfi2 + 4
	.long	rfi3, rfi3 + 4
#ifdef DDB
	.long	rfi4, rfi4 + 4
#endif
	.long	0, 0


	.globl	nopbat_start
nopbat_start:
	.long	nopbat_1s, nopbat_1e
	.long	0, 0

	.globl	nop32_start
nop32_start:
	.long	nop32_1s, nop32_1e
	.long	nop32_2s, nop32_2e
	.long	nop32_3s, nop32_3e
	.long	nop32_4s, nop32_4e
	.long	nop32_5s, nop32_5e
	.long	nop32_6s, nop32_6e
#ifdef DDB
	.long	nop32_7s, nop32_7e
#endif
	.long	0, 0