summaryrefslogtreecommitdiffstats
path: root/kernel/arch/arm64/kvm/vgic-v3-switch.S
blob: 617a012a01078393e806be0daaf068af24c230cd (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
/*
 * Copyright (C) 2012,2013 - ARM Ltd
 * Author: Marc Zyngier <marc.zyngier@arm.com>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <linux/linkage.h>
#include <linux/irqchip/arm-gic-v3.h>

#include <asm/assembler.h>
#include <asm/memory.h>
#include <asm/asm-offsets.h>
#include <asm/kvm.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_arm.h>

	.text
	.pushsection	.hyp.text, "ax"

/*
 * We store LRs in reverse order to let the CPU deal with streaming
 * access. Use this macro to make it look saner...
 */
#define LR_OFFSET(n)	(VGIC_V3_CPU_LR + (15 - n) * 8)

/*
 * Save the VGIC CPU state into memory
 * x0: Register pointing to VCPU struct
 * Do not corrupt x1!!!
 */
.macro	save_vgic_v3_state
	// Compute the address of struct vgic_cpu
	add	x3, x0, #VCPU_VGIC_CPU

	// Make sure stores to the GIC via the memory mapped interface
	// are now visible to the system register interface
	dsb	st

	// Save all interesting registers
	mrs_s	x4, ICH_HCR_EL2
	mrs_s	x5, ICH_VMCR_EL2
	mrs_s	x6, ICH_MISR_EL2
	mrs_s	x7, ICH_EISR_EL2
	mrs_s	x8, ICH_ELSR_EL2

	str	w4, [x3, #VGIC_V3_CPU_HCR]
	str	w5, [x3, #VGIC_V3_CPU_VMCR]
	str	w6, [x3, #VGIC_V3_CPU_MISR]
	str	w7, [x3, #VGIC_V3_CPU_EISR]
	str	w8, [x3, #VGIC_V3_CPU_ELRSR]

	msr_s	ICH_HCR_EL2, xzr

	mrs_s	x21, ICH_VTR_EL2
	mvn	w22, w21
	ubfiz	w23, w22, 2, 4	// w23 = (15 - ListRegs) * 4

	adr	x24, 1f
	add	x24, x24, x23
	br	x24

1:
	mrs_s	x20, ICH_LR15_EL2
	mrs_s	x19, ICH_LR14_EL2
	mrs_s	x18, ICH_LR13_EL2
	mrs_s	x17, ICH_LR12_EL2
	mrs_s	x16, ICH_LR11_EL2
	mrs_s	x15, ICH_LR10_EL2
	mrs_s	x14, ICH_LR9_EL2
	mrs_s	x13, ICH_LR8_EL2
	mrs_s	x12, ICH_LR7_EL2
	mrs_s	x11, ICH_LR6_EL2
	mrs_s	x10, ICH_LR5_EL2
	mrs_s	x9, ICH_LR4_EL2
	mrs_s	x8, ICH_LR3_EL2
	mrs_s	x7, ICH_LR2_EL2
	mrs_s	x6, ICH_LR1_EL2
	mrs_s	x5, ICH_LR0_EL2

	adr	x24, 1f
	add	x24, x24, x23
	br	x24

1:
	str	x20, [x3, #LR_OFFSET(15)]
	str	x19, [x3, #LR_OFFSET(14)]
	str	x18, [x3, #LR_OFFSET(13)]
	str	x17, [x3, #LR_OFFSET(12)]
	str	x16, [x3, #LR_OFFSET(11)]
	str	x15, [x3, #LR_OFFSET(10)]
	str	x14, [x3, #LR_OFFSET(9)]
	str	x13, [x3, #LR_OFFSET(8)]
	str	x12, [x3, #LR_OFFSET(7)]
	str	x11, [x3, #LR_OFFSET(6)]
	str	x10, [x3, #LR_OFFSET(5)]
	str	x9, [x3, #LR_OFFSET(4)]
	str	x8, [x3, #LR_OFFSET(3)]
	str	x7, [x3, #LR_OFFSET(2)]
	str	x6, [x3, #LR_OFFSET(1)]
	str	x5, [x3, #LR_OFFSET(0)]

	tbnz	w21, #29, 6f	// 6 bits
	tbz	w21, #30, 5f	// 5 bits
				// 7 bits
	mrs_s	x20, ICH_AP0R3_EL2
	str	w20, [x3, #(VGIC_V3_CPU_AP0R + 3*4)]
	mrs_s	x19, ICH_AP0R2_EL2
	str	w19, [x3, #(VGIC_V3_CPU_AP0R + 2*4)]
6:	mrs_s	x18, ICH_AP0R1_EL2
	str	w18, [x3, #(VGIC_V3_CPU_AP0R + 1*4)]
5:	mrs_s	x17, ICH_AP0R0_EL2
	str	w17, [x3, #VGIC_V3_CPU_AP0R]

	tbnz	w21, #29, 6f	// 6 bits
	tbz	w21, #30, 5f	// 5 bits
				// 7 bits
	mrs_s	x20, ICH_AP1R3_EL2
	str	w20, [x3, #(VGIC_V3_CPU_AP1R + 3*4)]
	mrs_s	x19, ICH_AP1R2_EL2
	str	w19, [x3, #(VGIC_V3_CPU_AP1R + 2*4)]
6:	mrs_s	x18, ICH_AP1R1_EL2
	str	w18, [x3, #(VGIC_V3_CPU_AP1R + 1*4)]
5:	mrs_s	x17, ICH_AP1R0_EL2
	str	w17, [x3, #VGIC_V3_CPU_AP1R]

	// Restore SRE_EL1 access and re-enable SRE at EL1.
	mrs_s	x5, ICC_SRE_EL2
	orr	x5, x5, #ICC_SRE_EL2_ENABLE
	msr_s	ICC_SRE_EL2, x5
	isb
	mov	x5, #1
	msr_s	ICC_SRE_EL1, x5
.endm

/*
 * Restore the VGIC CPU state from memory
 * x0: Register pointing to VCPU struct
 */
.macro	restore_vgic_v3_state
	// Compute the address of struct vgic_cpu
	add	x3, x0, #VCPU_VGIC_CPU

	// Restore all interesting registers
	ldr	w4, [x3, #VGIC_V3_CPU_HCR]
	ldr	w5, [x3, #VGIC_V3_CPU_VMCR]
	ldr	w25, [x3, #VGIC_V3_CPU_SRE]

	msr_s	ICC_SRE_EL1, x25

	// make sure SRE is valid before writing the other registers
	isb

	msr_s	ICH_HCR_EL2, x4
	msr_s	ICH_VMCR_EL2, x5

	mrs_s	x21, ICH_VTR_EL2

	tbnz	w21, #29, 6f	// 6 bits
	tbz	w21, #30, 5f	// 5 bits
				// 7 bits
	ldr	w20, [x3, #(VGIC_V3_CPU_AP1R + 3*4)]
	msr_s	ICH_AP1R3_EL2, x20
	ldr	w19, [x3, #(VGIC_V3_CPU_AP1R + 2*4)]
	msr_s	ICH_AP1R2_EL2, x19
6:	ldr	w18, [x3, #(VGIC_V3_CPU_AP1R + 1*4)]
	msr_s	ICH_AP1R1_EL2, x18
5:	ldr	w17, [x3, #VGIC_V3_CPU_AP1R]
	msr_s	ICH_AP1R0_EL2, x17

	tbnz	w21, #29, 6f	// 6 bits
	tbz	w21, #30, 5f	// 5 bits
				// 7 bits
	ldr	w20, [x3, #(VGIC_V3_CPU_AP0R + 3*4)]
	msr_s	ICH_AP0R3_EL2, x20
	ldr	w19, [x3, #(VGIC_V3_CPU_AP0R + 2*4)]
	msr_s	ICH_AP0R2_EL2, x19
6:	ldr	w18, [x3, #(VGIC_V3_CPU_AP0R + 1*4)]
	msr_s	ICH_AP0R1_EL2, x18
5:	ldr	w17, [x3, #VGIC_V3_CPU_AP0R]
	msr_s	ICH_AP0R0_EL2, x17

	and	w22, w21, #0xf
	mvn	w22, w21
	ubfiz	w23, w22, 2, 4	// w23 = (15 - ListRegs) * 4

	adr	x24, 1f
	add	x24, x24, x23
	br	x24

1:
	ldr	x20, [x3, #LR_OFFSET(15)]
	ldr	x19, [x3, #LR_OFFSET(14)]
	ldr	x18, [x3, #LR_OFFSET(13)]
	ldr	x17, [x3, #LR_OFFSET(12)]
	ldr	x16, [x3, #LR_OFFSET(11)]
	ldr	x15, [x3, #LR_OFFSET(10)]
	ldr	x14, [x3, #LR_OFFSET(9)]
	ldr	x13, [x3, #LR_OFFSET(8)]
	ldr	x12, [x3, #LR_OFFSET(7)]
	ldr	x11, [x3, #LR_OFFSET(6)]
	ldr	x10, [x3, #LR_OFFSET(5)]
	ldr	x9, [x3, #LR_OFFSET(4)]
	ldr	x8, [x3, #LR_OFFSET(3)]
	ldr	x7, [x3, #LR_OFFSET(2)]
	ldr	x6, [x3, #LR_OFFSET(1)]
	ldr	x5, [x3, #LR_OFFSET(0)]

	adr	x24, 1f
	add	x24, x24, x23
	br	x24

1:
	msr_s	ICH_LR15_EL2, x20
	msr_s	ICH_LR14_EL2, x19
	msr_s	ICH_LR13_EL2, x18
	msr_s	ICH_LR12_EL2, x17
	msr_s	ICH_LR11_EL2, x16
	msr_s	ICH_LR10_EL2, x15
	msr_s	ICH_LR9_EL2,  x14
	msr_s	ICH_LR8_EL2,  x13
	msr_s	ICH_LR7_EL2,  x12
	msr_s	ICH_LR6_EL2,  x11
	msr_s	ICH_LR5_EL2,  x10
	msr_s	ICH_LR4_EL2,   x9
	msr_s	ICH_LR3_EL2,   x8
	msr_s	ICH_LR2_EL2,   x7
	msr_s	ICH_LR1_EL2,   x6
	msr_s	ICH_LR0_EL2,   x5

	// Ensure that the above will have reached the
	// (re)distributors. This ensure the guest will read
	// the correct values from the memory-mapped interface.
	isb
	dsb	sy

	// Prevent the guest from touching the GIC system registers
	// if SRE isn't enabled for GICv3 emulation
	cbnz	x25, 1f
	mrs_s	x5, ICC_SRE_EL2
	and	x5, x5, #~ICC_SRE_EL2_ENABLE
	msr_s	ICC_SRE_EL2, x5
1:
.endm

ENTRY(__save_vgic_v3_state)
	save_vgic_v3_state
	ret
ENDPROC(__save_vgic_v3_state)

ENTRY(__restore_vgic_v3_state)
	restore_vgic_v3_state
	ret
ENDPROC(__restore_vgic_v3_state)

ENTRY(__vgic_v3_get_ich_vtr_el2)
	mrs_s	x0, ICH_VTR_EL2
	ret
ENDPROC(__vgic_v3_get_ich_vtr_el2)

	.popsection