1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2022 - Google LLC
4 * Author: Ard Biesheuvel <ardb@google.com>
5 */
6
7#include <linux/errno.h>
8#include <linux/init.h>
9#include <linux/linkage.h>
10#include <linux/types.h>
11
12#include <asm/scs.h>
13
14#include "pi.h"
15
16bool dynamic_scs_is_enabled;
17
18//
19// This minimal DWARF CFI parser is partially based on the code in
20// arch/arc/kernel/unwind.c, and on the document below:
21// https://refspecs.linuxbase.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html
22//
23
24#define DW_CFA_nop                          0x00
25#define DW_CFA_set_loc                      0x01
26#define DW_CFA_advance_loc1                 0x02
27#define DW_CFA_advance_loc2                 0x03
28#define DW_CFA_advance_loc4                 0x04
29#define DW_CFA_offset_extended              0x05
30#define DW_CFA_restore_extended             0x06
31#define DW_CFA_undefined                    0x07
32#define DW_CFA_same_value                   0x08
33#define DW_CFA_register                     0x09
34#define DW_CFA_remember_state               0x0a
35#define DW_CFA_restore_state                0x0b
36#define DW_CFA_def_cfa                      0x0c
37#define DW_CFA_def_cfa_register             0x0d
38#define DW_CFA_def_cfa_offset               0x0e
39#define DW_CFA_def_cfa_expression           0x0f
40#define DW_CFA_expression                   0x10
41#define DW_CFA_offset_extended_sf           0x11
42#define DW_CFA_def_cfa_sf                   0x12
43#define DW_CFA_def_cfa_offset_sf            0x13
44#define DW_CFA_val_offset                   0x14
45#define DW_CFA_val_offset_sf                0x15
46#define DW_CFA_val_expression               0x16
47#define DW_CFA_lo_user                      0x1c
48#define DW_CFA_negate_ra_state              0x2d
49#define DW_CFA_GNU_args_size                0x2e
50#define DW_CFA_GNU_negative_offset_extended 0x2f
51#define DW_CFA_hi_user                      0x3f
52
53enum {
54	PACIASP		= 0xd503233f,
55	AUTIASP		= 0xd50323bf,
56	SCS_PUSH	= 0xf800865e,
57	SCS_POP		= 0xf85f8e5e,
58};
59
60static void __always_inline scs_patch_loc(u64 loc)
61{
62	u32 insn = le32_to_cpup((void *)loc);
63
64	switch (insn) {
65	case PACIASP:
66		*(u32 *)loc = cpu_to_le32(SCS_PUSH);
67		break;
68	case AUTIASP:
69		*(u32 *)loc = cpu_to_le32(SCS_POP);
70		break;
71	default:
72		/*
73		 * While the DW_CFA_negate_ra_state directive is guaranteed to
74		 * appear right after a PACIASP/AUTIASP instruction, it may
75		 * also appear after a DW_CFA_restore_state directive that
76		 * restores a state that is only partially accurate, and is
77		 * followed by DW_CFA_negate_ra_state directive to toggle the
78		 * PAC bit again. So we permit other instructions here, and ignore
79		 * them.
80		 */
81		return;
82	}
83	if (IS_ENABLED(CONFIG_ARM64_WORKAROUND_CLEAN_CACHE))
84		asm("dc civac, %0" :: "r"(loc));
85	else
86		asm(ALTERNATIVE("dc cvau, %0", "nop", ARM64_HAS_CACHE_IDC)
87		    :: "r"(loc));
88}
89
90/*
91 * Skip one uleb128/sleb128 encoded quantity from the opcode stream. All bytes
92 * except the last one have bit #7 set.
93 */
94static int __always_inline skip_xleb128(const u8 **opcode, int size)
95{
96	u8 c;
97
98	do {
99		c = *(*opcode)++;
100		size--;
101	} while (c & BIT(7));
102
103	return size;
104}
105
106struct eh_frame {
107	/*
108	 * The size of this frame if 0 < size < U32_MAX, 0 terminates the list.
109	 */
110	u32	size;
111
112	/*
113	 * The first frame is a Common Information Entry (CIE) frame, followed
114	 * by one or more Frame Description Entry (FDE) frames. In the former
115	 * case, this field is 0, otherwise it is the negated offset relative
116	 * to the associated CIE frame.
117	 */
118	u32	cie_id_or_pointer;
119
120	union {
121		struct { // CIE
122			u8	version;
123			u8	augmentation_string[];
124		};
125
126		struct { // FDE
127			s32	initial_loc;
128			s32	range;
129			u8	opcodes[];
130		};
131	};
132};
133
134static int scs_handle_fde_frame(const struct eh_frame *frame,
135				bool fde_has_augmentation_data,
136				int code_alignment_factor,
137				bool dry_run)
138{
139	int size = frame->size - offsetof(struct eh_frame, opcodes) + 4;
140	u64 loc = (u64)offset_to_ptr(&frame->initial_loc);
141	const u8 *opcode = frame->opcodes;
142
143	if (fde_has_augmentation_data) {
144		int l;
145
146		// assume single byte uleb128_t
147		if (WARN_ON(*opcode & BIT(7)))
148			return -ENOEXEC;
149
150		l = *opcode++;
151		opcode += l;
152		size -= l + 1;
153	}
154
155	/*
156	 * Starting from 'loc', apply the CFA opcodes that advance the location
157	 * pointer, and identify the locations of the PAC instructions.
158	 */
159	while (size-- > 0) {
160		switch (*opcode++) {
161		case DW_CFA_nop:
162		case DW_CFA_remember_state:
163		case DW_CFA_restore_state:
164			break;
165
166		case DW_CFA_advance_loc1:
167			loc += *opcode++ * code_alignment_factor;
168			size--;
169			break;
170
171		case DW_CFA_advance_loc2:
172			loc += *opcode++ * code_alignment_factor;
173			loc += (*opcode++ << 8) * code_alignment_factor;
174			size -= 2;
175			break;
176
177		case DW_CFA_def_cfa:
178		case DW_CFA_offset_extended:
179			size = skip_xleb128(&opcode, size);
180			fallthrough;
181		case DW_CFA_def_cfa_offset:
182		case DW_CFA_def_cfa_offset_sf:
183		case DW_CFA_def_cfa_register:
184		case DW_CFA_same_value:
185		case DW_CFA_restore_extended:
186		case 0x80 ... 0xbf:
187			size = skip_xleb128(&opcode, size);
188			break;
189
190		case DW_CFA_negate_ra_state:
191			if (!dry_run)
192				scs_patch_loc(loc - 4);
193			break;
194
195		case 0x40 ... 0x7f:
196			// advance loc
197			loc += (opcode[-1] & 0x3f) * code_alignment_factor;
198			break;
199
200		case 0xc0 ... 0xff:
201			break;
202
203		default:
204			return -ENOEXEC;
205		}
206	}
207	return 0;
208}
209
210int scs_patch(const u8 eh_frame[], int size)
211{
212	const u8 *p = eh_frame;
213
214	while (size > 4) {
215		const struct eh_frame *frame = (const void *)p;
216		bool fde_has_augmentation_data = true;
217		int code_alignment_factor = 1;
218		int ret;
219
220		if (frame->size == 0 ||
221		    frame->size == U32_MAX ||
222		    frame->size > size)
223			break;
224
225		if (frame->cie_id_or_pointer == 0) {
226			const u8 *p = frame->augmentation_string;
227
228			/* a 'z' in the augmentation string must come first */
229			fde_has_augmentation_data = *p == 'z';
230
231			/*
232			 * The code alignment factor is a uleb128 encoded field
233			 * but given that the only sensible values are 1 or 4,
234			 * there is no point in decoding the whole thing.
235			 */
236			p += strlen(p) + 1;
237			if (!WARN_ON(*p & BIT(7)))
238				code_alignment_factor = *p;
239		} else {
240			ret = scs_handle_fde_frame(frame,
241						   fde_has_augmentation_data,
242						   code_alignment_factor,
243						   true);
244			if (ret)
245				return ret;
246			scs_handle_fde_frame(frame, fde_has_augmentation_data,
247					     code_alignment_factor, false);
248		}
249
250		p += sizeof(frame->size) + frame->size;
251		size -= sizeof(frame->size) + frame->size;
252	}
253	return 0;
254}
255