1// SPDX-License-Identifier: GPL-2.0-only
2// Copyright 2023 Google LLC
3// Authors: Ard Biesheuvel <ardb@google.com>
4//          Peter Collingbourne <pcc@google.com>
5
6#include <linux/elf.h>
7#include <linux/init.h>
8#include <linux/types.h>
9
10#include "pi.h"
11
12extern const Elf64_Rela rela_start[], rela_end[];
13extern const u64 relr_start[], relr_end[];
14
15void __init relocate_kernel(u64 offset)
16{
17	u64 *place = NULL;
18
19	for (const Elf64_Rela *rela = rela_start; rela < rela_end; rela++) {
20		if (ELF64_R_TYPE(rela->r_info) != R_AARCH64_RELATIVE)
21			continue;
22		*(u64 *)(rela->r_offset + offset) = rela->r_addend + offset;
23	}
24
25	if (!IS_ENABLED(CONFIG_RELR) || !offset)
26		return;
27
28	/*
29	 * Apply RELR relocations.
30	 *
31	 * RELR is a compressed format for storing relative relocations. The
32	 * encoded sequence of entries looks like:
33	 * [ AAAAAAAA BBBBBBB1 BBBBBBB1 ... AAAAAAAA BBBBBB1 ... ]
34	 *
35	 * i.e. start with an address, followed by any number of bitmaps. The
36	 * address entry encodes 1 relocation. The subsequent bitmap entries
37	 * encode up to 63 relocations each, at subsequent offsets following
38	 * the last address entry.
39	 *
40	 * The bitmap entries must have 1 in the least significant bit. The
41	 * assumption here is that an address cannot have 1 in lsb. Odd
42	 * addresses are not supported. Any odd addresses are stored in the
43	 * RELA section, which is handled above.
44	 *
45	 * With the exception of the least significant bit, each bit in the
46	 * bitmap corresponds with a machine word that follows the base address
47	 * word, and the bit value indicates whether or not a relocation needs
48	 * to be applied to it. The second least significant bit represents the
49	 * machine word immediately following the initial address, and each bit
50	 * that follows represents the next word, in linear order. As such, a
51	 * single bitmap can encode up to 63 relocations in a 64-bit object.
52	 */
53	for (const u64 *relr = relr_start; relr < relr_end; relr++) {
54		if ((*relr & 1) == 0) {
55			place = (u64 *)(*relr + offset);
56			*place++ += offset;
57		} else {
58			for (u64 *p = place, r = *relr >> 1; r; p++, r >>= 1)
59				if (r & 1)
60					*p += offset;
61			place += 63;
62		}
63	}
64}
65