1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2019, Intel Corporation.
4 *
5 * Heterogeneous Memory Attributes Table (HMAT) representation
6 *
7 * This program parses and reports the platform's HMAT tables, and registers
8 * the applicable attributes with the node's interfaces.
9 */
10
11#define pr_fmt(fmt) "acpi/hmat: " fmt
12
13#include <linux/acpi.h>
14#include <linux/bitops.h>
15#include <linux/device.h>
16#include <linux/init.h>
17#include <linux/list.h>
18#include <linux/mm.h>
19#include <linux/platform_device.h>
20#include <linux/list_sort.h>
21#include <linux/memregion.h>
22#include <linux/memory.h>
23#include <linux/mutex.h>
24#include <linux/node.h>
25#include <linux/sysfs.h>
26#include <linux/dax.h>
27#include <linux/memory-tiers.h>
28
29static u8 hmat_revision;
30static int hmat_disable __initdata;
31
32void __init disable_hmat(void)
33{
34	hmat_disable = 1;
35}
36
37static LIST_HEAD(targets);
38static LIST_HEAD(initiators);
39static LIST_HEAD(localities);
40
41static DEFINE_MUTEX(target_lock);
42
43/*
44 * The defined enum order is used to prioritize attributes to break ties when
45 * selecting the best performing node.
46 */
47enum locality_types {
48	WRITE_LATENCY,
49	READ_LATENCY,
50	WRITE_BANDWIDTH,
51	READ_BANDWIDTH,
52};
53
54static struct memory_locality *localities_types[4];
55
56struct target_cache {
57	struct list_head node;
58	struct node_cache_attrs cache_attrs;
59};
60
61enum {
62	NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL = ACCESS_COORDINATE_MAX,
63	NODE_ACCESS_CLASS_GENPORT_SINK_CPU,
64	NODE_ACCESS_CLASS_MAX,
65};
66
67struct memory_target {
68	struct list_head node;
69	unsigned int memory_pxm;
70	unsigned int processor_pxm;
71	struct resource memregions;
72	struct access_coordinate coord[NODE_ACCESS_CLASS_MAX];
73	struct list_head caches;
74	struct node_cache_attrs cache_attrs;
75	u8 gen_port_device_handle[ACPI_SRAT_DEVICE_HANDLE_SIZE];
76	bool registered;
77	bool ext_updated;	/* externally updated */
78};
79
80struct memory_initiator {
81	struct list_head node;
82	unsigned int processor_pxm;
83	bool has_cpu;
84};
85
86struct memory_locality {
87	struct list_head node;
88	struct acpi_hmat_locality *hmat_loc;
89};
90
91static struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm)
92{
93	struct memory_initiator *initiator;
94
95	list_for_each_entry(initiator, &initiators, node)
96		if (initiator->processor_pxm == cpu_pxm)
97			return initiator;
98	return NULL;
99}
100
101static struct memory_target *find_mem_target(unsigned int mem_pxm)
102{
103	struct memory_target *target;
104
105	list_for_each_entry(target, &targets, node)
106		if (target->memory_pxm == mem_pxm)
107			return target;
108	return NULL;
109}
110
111static struct memory_target *acpi_find_genport_target(u32 uid)
112{
113	struct memory_target *target;
114	u32 target_uid;
115	u8 *uid_ptr;
116
117	list_for_each_entry(target, &targets, node) {
118		uid_ptr = target->gen_port_device_handle + 8;
119		target_uid = *(u32 *)uid_ptr;
120		if (uid == target_uid)
121			return target;
122	}
123
124	return NULL;
125}
126
127/**
128 * acpi_get_genport_coordinates - Retrieve the access coordinates for a generic port
129 * @uid: ACPI unique id
130 * @coord: The access coordinates written back out for the generic port.
131 *	   Expect 2 levels array.
132 *
133 * Return: 0 on success. Errno on failure.
134 *
135 * Only supports device handles that are ACPI. Assume ACPI0016 HID for CXL.
136 */
137int acpi_get_genport_coordinates(u32 uid,
138				 struct access_coordinate *coord)
139{
140	struct memory_target *target;
141
142	guard(mutex)(&target_lock);
143	target = acpi_find_genport_target(uid);
144	if (!target)
145		return -ENOENT;
146
147	coord[ACCESS_COORDINATE_LOCAL] =
148		target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL];
149	coord[ACCESS_COORDINATE_CPU] =
150		target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_CPU];
151
152	return 0;
153}
154EXPORT_SYMBOL_NS_GPL(acpi_get_genport_coordinates, CXL);
155
156static __init void alloc_memory_initiator(unsigned int cpu_pxm)
157{
158	struct memory_initiator *initiator;
159
160	if (pxm_to_node(cpu_pxm) == NUMA_NO_NODE)
161		return;
162
163	initiator = find_mem_initiator(cpu_pxm);
164	if (initiator)
165		return;
166
167	initiator = kzalloc(sizeof(*initiator), GFP_KERNEL);
168	if (!initiator)
169		return;
170
171	initiator->processor_pxm = cpu_pxm;
172	initiator->has_cpu = node_state(pxm_to_node(cpu_pxm), N_CPU);
173	list_add_tail(&initiator->node, &initiators);
174}
175
176static __init struct memory_target *alloc_target(unsigned int mem_pxm)
177{
178	struct memory_target *target;
179
180	target = find_mem_target(mem_pxm);
181	if (!target) {
182		target = kzalloc(sizeof(*target), GFP_KERNEL);
183		if (!target)
184			return NULL;
185		target->memory_pxm = mem_pxm;
186		target->processor_pxm = PXM_INVAL;
187		target->memregions = (struct resource) {
188			.name	= "ACPI mem",
189			.start	= 0,
190			.end	= -1,
191			.flags	= IORESOURCE_MEM,
192		};
193		list_add_tail(&target->node, &targets);
194		INIT_LIST_HEAD(&target->caches);
195	}
196
197	return target;
198}
199
200static __init void alloc_memory_target(unsigned int mem_pxm,
201				       resource_size_t start,
202				       resource_size_t len)
203{
204	struct memory_target *target;
205
206	target = alloc_target(mem_pxm);
207	if (!target)
208		return;
209
210	/*
211	 * There are potentially multiple ranges per PXM, so record each
212	 * in the per-target memregions resource tree.
213	 */
214	if (!__request_region(&target->memregions, start, len, "memory target",
215				IORESOURCE_MEM))
216		pr_warn("failed to reserve %#llx - %#llx in pxm: %d\n",
217				start, start + len, mem_pxm);
218}
219
220static __init void alloc_genport_target(unsigned int mem_pxm, u8 *handle)
221{
222	struct memory_target *target;
223
224	target = alloc_target(mem_pxm);
225	if (!target)
226		return;
227
228	memcpy(target->gen_port_device_handle, handle,
229	       ACPI_SRAT_DEVICE_HANDLE_SIZE);
230}
231
232static __init const char *hmat_data_type(u8 type)
233{
234	switch (type) {
235	case ACPI_HMAT_ACCESS_LATENCY:
236		return "Access Latency";
237	case ACPI_HMAT_READ_LATENCY:
238		return "Read Latency";
239	case ACPI_HMAT_WRITE_LATENCY:
240		return "Write Latency";
241	case ACPI_HMAT_ACCESS_BANDWIDTH:
242		return "Access Bandwidth";
243	case ACPI_HMAT_READ_BANDWIDTH:
244		return "Read Bandwidth";
245	case ACPI_HMAT_WRITE_BANDWIDTH:
246		return "Write Bandwidth";
247	default:
248		return "Reserved";
249	}
250}
251
252static __init const char *hmat_data_type_suffix(u8 type)
253{
254	switch (type) {
255	case ACPI_HMAT_ACCESS_LATENCY:
256	case ACPI_HMAT_READ_LATENCY:
257	case ACPI_HMAT_WRITE_LATENCY:
258		return " nsec";
259	case ACPI_HMAT_ACCESS_BANDWIDTH:
260	case ACPI_HMAT_READ_BANDWIDTH:
261	case ACPI_HMAT_WRITE_BANDWIDTH:
262		return " MB/s";
263	default:
264		return "";
265	}
266}
267
268static u32 hmat_normalize(u16 entry, u64 base, u8 type)
269{
270	u32 value;
271
272	/*
273	 * Check for invalid and overflow values
274	 */
275	if (entry == 0xffff || !entry)
276		return 0;
277	else if (base > (UINT_MAX / (entry)))
278		return 0;
279
280	/*
281	 * Divide by the base unit for version 1, convert latency from
282	 * picosenonds to nanoseconds if revision 2.
283	 */
284	value = entry * base;
285	if (hmat_revision == 1) {
286		if (value < 10)
287			return 0;
288		value = DIV_ROUND_UP(value, 10);
289	} else if (hmat_revision == 2) {
290		switch (type) {
291		case ACPI_HMAT_ACCESS_LATENCY:
292		case ACPI_HMAT_READ_LATENCY:
293		case ACPI_HMAT_WRITE_LATENCY:
294			value = DIV_ROUND_UP(value, 1000);
295			break;
296		default:
297			break;
298		}
299	}
300	return value;
301}
302
303static void hmat_update_target_access(struct memory_target *target,
304				      u8 type, u32 value, int access)
305{
306	switch (type) {
307	case ACPI_HMAT_ACCESS_LATENCY:
308		target->coord[access].read_latency = value;
309		target->coord[access].write_latency = value;
310		break;
311	case ACPI_HMAT_READ_LATENCY:
312		target->coord[access].read_latency = value;
313		break;
314	case ACPI_HMAT_WRITE_LATENCY:
315		target->coord[access].write_latency = value;
316		break;
317	case ACPI_HMAT_ACCESS_BANDWIDTH:
318		target->coord[access].read_bandwidth = value;
319		target->coord[access].write_bandwidth = value;
320		break;
321	case ACPI_HMAT_READ_BANDWIDTH:
322		target->coord[access].read_bandwidth = value;
323		break;
324	case ACPI_HMAT_WRITE_BANDWIDTH:
325		target->coord[access].write_bandwidth = value;
326		break;
327	default:
328		break;
329	}
330}
331
332int hmat_update_target_coordinates(int nid, struct access_coordinate *coord,
333				   enum access_coordinate_class access)
334{
335	struct memory_target *target;
336	int pxm;
337
338	if (nid == NUMA_NO_NODE)
339		return -EINVAL;
340
341	pxm = node_to_pxm(nid);
342	guard(mutex)(&target_lock);
343	target = find_mem_target(pxm);
344	if (!target)
345		return -ENODEV;
346
347	hmat_update_target_access(target, ACPI_HMAT_READ_LATENCY,
348				  coord->read_latency, access);
349	hmat_update_target_access(target, ACPI_HMAT_WRITE_LATENCY,
350				  coord->write_latency, access);
351	hmat_update_target_access(target, ACPI_HMAT_READ_BANDWIDTH,
352				  coord->read_bandwidth, access);
353	hmat_update_target_access(target, ACPI_HMAT_WRITE_BANDWIDTH,
354				  coord->write_bandwidth, access);
355	target->ext_updated = true;
356
357	return 0;
358}
359EXPORT_SYMBOL_GPL(hmat_update_target_coordinates);
360
361static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc)
362{
363	struct memory_locality *loc;
364
365	loc = kzalloc(sizeof(*loc), GFP_KERNEL);
366	if (!loc) {
367		pr_notice_once("Failed to allocate HMAT locality\n");
368		return;
369	}
370
371	loc->hmat_loc = hmat_loc;
372	list_add_tail(&loc->node, &localities);
373
374	switch (hmat_loc->data_type) {
375	case ACPI_HMAT_ACCESS_LATENCY:
376		localities_types[READ_LATENCY] = loc;
377		localities_types[WRITE_LATENCY] = loc;
378		break;
379	case ACPI_HMAT_READ_LATENCY:
380		localities_types[READ_LATENCY] = loc;
381		break;
382	case ACPI_HMAT_WRITE_LATENCY:
383		localities_types[WRITE_LATENCY] = loc;
384		break;
385	case ACPI_HMAT_ACCESS_BANDWIDTH:
386		localities_types[READ_BANDWIDTH] = loc;
387		localities_types[WRITE_BANDWIDTH] = loc;
388		break;
389	case ACPI_HMAT_READ_BANDWIDTH:
390		localities_types[READ_BANDWIDTH] = loc;
391		break;
392	case ACPI_HMAT_WRITE_BANDWIDTH:
393		localities_types[WRITE_BANDWIDTH] = loc;
394		break;
395	default:
396		break;
397	}
398}
399
400static __init void hmat_update_target(unsigned int tgt_pxm, unsigned int init_pxm,
401				      u8 mem_hier, u8 type, u32 value)
402{
403	struct memory_target *target = find_mem_target(tgt_pxm);
404
405	if (mem_hier != ACPI_HMAT_MEMORY)
406		return;
407
408	if (target && target->processor_pxm == init_pxm) {
409		hmat_update_target_access(target, type, value,
410					  ACCESS_COORDINATE_LOCAL);
411		/* If the node has a CPU, update access 1 */
412		if (node_state(pxm_to_node(init_pxm), N_CPU))
413			hmat_update_target_access(target, type, value,
414						  ACCESS_COORDINATE_CPU);
415	}
416}
417
418static __init int hmat_parse_locality(union acpi_subtable_headers *header,
419				      const unsigned long end)
420{
421	struct acpi_hmat_locality *hmat_loc = (void *)header;
422	unsigned int init, targ, total_size, ipds, tpds;
423	u32 *inits, *targs, value;
424	u16 *entries;
425	u8 type, mem_hier;
426
427	if (hmat_loc->header.length < sizeof(*hmat_loc)) {
428		pr_notice("Unexpected locality header length: %u\n",
429			 hmat_loc->header.length);
430		return -EINVAL;
431	}
432
433	type = hmat_loc->data_type;
434	mem_hier = hmat_loc->flags & ACPI_HMAT_MEMORY_HIERARCHY;
435	ipds = hmat_loc->number_of_initiator_Pds;
436	tpds = hmat_loc->number_of_target_Pds;
437	total_size = sizeof(*hmat_loc) + sizeof(*entries) * ipds * tpds +
438		     sizeof(*inits) * ipds + sizeof(*targs) * tpds;
439	if (hmat_loc->header.length < total_size) {
440		pr_notice("Unexpected locality header length:%u, minimum required:%u\n",
441			 hmat_loc->header.length, total_size);
442		return -EINVAL;
443	}
444
445	pr_info("Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n",
446		hmat_loc->flags, hmat_data_type(type), ipds, tpds,
447		hmat_loc->entry_base_unit);
448
449	inits = (u32 *)(hmat_loc + 1);
450	targs = inits + ipds;
451	entries = (u16 *)(targs + tpds);
452	for (init = 0; init < ipds; init++) {
453		alloc_memory_initiator(inits[init]);
454		for (targ = 0; targ < tpds; targ++) {
455			value = hmat_normalize(entries[init * tpds + targ],
456					       hmat_loc->entry_base_unit,
457					       type);
458			pr_info("  Initiator-Target[%u-%u]:%u%s\n",
459				inits[init], targs[targ], value,
460				hmat_data_type_suffix(type));
461
462			hmat_update_target(targs[targ], inits[init],
463					   mem_hier, type, value);
464		}
465	}
466
467	if (mem_hier == ACPI_HMAT_MEMORY)
468		hmat_add_locality(hmat_loc);
469
470	return 0;
471}
472
473static __init int hmat_parse_cache(union acpi_subtable_headers *header,
474				   const unsigned long end)
475{
476	struct acpi_hmat_cache *cache = (void *)header;
477	struct memory_target *target;
478	struct target_cache *tcache;
479	u32 attrs;
480
481	if (cache->header.length < sizeof(*cache)) {
482		pr_notice("Unexpected cache header length: %u\n",
483			 cache->header.length);
484		return -EINVAL;
485	}
486
487	attrs = cache->cache_attributes;
488	pr_info("Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n",
489		cache->memory_PD, cache->cache_size, attrs,
490		cache->number_of_SMBIOShandles);
491
492	target = find_mem_target(cache->memory_PD);
493	if (!target)
494		return 0;
495
496	tcache = kzalloc(sizeof(*tcache), GFP_KERNEL);
497	if (!tcache) {
498		pr_notice_once("Failed to allocate HMAT cache info\n");
499		return 0;
500	}
501
502	tcache->cache_attrs.size = cache->cache_size;
503	tcache->cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4;
504	tcache->cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16;
505
506	switch ((attrs & ACPI_HMAT_CACHE_ASSOCIATIVITY) >> 8) {
507	case ACPI_HMAT_CA_DIRECT_MAPPED:
508		tcache->cache_attrs.indexing = NODE_CACHE_DIRECT_MAP;
509		break;
510	case ACPI_HMAT_CA_COMPLEX_CACHE_INDEXING:
511		tcache->cache_attrs.indexing = NODE_CACHE_INDEXED;
512		break;
513	case ACPI_HMAT_CA_NONE:
514	default:
515		tcache->cache_attrs.indexing = NODE_CACHE_OTHER;
516		break;
517	}
518
519	switch ((attrs & ACPI_HMAT_WRITE_POLICY) >> 12) {
520	case ACPI_HMAT_CP_WB:
521		tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_BACK;
522		break;
523	case ACPI_HMAT_CP_WT:
524		tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH;
525		break;
526	case ACPI_HMAT_CP_NONE:
527	default:
528		tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER;
529		break;
530	}
531	list_add_tail(&tcache->node, &target->caches);
532
533	return 0;
534}
535
536static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *header,
537					      const unsigned long end)
538{
539	struct acpi_hmat_proximity_domain *p = (void *)header;
540	struct memory_target *target = NULL;
541
542	if (p->header.length != sizeof(*p)) {
543		pr_notice("Unexpected address range header length: %u\n",
544			 p->header.length);
545		return -EINVAL;
546	}
547
548	if (hmat_revision == 1)
549		pr_info("Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n",
550			p->reserved3, p->reserved4, p->flags, p->processor_PD,
551			p->memory_PD);
552	else
553		pr_info("Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n",
554			p->flags, p->processor_PD, p->memory_PD);
555
556	if ((hmat_revision == 1 && p->flags & ACPI_HMAT_MEMORY_PD_VALID) ||
557	    hmat_revision > 1) {
558		target = find_mem_target(p->memory_PD);
559		if (!target) {
560			pr_debug("Memory Domain missing from SRAT\n");
561			return -EINVAL;
562		}
563	}
564	if (target && p->flags & ACPI_HMAT_PROCESSOR_PD_VALID) {
565		int p_node = pxm_to_node(p->processor_PD);
566
567		if (p_node == NUMA_NO_NODE) {
568			pr_debug("Invalid Processor Domain\n");
569			return -EINVAL;
570		}
571		target->processor_pxm = p->processor_PD;
572	}
573
574	return 0;
575}
576
577static int __init hmat_parse_subtable(union acpi_subtable_headers *header,
578				      const unsigned long end)
579{
580	struct acpi_hmat_structure *hdr = (void *)header;
581
582	if (!hdr)
583		return -EINVAL;
584
585	switch (hdr->type) {
586	case ACPI_HMAT_TYPE_PROXIMITY:
587		return hmat_parse_proximity_domain(header, end);
588	case ACPI_HMAT_TYPE_LOCALITY:
589		return hmat_parse_locality(header, end);
590	case ACPI_HMAT_TYPE_CACHE:
591		return hmat_parse_cache(header, end);
592	default:
593		return -EINVAL;
594	}
595}
596
597static __init int srat_parse_mem_affinity(union acpi_subtable_headers *header,
598					  const unsigned long end)
599{
600	struct acpi_srat_mem_affinity *ma = (void *)header;
601
602	if (!ma)
603		return -EINVAL;
604	if (!(ma->flags & ACPI_SRAT_MEM_ENABLED))
605		return 0;
606	alloc_memory_target(ma->proximity_domain, ma->base_address, ma->length);
607	return 0;
608}
609
610static __init int srat_parse_genport_affinity(union acpi_subtable_headers *header,
611					      const unsigned long end)
612{
613	struct acpi_srat_generic_affinity *ga = (void *)header;
614
615	if (!ga)
616		return -EINVAL;
617
618	if (!(ga->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED))
619		return 0;
620
621	/* Skip PCI device_handle for now */
622	if (ga->device_handle_type != 0)
623		return 0;
624
625	alloc_genport_target(ga->proximity_domain,
626			     (u8 *)ga->device_handle);
627
628	return 0;
629}
630
631static u32 hmat_initiator_perf(struct memory_target *target,
632			       struct memory_initiator *initiator,
633			       struct acpi_hmat_locality *hmat_loc)
634{
635	unsigned int ipds, tpds, i, idx = 0, tdx = 0;
636	u32 *inits, *targs;
637	u16 *entries;
638
639	ipds = hmat_loc->number_of_initiator_Pds;
640	tpds = hmat_loc->number_of_target_Pds;
641	inits = (u32 *)(hmat_loc + 1);
642	targs = inits + ipds;
643	entries = (u16 *)(targs + tpds);
644
645	for (i = 0; i < ipds; i++) {
646		if (inits[i] == initiator->processor_pxm) {
647			idx = i;
648			break;
649		}
650	}
651
652	if (i == ipds)
653		return 0;
654
655	for (i = 0; i < tpds; i++) {
656		if (targs[i] == target->memory_pxm) {
657			tdx = i;
658			break;
659		}
660	}
661	if (i == tpds)
662		return 0;
663
664	return hmat_normalize(entries[idx * tpds + tdx],
665			      hmat_loc->entry_base_unit,
666			      hmat_loc->data_type);
667}
668
669static bool hmat_update_best(u8 type, u32 value, u32 *best)
670{
671	bool updated = false;
672
673	if (!value)
674		return false;
675
676	switch (type) {
677	case ACPI_HMAT_ACCESS_LATENCY:
678	case ACPI_HMAT_READ_LATENCY:
679	case ACPI_HMAT_WRITE_LATENCY:
680		if (!*best || *best > value) {
681			*best = value;
682			updated = true;
683		}
684		break;
685	case ACPI_HMAT_ACCESS_BANDWIDTH:
686	case ACPI_HMAT_READ_BANDWIDTH:
687	case ACPI_HMAT_WRITE_BANDWIDTH:
688		if (!*best || *best < value) {
689			*best = value;
690			updated = true;
691		}
692		break;
693	}
694
695	return updated;
696}
697
698static int initiator_cmp(void *priv, const struct list_head *a,
699			 const struct list_head *b)
700{
701	struct memory_initiator *ia;
702	struct memory_initiator *ib;
703
704	ia = list_entry(a, struct memory_initiator, node);
705	ib = list_entry(b, struct memory_initiator, node);
706
707	return ia->processor_pxm - ib->processor_pxm;
708}
709
710static int initiators_to_nodemask(unsigned long *p_nodes)
711{
712	struct memory_initiator *initiator;
713
714	if (list_empty(&initiators))
715		return -ENXIO;
716
717	list_for_each_entry(initiator, &initiators, node)
718		set_bit(initiator->processor_pxm, p_nodes);
719
720	return 0;
721}
722
723static void hmat_update_target_attrs(struct memory_target *target,
724				     unsigned long *p_nodes, int access)
725{
726	struct memory_initiator *initiator;
727	unsigned int cpu_nid;
728	struct memory_locality *loc = NULL;
729	u32 best = 0;
730	int i;
731
732	/* Don't update if an external agent has changed the data.  */
733	if (target->ext_updated)
734		return;
735
736	/* Don't update for generic port if there's no device handle */
737	if ((access == NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL ||
738	     access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) &&
739	    !(*(u16 *)target->gen_port_device_handle))
740		return;
741
742	bitmap_zero(p_nodes, MAX_NUMNODES);
743	/*
744	 * If the Address Range Structure provides a local processor pxm, set
745	 * only that one. Otherwise, find the best performance attributes and
746	 * collect all initiators that match.
747	 */
748	if (target->processor_pxm != PXM_INVAL) {
749		cpu_nid = pxm_to_node(target->processor_pxm);
750		if (access == ACCESS_COORDINATE_LOCAL ||
751		    node_state(cpu_nid, N_CPU)) {
752			set_bit(target->processor_pxm, p_nodes);
753			return;
754		}
755	}
756
757	if (list_empty(&localities))
758		return;
759
760	/*
761	 * We need the initiator list sorted so we can use bitmap_clear for
762	 * previously set initiators when we find a better memory accessor.
763	 * We'll also use the sorting to prime the candidate nodes with known
764	 * initiators.
765	 */
766	list_sort(NULL, &initiators, initiator_cmp);
767	if (initiators_to_nodemask(p_nodes) < 0)
768		return;
769
770	for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) {
771		loc = localities_types[i];
772		if (!loc)
773			continue;
774
775		best = 0;
776		list_for_each_entry(initiator, &initiators, node) {
777			u32 value;
778
779			if ((access == ACCESS_COORDINATE_CPU ||
780			     access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) &&
781			    !initiator->has_cpu) {
782				clear_bit(initiator->processor_pxm, p_nodes);
783				continue;
784			}
785			if (!test_bit(initiator->processor_pxm, p_nodes))
786				continue;
787
788			value = hmat_initiator_perf(target, initiator, loc->hmat_loc);
789			if (hmat_update_best(loc->hmat_loc->data_type, value, &best))
790				bitmap_clear(p_nodes, 0, initiator->processor_pxm);
791			if (value != best)
792				clear_bit(initiator->processor_pxm, p_nodes);
793		}
794		if (best)
795			hmat_update_target_access(target, loc->hmat_loc->data_type, best, access);
796	}
797}
798
799static void __hmat_register_target_initiators(struct memory_target *target,
800					      unsigned long *p_nodes,
801					      int access)
802{
803	unsigned int mem_nid, cpu_nid;
804	int i;
805
806	mem_nid = pxm_to_node(target->memory_pxm);
807	hmat_update_target_attrs(target, p_nodes, access);
808	for_each_set_bit(i, p_nodes, MAX_NUMNODES) {
809		cpu_nid = pxm_to_node(i);
810		register_memory_node_under_compute_node(mem_nid, cpu_nid, access);
811	}
812}
813
814static void hmat_update_generic_target(struct memory_target *target)
815{
816	static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
817
818	hmat_update_target_attrs(target, p_nodes,
819				 NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL);
820	hmat_update_target_attrs(target, p_nodes,
821				 NODE_ACCESS_CLASS_GENPORT_SINK_CPU);
822}
823
824static void hmat_register_target_initiators(struct memory_target *target)
825{
826	static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
827
828	__hmat_register_target_initiators(target, p_nodes,
829					  ACCESS_COORDINATE_LOCAL);
830	__hmat_register_target_initiators(target, p_nodes,
831					  ACCESS_COORDINATE_CPU);
832}
833
834static void hmat_register_target_cache(struct memory_target *target)
835{
836	unsigned mem_nid = pxm_to_node(target->memory_pxm);
837	struct target_cache *tcache;
838
839	list_for_each_entry(tcache, &target->caches, node)
840		node_add_cache(mem_nid, &tcache->cache_attrs);
841}
842
843static void hmat_register_target_perf(struct memory_target *target, int access)
844{
845	unsigned mem_nid = pxm_to_node(target->memory_pxm);
846	node_set_perf_attrs(mem_nid, &target->coord[access], access);
847}
848
849static void hmat_register_target_devices(struct memory_target *target)
850{
851	struct resource *res;
852
853	/*
854	 * Do not bother creating devices if no driver is available to
855	 * consume them.
856	 */
857	if (!IS_ENABLED(CONFIG_DEV_DAX_HMEM))
858		return;
859
860	for (res = target->memregions.child; res; res = res->sibling) {
861		int target_nid = pxm_to_node(target->memory_pxm);
862
863		hmem_register_resource(target_nid, res);
864	}
865}
866
867static void hmat_register_target(struct memory_target *target)
868{
869	int nid = pxm_to_node(target->memory_pxm);
870
871	/*
872	 * Devices may belong to either an offline or online
873	 * node, so unconditionally add them.
874	 */
875	hmat_register_target_devices(target);
876
877	/*
878	 * Register generic port perf numbers. The nid may not be
879	 * initialized and is still NUMA_NO_NODE.
880	 */
881	mutex_lock(&target_lock);
882	if (*(u16 *)target->gen_port_device_handle) {
883		hmat_update_generic_target(target);
884		target->registered = true;
885	}
886	mutex_unlock(&target_lock);
887
888	/*
889	 * Skip offline nodes. This can happen when memory
890	 * marked EFI_MEMORY_SP, "specific purpose", is applied
891	 * to all the memory in a proximity domain leading to
892	 * the node being marked offline / unplugged, or if
893	 * memory-only "hotplug" node is offline.
894	 */
895	if (nid == NUMA_NO_NODE || !node_online(nid))
896		return;
897
898	mutex_lock(&target_lock);
899	if (!target->registered) {
900		hmat_register_target_initiators(target);
901		hmat_register_target_cache(target);
902		hmat_register_target_perf(target, ACCESS_COORDINATE_LOCAL);
903		hmat_register_target_perf(target, ACCESS_COORDINATE_CPU);
904		target->registered = true;
905	}
906	mutex_unlock(&target_lock);
907}
908
909static void hmat_register_targets(void)
910{
911	struct memory_target *target;
912
913	list_for_each_entry(target, &targets, node)
914		hmat_register_target(target);
915}
916
917static int hmat_callback(struct notifier_block *self,
918			 unsigned long action, void *arg)
919{
920	struct memory_target *target;
921	struct memory_notify *mnb = arg;
922	int pxm, nid = mnb->status_change_nid;
923
924	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
925		return NOTIFY_OK;
926
927	pxm = node_to_pxm(nid);
928	target = find_mem_target(pxm);
929	if (!target)
930		return NOTIFY_OK;
931
932	hmat_register_target(target);
933	return NOTIFY_OK;
934}
935
936static int hmat_set_default_dram_perf(void)
937{
938	int rc;
939	int nid, pxm;
940	struct memory_target *target;
941	struct access_coordinate *attrs;
942
943	if (!default_dram_type)
944		return -EIO;
945
946	for_each_node_mask(nid, default_dram_type->nodes) {
947		pxm = node_to_pxm(nid);
948		target = find_mem_target(pxm);
949		if (!target)
950			continue;
951		attrs = &target->coord[1];
952		rc = mt_set_default_dram_perf(nid, attrs, "ACPI HMAT");
953		if (rc)
954			return rc;
955	}
956
957	return 0;
958}
959
960static int hmat_calculate_adistance(struct notifier_block *self,
961				    unsigned long nid, void *data)
962{
963	static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
964	struct memory_target *target;
965	struct access_coordinate *perf;
966	int *adist = data;
967	int pxm;
968
969	pxm = node_to_pxm(nid);
970	target = find_mem_target(pxm);
971	if (!target)
972		return NOTIFY_OK;
973
974	mutex_lock(&target_lock);
975	hmat_update_target_attrs(target, p_nodes, ACCESS_COORDINATE_CPU);
976	mutex_unlock(&target_lock);
977
978	perf = &target->coord[1];
979
980	if (mt_perf_to_adistance(perf, adist))
981		return NOTIFY_OK;
982
983	return NOTIFY_STOP;
984}
985
986static struct notifier_block hmat_adist_nb __meminitdata = {
987	.notifier_call = hmat_calculate_adistance,
988	.priority = 100,
989};
990
991static __init void hmat_free_structures(void)
992{
993	struct memory_target *target, *tnext;
994	struct memory_locality *loc, *lnext;
995	struct memory_initiator *initiator, *inext;
996	struct target_cache *tcache, *cnext;
997
998	list_for_each_entry_safe(target, tnext, &targets, node) {
999		struct resource *res, *res_next;
1000
1001		list_for_each_entry_safe(tcache, cnext, &target->caches, node) {
1002			list_del(&tcache->node);
1003			kfree(tcache);
1004		}
1005
1006		list_del(&target->node);
1007		res = target->memregions.child;
1008		while (res) {
1009			res_next = res->sibling;
1010			__release_region(&target->memregions, res->start,
1011					resource_size(res));
1012			res = res_next;
1013		}
1014		kfree(target);
1015	}
1016
1017	list_for_each_entry_safe(initiator, inext, &initiators, node) {
1018		list_del(&initiator->node);
1019		kfree(initiator);
1020	}
1021
1022	list_for_each_entry_safe(loc, lnext, &localities, node) {
1023		list_del(&loc->node);
1024		kfree(loc);
1025	}
1026}
1027
1028static __init int hmat_init(void)
1029{
1030	struct acpi_table_header *tbl;
1031	enum acpi_hmat_type i;
1032	acpi_status status;
1033
1034	if (srat_disabled() || hmat_disable)
1035		return 0;
1036
1037	status = acpi_get_table(ACPI_SIG_SRAT, 0, &tbl);
1038	if (ACPI_FAILURE(status))
1039		return 0;
1040
1041	if (acpi_table_parse_entries(ACPI_SIG_SRAT,
1042				sizeof(struct acpi_table_srat),
1043				ACPI_SRAT_TYPE_MEMORY_AFFINITY,
1044				srat_parse_mem_affinity, 0) < 0)
1045		goto out_put;
1046
1047	if (acpi_table_parse_entries(ACPI_SIG_SRAT,
1048				     sizeof(struct acpi_table_srat),
1049				     ACPI_SRAT_TYPE_GENERIC_PORT_AFFINITY,
1050				     srat_parse_genport_affinity, 0) < 0)
1051		goto out_put;
1052
1053	acpi_put_table(tbl);
1054
1055	status = acpi_get_table(ACPI_SIG_HMAT, 0, &tbl);
1056	if (ACPI_FAILURE(status))
1057		goto out_put;
1058
1059	hmat_revision = tbl->revision;
1060	switch (hmat_revision) {
1061	case 1:
1062	case 2:
1063		break;
1064	default:
1065		pr_notice("Ignoring: Unknown revision:%d\n", hmat_revision);
1066		goto out_put;
1067	}
1068
1069	for (i = ACPI_HMAT_TYPE_PROXIMITY; i < ACPI_HMAT_TYPE_RESERVED; i++) {
1070		if (acpi_table_parse_entries(ACPI_SIG_HMAT,
1071					     sizeof(struct acpi_table_hmat), i,
1072					     hmat_parse_subtable, 0) < 0) {
1073			pr_notice("Ignoring: Invalid table");
1074			goto out_put;
1075		}
1076	}
1077	hmat_register_targets();
1078
1079	/* Keep the table and structures if the notifier may use them */
1080	if (hotplug_memory_notifier(hmat_callback, HMAT_CALLBACK_PRI))
1081		goto out_put;
1082
1083	if (!hmat_set_default_dram_perf())
1084		register_mt_adistance_algorithm(&hmat_adist_nb);
1085
1086	return 0;
1087out_put:
1088	hmat_free_structures();
1089	acpi_put_table(tbl);
1090	return 0;
1091}
1092subsys_initcall(hmat_init);
1093