1// SPDX-License-Identifier: GPL-2.0 2#include <linux/slab.h> 3#include <linux/lockdep.h> 4#include <linux/sysfs.h> 5#include <linux/kobject.h> 6#include <linux/memory.h> 7#include <linux/memory-tiers.h> 8#include <linux/notifier.h> 9 10#include "internal.h" 11 12struct memory_tier { 13 /* hierarchy of memory tiers */ 14 struct list_head list; 15 /* list of all memory types part of this tier */ 16 struct list_head memory_types; 17 /* 18 * start value of abstract distance. memory tier maps 19 * an abstract distance range, 20 * adistance_start .. adistance_start + MEMTIER_CHUNK_SIZE 21 */ 22 int adistance_start; 23 struct device dev; 24 /* All the nodes that are part of all the lower memory tiers. */ 25 nodemask_t lower_tier_mask; 26}; 27 28struct demotion_nodes { 29 nodemask_t preferred; 30}; 31 32struct node_memory_type_map { 33 struct memory_dev_type *memtype; 34 int map_count; 35}; 36 37static DEFINE_MUTEX(memory_tier_lock); 38static LIST_HEAD(memory_tiers); 39static struct node_memory_type_map node_memory_types[MAX_NUMNODES]; 40struct memory_dev_type *default_dram_type; 41 42static const struct bus_type memory_tier_subsys = { 43 .name = "memory_tiering", 44 .dev_name = "memory_tier", 45}; 46 47#ifdef CONFIG_MIGRATION 48static int top_tier_adistance; 49/* 50 * node_demotion[] examples: 51 * 52 * Example 1: 53 * 54 * Node 0 & 1 are CPU + DRAM nodes, node 2 & 3 are PMEM nodes. 55 * 56 * node distances: 57 * node 0 1 2 3 58 * 0 10 20 30 40 59 * 1 20 10 40 30 60 * 2 30 40 10 40 61 * 3 40 30 40 10 62 * 63 * memory_tiers0 = 0-1 64 * memory_tiers1 = 2-3 65 * 66 * node_demotion[0].preferred = 2 67 * node_demotion[1].preferred = 3 68 * node_demotion[2].preferred = <empty> 69 * node_demotion[3].preferred = <empty> 70 * 71 * Example 2: 72 * 73 * Node 0 & 1 are CPU + DRAM nodes, node 2 is memory-only DRAM node. 74 * 75 * node distances: 76 * node 0 1 2 77 * 0 10 20 30 78 * 1 20 10 30 79 * 2 30 30 10 80 * 81 * memory_tiers0 = 0-2 82 * 83 * node_demotion[0].preferred = <empty> 84 * node_demotion[1].preferred = <empty> 85 * node_demotion[2].preferred = <empty> 86 * 87 * Example 3: 88 * 89 * Node 0 is CPU + DRAM nodes, Node 1 is HBM node, node 2 is PMEM node. 90 * 91 * node distances: 92 * node 0 1 2 93 * 0 10 20 30 94 * 1 20 10 40 95 * 2 30 40 10 96 * 97 * memory_tiers0 = 1 98 * memory_tiers1 = 0 99 * memory_tiers2 = 2 100 * 101 * node_demotion[0].preferred = 2 102 * node_demotion[1].preferred = 0 103 * node_demotion[2].preferred = <empty> 104 * 105 */ 106static struct demotion_nodes *node_demotion __read_mostly; 107#endif /* CONFIG_MIGRATION */ 108 109static BLOCKING_NOTIFIER_HEAD(mt_adistance_algorithms); 110 111static bool default_dram_perf_error; 112static struct access_coordinate default_dram_perf; 113static int default_dram_perf_ref_nid = NUMA_NO_NODE; 114static const char *default_dram_perf_ref_source; 115 116static inline struct memory_tier *to_memory_tier(struct device *device) 117{ 118 return container_of(device, struct memory_tier, dev); 119} 120 121static __always_inline nodemask_t get_memtier_nodemask(struct memory_tier *memtier) 122{ 123 nodemask_t nodes = NODE_MASK_NONE; 124 struct memory_dev_type *memtype; 125 126 list_for_each_entry(memtype, &memtier->memory_types, tier_sibling) 127 nodes_or(nodes, nodes, memtype->nodes); 128 129 return nodes; 130} 131 132static void memory_tier_device_release(struct device *dev) 133{ 134 struct memory_tier *tier = to_memory_tier(dev); 135 /* 136 * synchronize_rcu in clear_node_memory_tier makes sure 137 * we don't have rcu access to this memory tier. 138 */ 139 kfree(tier); 140} 141 142static ssize_t nodelist_show(struct device *dev, 143 struct device_attribute *attr, char *buf) 144{ 145 int ret; 146 nodemask_t nmask; 147 148 mutex_lock(&memory_tier_lock); 149 nmask = get_memtier_nodemask(to_memory_tier(dev)); 150 ret = sysfs_emit(buf, "%*pbl\n", nodemask_pr_args(&nmask)); 151 mutex_unlock(&memory_tier_lock); 152 return ret; 153} 154static DEVICE_ATTR_RO(nodelist); 155 156static struct attribute *memtier_dev_attrs[] = { 157 &dev_attr_nodelist.attr, 158 NULL 159}; 160 161static const struct attribute_group memtier_dev_group = { 162 .attrs = memtier_dev_attrs, 163}; 164 165static const struct attribute_group *memtier_dev_groups[] = { 166 &memtier_dev_group, 167 NULL 168}; 169 170static struct memory_tier *find_create_memory_tier(struct memory_dev_type *memtype) 171{ 172 int ret; 173 bool found_slot = false; 174 struct memory_tier *memtier, *new_memtier; 175 int adistance = memtype->adistance; 176 unsigned int memtier_adistance_chunk_size = MEMTIER_CHUNK_SIZE; 177 178 lockdep_assert_held_once(&memory_tier_lock); 179 180 adistance = round_down(adistance, memtier_adistance_chunk_size); 181 /* 182 * If the memtype is already part of a memory tier, 183 * just return that. 184 */ 185 if (!list_empty(&memtype->tier_sibling)) { 186 list_for_each_entry(memtier, &memory_tiers, list) { 187 if (adistance == memtier->adistance_start) 188 return memtier; 189 } 190 WARN_ON(1); 191 return ERR_PTR(-EINVAL); 192 } 193 194 list_for_each_entry(memtier, &memory_tiers, list) { 195 if (adistance == memtier->adistance_start) { 196 goto link_memtype; 197 } else if (adistance < memtier->adistance_start) { 198 found_slot = true; 199 break; 200 } 201 } 202 203 new_memtier = kzalloc(sizeof(struct memory_tier), GFP_KERNEL); 204 if (!new_memtier) 205 return ERR_PTR(-ENOMEM); 206 207 new_memtier->adistance_start = adistance; 208 INIT_LIST_HEAD(&new_memtier->list); 209 INIT_LIST_HEAD(&new_memtier->memory_types); 210 if (found_slot) 211 list_add_tail(&new_memtier->list, &memtier->list); 212 else 213 list_add_tail(&new_memtier->list, &memory_tiers); 214 215 new_memtier->dev.id = adistance >> MEMTIER_CHUNK_BITS; 216 new_memtier->dev.bus = &memory_tier_subsys; 217 new_memtier->dev.release = memory_tier_device_release; 218 new_memtier->dev.groups = memtier_dev_groups; 219 220 ret = device_register(&new_memtier->dev); 221 if (ret) { 222 list_del(&new_memtier->list); 223 put_device(&new_memtier->dev); 224 return ERR_PTR(ret); 225 } 226 memtier = new_memtier; 227 228link_memtype: 229 list_add(&memtype->tier_sibling, &memtier->memory_types); 230 return memtier; 231} 232 233static struct memory_tier *__node_get_memory_tier(int node) 234{ 235 pg_data_t *pgdat; 236 237 pgdat = NODE_DATA(node); 238 if (!pgdat) 239 return NULL; 240 /* 241 * Since we hold memory_tier_lock, we can avoid 242 * RCU read locks when accessing the details. No 243 * parallel updates are possible here. 244 */ 245 return rcu_dereference_check(pgdat->memtier, 246 lockdep_is_held(&memory_tier_lock)); 247} 248 249#ifdef CONFIG_MIGRATION 250bool node_is_toptier(int node) 251{ 252 bool toptier; 253 pg_data_t *pgdat; 254 struct memory_tier *memtier; 255 256 pgdat = NODE_DATA(node); 257 if (!pgdat) 258 return false; 259 260 rcu_read_lock(); 261 memtier = rcu_dereference(pgdat->memtier); 262 if (!memtier) { 263 toptier = true; 264 goto out; 265 } 266 if (memtier->adistance_start <= top_tier_adistance) 267 toptier = true; 268 else 269 toptier = false; 270out: 271 rcu_read_unlock(); 272 return toptier; 273} 274 275void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets) 276{ 277 struct memory_tier *memtier; 278 279 /* 280 * pg_data_t.memtier updates includes a synchronize_rcu() 281 * which ensures that we either find NULL or a valid memtier 282 * in NODE_DATA. protect the access via rcu_read_lock(); 283 */ 284 rcu_read_lock(); 285 memtier = rcu_dereference(pgdat->memtier); 286 if (memtier) 287 *targets = memtier->lower_tier_mask; 288 else 289 *targets = NODE_MASK_NONE; 290 rcu_read_unlock(); 291} 292 293/** 294 * next_demotion_node() - Get the next node in the demotion path 295 * @node: The starting node to lookup the next node 296 * 297 * Return: node id for next memory node in the demotion path hierarchy 298 * from @node; NUMA_NO_NODE if @node is terminal. This does not keep 299 * @node online or guarantee that it *continues* to be the next demotion 300 * target. 301 */ 302int next_demotion_node(int node) 303{ 304 struct demotion_nodes *nd; 305 int target; 306 307 if (!node_demotion) 308 return NUMA_NO_NODE; 309 310 nd = &node_demotion[node]; 311 312 /* 313 * node_demotion[] is updated without excluding this 314 * function from running. 315 * 316 * Make sure to use RCU over entire code blocks if 317 * node_demotion[] reads need to be consistent. 318 */ 319 rcu_read_lock(); 320 /* 321 * If there are multiple target nodes, just select one 322 * target node randomly. 323 * 324 * In addition, we can also use round-robin to select 325 * target node, but we should introduce another variable 326 * for node_demotion[] to record last selected target node, 327 * that may cause cache ping-pong due to the changing of 328 * last target node. Or introducing per-cpu data to avoid 329 * caching issue, which seems more complicated. So selecting 330 * target node randomly seems better until now. 331 */ 332 target = node_random(&nd->preferred); 333 rcu_read_unlock(); 334 335 return target; 336} 337 338static void disable_all_demotion_targets(void) 339{ 340 struct memory_tier *memtier; 341 int node; 342 343 for_each_node_state(node, N_MEMORY) { 344 node_demotion[node].preferred = NODE_MASK_NONE; 345 /* 346 * We are holding memory_tier_lock, it is safe 347 * to access pgda->memtier. 348 */ 349 memtier = __node_get_memory_tier(node); 350 if (memtier) 351 memtier->lower_tier_mask = NODE_MASK_NONE; 352 } 353 /* 354 * Ensure that the "disable" is visible across the system. 355 * Readers will see either a combination of before+disable 356 * state or disable+after. They will never see before and 357 * after state together. 358 */ 359 synchronize_rcu(); 360} 361 362static void dump_demotion_targets(void) 363{ 364 int node; 365 366 for_each_node_state(node, N_MEMORY) { 367 struct memory_tier *memtier = __node_get_memory_tier(node); 368 nodemask_t preferred = node_demotion[node].preferred; 369 370 if (!memtier) 371 continue; 372 373 if (nodes_empty(preferred)) 374 pr_info("Demotion targets for Node %d: null\n", node); 375 else 376 pr_info("Demotion targets for Node %d: preferred: %*pbl, fallback: %*pbl\n", 377 node, nodemask_pr_args(&preferred), 378 nodemask_pr_args(&memtier->lower_tier_mask)); 379 } 380} 381 382/* 383 * Find an automatic demotion target for all memory 384 * nodes. Failing here is OK. It might just indicate 385 * being at the end of a chain. 386 */ 387static void establish_demotion_targets(void) 388{ 389 struct memory_tier *memtier; 390 struct demotion_nodes *nd; 391 int target = NUMA_NO_NODE, node; 392 int distance, best_distance; 393 nodemask_t tier_nodes, lower_tier; 394 395 lockdep_assert_held_once(&memory_tier_lock); 396 397 if (!node_demotion) 398 return; 399 400 disable_all_demotion_targets(); 401 402 for_each_node_state(node, N_MEMORY) { 403 best_distance = -1; 404 nd = &node_demotion[node]; 405 406 memtier = __node_get_memory_tier(node); 407 if (!memtier || list_is_last(&memtier->list, &memory_tiers)) 408 continue; 409 /* 410 * Get the lower memtier to find the demotion node list. 411 */ 412 memtier = list_next_entry(memtier, list); 413 tier_nodes = get_memtier_nodemask(memtier); 414 /* 415 * find_next_best_node, use 'used' nodemask as a skip list. 416 * Add all memory nodes except the selected memory tier 417 * nodelist to skip list so that we find the best node from the 418 * memtier nodelist. 419 */ 420 nodes_andnot(tier_nodes, node_states[N_MEMORY], tier_nodes); 421 422 /* 423 * Find all the nodes in the memory tier node list of same best distance. 424 * add them to the preferred mask. We randomly select between nodes 425 * in the preferred mask when allocating pages during demotion. 426 */ 427 do { 428 target = find_next_best_node(node, &tier_nodes); 429 if (target == NUMA_NO_NODE) 430 break; 431 432 distance = node_distance(node, target); 433 if (distance == best_distance || best_distance == -1) { 434 best_distance = distance; 435 node_set(target, nd->preferred); 436 } else { 437 break; 438 } 439 } while (1); 440 } 441 /* 442 * Promotion is allowed from a memory tier to higher 443 * memory tier only if the memory tier doesn't include 444 * compute. We want to skip promotion from a memory tier, 445 * if any node that is part of the memory tier have CPUs. 446 * Once we detect such a memory tier, we consider that tier 447 * as top tiper from which promotion is not allowed. 448 */ 449 list_for_each_entry_reverse(memtier, &memory_tiers, list) { 450 tier_nodes = get_memtier_nodemask(memtier); 451 nodes_and(tier_nodes, node_states[N_CPU], tier_nodes); 452 if (!nodes_empty(tier_nodes)) { 453 /* 454 * abstract distance below the max value of this memtier 455 * is considered toptier. 456 */ 457 top_tier_adistance = memtier->adistance_start + 458 MEMTIER_CHUNK_SIZE - 1; 459 break; 460 } 461 } 462 /* 463 * Now build the lower_tier mask for each node collecting node mask from 464 * all memory tier below it. This allows us to fallback demotion page 465 * allocation to a set of nodes that is closer the above selected 466 * preferred node. 467 */ 468 lower_tier = node_states[N_MEMORY]; 469 list_for_each_entry(memtier, &memory_tiers, list) { 470 /* 471 * Keep removing current tier from lower_tier nodes, 472 * This will remove all nodes in current and above 473 * memory tier from the lower_tier mask. 474 */ 475 tier_nodes = get_memtier_nodemask(memtier); 476 nodes_andnot(lower_tier, lower_tier, tier_nodes); 477 memtier->lower_tier_mask = lower_tier; 478 } 479 480 dump_demotion_targets(); 481} 482 483#else 484static inline void establish_demotion_targets(void) {} 485#endif /* CONFIG_MIGRATION */ 486 487static inline void __init_node_memory_type(int node, struct memory_dev_type *memtype) 488{ 489 if (!node_memory_types[node].memtype) 490 node_memory_types[node].memtype = memtype; 491 /* 492 * for each device getting added in the same NUMA node 493 * with this specific memtype, bump the map count. We 494 * Only take memtype device reference once, so that 495 * changing a node memtype can be done by droping the 496 * only reference count taken here. 497 */ 498 499 if (node_memory_types[node].memtype == memtype) { 500 if (!node_memory_types[node].map_count++) 501 kref_get(&memtype->kref); 502 } 503} 504 505static struct memory_tier *set_node_memory_tier(int node) 506{ 507 struct memory_tier *memtier; 508 struct memory_dev_type *memtype; 509 pg_data_t *pgdat = NODE_DATA(node); 510 511 512 lockdep_assert_held_once(&memory_tier_lock); 513 514 if (!node_state(node, N_MEMORY)) 515 return ERR_PTR(-EINVAL); 516 517 __init_node_memory_type(node, default_dram_type); 518 519 memtype = node_memory_types[node].memtype; 520 node_set(node, memtype->nodes); 521 memtier = find_create_memory_tier(memtype); 522 if (!IS_ERR(memtier)) 523 rcu_assign_pointer(pgdat->memtier, memtier); 524 return memtier; 525} 526 527static void destroy_memory_tier(struct memory_tier *memtier) 528{ 529 list_del(&memtier->list); 530 device_unregister(&memtier->dev); 531} 532 533static bool clear_node_memory_tier(int node) 534{ 535 bool cleared = false; 536 pg_data_t *pgdat; 537 struct memory_tier *memtier; 538 539 pgdat = NODE_DATA(node); 540 if (!pgdat) 541 return false; 542 543 /* 544 * Make sure that anybody looking at NODE_DATA who finds 545 * a valid memtier finds memory_dev_types with nodes still 546 * linked to the memtier. We achieve this by waiting for 547 * rcu read section to finish using synchronize_rcu. 548 * This also enables us to free the destroyed memory tier 549 * with kfree instead of kfree_rcu 550 */ 551 memtier = __node_get_memory_tier(node); 552 if (memtier) { 553 struct memory_dev_type *memtype; 554 555 rcu_assign_pointer(pgdat->memtier, NULL); 556 synchronize_rcu(); 557 memtype = node_memory_types[node].memtype; 558 node_clear(node, memtype->nodes); 559 if (nodes_empty(memtype->nodes)) { 560 list_del_init(&memtype->tier_sibling); 561 if (list_empty(&memtier->memory_types)) 562 destroy_memory_tier(memtier); 563 } 564 cleared = true; 565 } 566 return cleared; 567} 568 569static void release_memtype(struct kref *kref) 570{ 571 struct memory_dev_type *memtype; 572 573 memtype = container_of(kref, struct memory_dev_type, kref); 574 kfree(memtype); 575} 576 577struct memory_dev_type *alloc_memory_type(int adistance) 578{ 579 struct memory_dev_type *memtype; 580 581 memtype = kmalloc(sizeof(*memtype), GFP_KERNEL); 582 if (!memtype) 583 return ERR_PTR(-ENOMEM); 584 585 memtype->adistance = adistance; 586 INIT_LIST_HEAD(&memtype->tier_sibling); 587 memtype->nodes = NODE_MASK_NONE; 588 kref_init(&memtype->kref); 589 return memtype; 590} 591EXPORT_SYMBOL_GPL(alloc_memory_type); 592 593void put_memory_type(struct memory_dev_type *memtype) 594{ 595 kref_put(&memtype->kref, release_memtype); 596} 597EXPORT_SYMBOL_GPL(put_memory_type); 598 599void init_node_memory_type(int node, struct memory_dev_type *memtype) 600{ 601 602 mutex_lock(&memory_tier_lock); 603 __init_node_memory_type(node, memtype); 604 mutex_unlock(&memory_tier_lock); 605} 606EXPORT_SYMBOL_GPL(init_node_memory_type); 607 608void clear_node_memory_type(int node, struct memory_dev_type *memtype) 609{ 610 mutex_lock(&memory_tier_lock); 611 if (node_memory_types[node].memtype == memtype || !memtype) 612 node_memory_types[node].map_count--; 613 /* 614 * If we umapped all the attached devices to this node, 615 * clear the node memory type. 616 */ 617 if (!node_memory_types[node].map_count) { 618 memtype = node_memory_types[node].memtype; 619 node_memory_types[node].memtype = NULL; 620 put_memory_type(memtype); 621 } 622 mutex_unlock(&memory_tier_lock); 623} 624EXPORT_SYMBOL_GPL(clear_node_memory_type); 625 626static void dump_hmem_attrs(struct access_coordinate *coord, const char *prefix) 627{ 628 pr_info( 629"%sread_latency: %u, write_latency: %u, read_bandwidth: %u, write_bandwidth: %u\n", 630 prefix, coord->read_latency, coord->write_latency, 631 coord->read_bandwidth, coord->write_bandwidth); 632} 633 634int mt_set_default_dram_perf(int nid, struct access_coordinate *perf, 635 const char *source) 636{ 637 int rc = 0; 638 639 mutex_lock(&memory_tier_lock); 640 if (default_dram_perf_error) { 641 rc = -EIO; 642 goto out; 643 } 644 645 if (perf->read_latency + perf->write_latency == 0 || 646 perf->read_bandwidth + perf->write_bandwidth == 0) { 647 rc = -EINVAL; 648 goto out; 649 } 650 651 if (default_dram_perf_ref_nid == NUMA_NO_NODE) { 652 default_dram_perf = *perf; 653 default_dram_perf_ref_nid = nid; 654 default_dram_perf_ref_source = kstrdup(source, GFP_KERNEL); 655 goto out; 656 } 657 658 /* 659 * The performance of all default DRAM nodes is expected to be 660 * same (that is, the variation is less than 10%). And it 661 * will be used as base to calculate the abstract distance of 662 * other memory nodes. 663 */ 664 if (abs(perf->read_latency - default_dram_perf.read_latency) * 10 > 665 default_dram_perf.read_latency || 666 abs(perf->write_latency - default_dram_perf.write_latency) * 10 > 667 default_dram_perf.write_latency || 668 abs(perf->read_bandwidth - default_dram_perf.read_bandwidth) * 10 > 669 default_dram_perf.read_bandwidth || 670 abs(perf->write_bandwidth - default_dram_perf.write_bandwidth) * 10 > 671 default_dram_perf.write_bandwidth) { 672 pr_info( 673"memory-tiers: the performance of DRAM node %d mismatches that of the reference\n" 674"DRAM node %d.\n", nid, default_dram_perf_ref_nid); 675 pr_info(" performance of reference DRAM node %d:\n", 676 default_dram_perf_ref_nid); 677 dump_hmem_attrs(&default_dram_perf, " "); 678 pr_info(" performance of DRAM node %d:\n", nid); 679 dump_hmem_attrs(perf, " "); 680 pr_info( 681" disable default DRAM node performance based abstract distance algorithm.\n"); 682 default_dram_perf_error = true; 683 rc = -EINVAL; 684 } 685 686out: 687 mutex_unlock(&memory_tier_lock); 688 return rc; 689} 690 691int mt_perf_to_adistance(struct access_coordinate *perf, int *adist) 692{ 693 if (default_dram_perf_error) 694 return -EIO; 695 696 if (default_dram_perf_ref_nid == NUMA_NO_NODE) 697 return -ENOENT; 698 699 if (perf->read_latency + perf->write_latency == 0 || 700 perf->read_bandwidth + perf->write_bandwidth == 0) 701 return -EINVAL; 702 703 mutex_lock(&memory_tier_lock); 704 /* 705 * The abstract distance of a memory node is in direct proportion to 706 * its memory latency (read + write) and inversely proportional to its 707 * memory bandwidth (read + write). The abstract distance, memory 708 * latency, and memory bandwidth of the default DRAM nodes are used as 709 * the base. 710 */ 711 *adist = MEMTIER_ADISTANCE_DRAM * 712 (perf->read_latency + perf->write_latency) / 713 (default_dram_perf.read_latency + default_dram_perf.write_latency) * 714 (default_dram_perf.read_bandwidth + default_dram_perf.write_bandwidth) / 715 (perf->read_bandwidth + perf->write_bandwidth); 716 mutex_unlock(&memory_tier_lock); 717 718 return 0; 719} 720EXPORT_SYMBOL_GPL(mt_perf_to_adistance); 721 722/** 723 * register_mt_adistance_algorithm() - Register memory tiering abstract distance algorithm 724 * @nb: The notifier block which describe the algorithm 725 * 726 * Return: 0 on success, errno on error. 727 * 728 * Every memory tiering abstract distance algorithm provider needs to 729 * register the algorithm with register_mt_adistance_algorithm(). To 730 * calculate the abstract distance for a specified memory node, the 731 * notifier function will be called unless some high priority 732 * algorithm has provided result. The prototype of the notifier 733 * function is as follows, 734 * 735 * int (*algorithm_notifier)(struct notifier_block *nb, 736 * unsigned long nid, void *data); 737 * 738 * Where "nid" specifies the memory node, "data" is the pointer to the 739 * returned abstract distance (that is, "int *adist"). If the 740 * algorithm provides the result, NOTIFY_STOP should be returned. 741 * Otherwise, return_value & %NOTIFY_STOP_MASK == 0 to allow the next 742 * algorithm in the chain to provide the result. 743 */ 744int register_mt_adistance_algorithm(struct notifier_block *nb) 745{ 746 return blocking_notifier_chain_register(&mt_adistance_algorithms, nb); 747} 748EXPORT_SYMBOL_GPL(register_mt_adistance_algorithm); 749 750/** 751 * unregister_mt_adistance_algorithm() - Unregister memory tiering abstract distance algorithm 752 * @nb: the notifier block which describe the algorithm 753 * 754 * Return: 0 on success, errno on error. 755 */ 756int unregister_mt_adistance_algorithm(struct notifier_block *nb) 757{ 758 return blocking_notifier_chain_unregister(&mt_adistance_algorithms, nb); 759} 760EXPORT_SYMBOL_GPL(unregister_mt_adistance_algorithm); 761 762/** 763 * mt_calc_adistance() - Calculate abstract distance with registered algorithms 764 * @node: the node to calculate abstract distance for 765 * @adist: the returned abstract distance 766 * 767 * Return: if return_value & %NOTIFY_STOP_MASK != 0, then some 768 * abstract distance algorithm provides the result, and return it via 769 * @adist. Otherwise, no algorithm can provide the result and @adist 770 * will be kept as it is. 771 */ 772int mt_calc_adistance(int node, int *adist) 773{ 774 return blocking_notifier_call_chain(&mt_adistance_algorithms, node, adist); 775} 776EXPORT_SYMBOL_GPL(mt_calc_adistance); 777 778static int __meminit memtier_hotplug_callback(struct notifier_block *self, 779 unsigned long action, void *_arg) 780{ 781 struct memory_tier *memtier; 782 struct memory_notify *arg = _arg; 783 784 /* 785 * Only update the node migration order when a node is 786 * changing status, like online->offline. 787 */ 788 if (arg->status_change_nid < 0) 789 return notifier_from_errno(0); 790 791 switch (action) { 792 case MEM_OFFLINE: 793 mutex_lock(&memory_tier_lock); 794 if (clear_node_memory_tier(arg->status_change_nid)) 795 establish_demotion_targets(); 796 mutex_unlock(&memory_tier_lock); 797 break; 798 case MEM_ONLINE: 799 mutex_lock(&memory_tier_lock); 800 memtier = set_node_memory_tier(arg->status_change_nid); 801 if (!IS_ERR(memtier)) 802 establish_demotion_targets(); 803 mutex_unlock(&memory_tier_lock); 804 break; 805 } 806 807 return notifier_from_errno(0); 808} 809 810static int __init memory_tier_init(void) 811{ 812 int ret, node; 813 struct memory_tier *memtier; 814 815 ret = subsys_virtual_register(&memory_tier_subsys, NULL); 816 if (ret) 817 panic("%s() failed to register memory tier subsystem\n", __func__); 818 819#ifdef CONFIG_MIGRATION 820 node_demotion = kcalloc(nr_node_ids, sizeof(struct demotion_nodes), 821 GFP_KERNEL); 822 WARN_ON(!node_demotion); 823#endif 824 mutex_lock(&memory_tier_lock); 825 /* 826 * For now we can have 4 faster memory tiers with smaller adistance 827 * than default DRAM tier. 828 */ 829 default_dram_type = alloc_memory_type(MEMTIER_ADISTANCE_DRAM); 830 if (IS_ERR(default_dram_type)) 831 panic("%s() failed to allocate default DRAM tier\n", __func__); 832 833 /* 834 * Look at all the existing N_MEMORY nodes and add them to 835 * default memory tier or to a tier if we already have memory 836 * types assigned. 837 */ 838 for_each_node_state(node, N_MEMORY) { 839 memtier = set_node_memory_tier(node); 840 if (IS_ERR(memtier)) 841 /* 842 * Continue with memtiers we are able to setup 843 */ 844 break; 845 } 846 establish_demotion_targets(); 847 mutex_unlock(&memory_tier_lock); 848 849 hotplug_memory_notifier(memtier_hotplug_callback, MEMTIER_HOTPLUG_PRI); 850 return 0; 851} 852subsys_initcall(memory_tier_init); 853 854bool numa_demotion_enabled = false; 855 856#ifdef CONFIG_MIGRATION 857#ifdef CONFIG_SYSFS 858static ssize_t demotion_enabled_show(struct kobject *kobj, 859 struct kobj_attribute *attr, char *buf) 860{ 861 return sysfs_emit(buf, "%s\n", 862 numa_demotion_enabled ? "true" : "false"); 863} 864 865static ssize_t demotion_enabled_store(struct kobject *kobj, 866 struct kobj_attribute *attr, 867 const char *buf, size_t count) 868{ 869 ssize_t ret; 870 871 ret = kstrtobool(buf, &numa_demotion_enabled); 872 if (ret) 873 return ret; 874 875 return count; 876} 877 878static struct kobj_attribute numa_demotion_enabled_attr = 879 __ATTR_RW(demotion_enabled); 880 881static struct attribute *numa_attrs[] = { 882 &numa_demotion_enabled_attr.attr, 883 NULL, 884}; 885 886static const struct attribute_group numa_attr_group = { 887 .attrs = numa_attrs, 888}; 889 890static int __init numa_init_sysfs(void) 891{ 892 int err; 893 struct kobject *numa_kobj; 894 895 numa_kobj = kobject_create_and_add("numa", mm_kobj); 896 if (!numa_kobj) { 897 pr_err("failed to create numa kobject\n"); 898 return -ENOMEM; 899 } 900 err = sysfs_create_group(numa_kobj, &numa_attr_group); 901 if (err) { 902 pr_err("failed to register numa group\n"); 903 goto delete_obj; 904 } 905 return 0; 906 907delete_obj: 908 kobject_put(numa_kobj); 909 return err; 910} 911subsys_initcall(numa_init_sysfs); 912#endif /* CONFIG_SYSFS */ 913#endif 914