1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Interconnect framework core driver 4 * 5 * Copyright (c) 2017-2019, Linaro Ltd. 6 * Author: Georgi Djakov <georgi.djakov@linaro.org> 7 */ 8 9#include <linux/debugfs.h> 10#include <linux/device.h> 11#include <linux/idr.h> 12#include <linux/init.h> 13#include <linux/interconnect.h> 14#include <linux/interconnect-provider.h> 15#include <linux/list.h> 16#include <linux/mutex.h> 17#include <linux/slab.h> 18#include <linux/of.h> 19#include <linux/overflow.h> 20 21#include "internal.h" 22 23#define CREATE_TRACE_POINTS 24#include "trace.h" 25 26static DEFINE_IDR(icc_idr); 27static LIST_HEAD(icc_providers); 28static int providers_count; 29static bool synced_state; 30static DEFINE_MUTEX(icc_lock); 31static DEFINE_MUTEX(icc_bw_lock); 32static struct dentry *icc_debugfs_dir; 33 34static void icc_summary_show_one(struct seq_file *s, struct icc_node *n) 35{ 36 if (!n) 37 return; 38 39 seq_printf(s, "%-42s %12u %12u\n", 40 n->name, n->avg_bw, n->peak_bw); 41} 42 43static int icc_summary_show(struct seq_file *s, void *data) 44{ 45 struct icc_provider *provider; 46 47 seq_puts(s, " node tag avg peak\n"); 48 seq_puts(s, "--------------------------------------------------------------------\n"); 49 50 mutex_lock(&icc_lock); 51 52 list_for_each_entry(provider, &icc_providers, provider_list) { 53 struct icc_node *n; 54 55 list_for_each_entry(n, &provider->nodes, node_list) { 56 struct icc_req *r; 57 58 icc_summary_show_one(s, n); 59 hlist_for_each_entry(r, &n->req_list, req_node) { 60 u32 avg_bw = 0, peak_bw = 0; 61 62 if (!r->dev) 63 continue; 64 65 if (r->enabled) { 66 avg_bw = r->avg_bw; 67 peak_bw = r->peak_bw; 68 } 69 70 seq_printf(s, " %-27s %12u %12u %12u\n", 71 dev_name(r->dev), r->tag, avg_bw, peak_bw); 72 } 73 } 74 } 75 76 mutex_unlock(&icc_lock); 77 78 return 0; 79} 80DEFINE_SHOW_ATTRIBUTE(icc_summary); 81 82static void icc_graph_show_link(struct seq_file *s, int level, 83 struct icc_node *n, struct icc_node *m) 84{ 85 seq_printf(s, "%s\"%d:%s\" -> \"%d:%s\"\n", 86 level == 2 ? "\t\t" : "\t", 87 n->id, n->name, m->id, m->name); 88} 89 90static void icc_graph_show_node(struct seq_file *s, struct icc_node *n) 91{ 92 seq_printf(s, "\t\t\"%d:%s\" [label=\"%d:%s", 93 n->id, n->name, n->id, n->name); 94 seq_printf(s, "\n\t\t\t|avg_bw=%ukBps", n->avg_bw); 95 seq_printf(s, "\n\t\t\t|peak_bw=%ukBps", n->peak_bw); 96 seq_puts(s, "\"]\n"); 97} 98 99static int icc_graph_show(struct seq_file *s, void *data) 100{ 101 struct icc_provider *provider; 102 struct icc_node *n; 103 int cluster_index = 0; 104 int i; 105 106 seq_puts(s, "digraph {\n\trankdir = LR\n\tnode [shape = record]\n"); 107 mutex_lock(&icc_lock); 108 109 /* draw providers as cluster subgraphs */ 110 cluster_index = 0; 111 list_for_each_entry(provider, &icc_providers, provider_list) { 112 seq_printf(s, "\tsubgraph cluster_%d {\n", ++cluster_index); 113 if (provider->dev) 114 seq_printf(s, "\t\tlabel = \"%s\"\n", 115 dev_name(provider->dev)); 116 117 /* draw nodes */ 118 list_for_each_entry(n, &provider->nodes, node_list) 119 icc_graph_show_node(s, n); 120 121 /* draw internal links */ 122 list_for_each_entry(n, &provider->nodes, node_list) 123 for (i = 0; i < n->num_links; ++i) 124 if (n->provider == n->links[i]->provider) 125 icc_graph_show_link(s, 2, n, 126 n->links[i]); 127 128 seq_puts(s, "\t}\n"); 129 } 130 131 /* draw external links */ 132 list_for_each_entry(provider, &icc_providers, provider_list) 133 list_for_each_entry(n, &provider->nodes, node_list) 134 for (i = 0; i < n->num_links; ++i) 135 if (n->provider != n->links[i]->provider) 136 icc_graph_show_link(s, 1, n, 137 n->links[i]); 138 139 mutex_unlock(&icc_lock); 140 seq_puts(s, "}"); 141 142 return 0; 143} 144DEFINE_SHOW_ATTRIBUTE(icc_graph); 145 146static struct icc_node *node_find(const int id) 147{ 148 return idr_find(&icc_idr, id); 149} 150 151static struct icc_node *node_find_by_name(const char *name) 152{ 153 struct icc_provider *provider; 154 struct icc_node *n; 155 156 list_for_each_entry(provider, &icc_providers, provider_list) { 157 list_for_each_entry(n, &provider->nodes, node_list) { 158 if (!strcmp(n->name, name)) 159 return n; 160 } 161 } 162 163 return NULL; 164} 165 166static struct icc_path *path_init(struct device *dev, struct icc_node *dst, 167 ssize_t num_nodes) 168{ 169 struct icc_node *node = dst; 170 struct icc_path *path; 171 int i; 172 173 path = kzalloc(struct_size(path, reqs, num_nodes), GFP_KERNEL); 174 if (!path) 175 return ERR_PTR(-ENOMEM); 176 177 path->num_nodes = num_nodes; 178 179 mutex_lock(&icc_bw_lock); 180 181 for (i = num_nodes - 1; i >= 0; i--) { 182 node->provider->users++; 183 hlist_add_head(&path->reqs[i].req_node, &node->req_list); 184 path->reqs[i].node = node; 185 path->reqs[i].dev = dev; 186 path->reqs[i].enabled = true; 187 /* reference to previous node was saved during path traversal */ 188 node = node->reverse; 189 } 190 191 mutex_unlock(&icc_bw_lock); 192 193 return path; 194} 195 196static struct icc_path *path_find(struct device *dev, struct icc_node *src, 197 struct icc_node *dst) 198{ 199 struct icc_path *path = ERR_PTR(-EPROBE_DEFER); 200 struct icc_node *n, *node = NULL; 201 struct list_head traverse_list; 202 struct list_head edge_list; 203 struct list_head visited_list; 204 size_t i, depth = 1; 205 bool found = false; 206 207 INIT_LIST_HEAD(&traverse_list); 208 INIT_LIST_HEAD(&edge_list); 209 INIT_LIST_HEAD(&visited_list); 210 211 list_add(&src->search_list, &traverse_list); 212 src->reverse = NULL; 213 214 do { 215 list_for_each_entry_safe(node, n, &traverse_list, search_list) { 216 if (node == dst) { 217 found = true; 218 list_splice_init(&edge_list, &visited_list); 219 list_splice_init(&traverse_list, &visited_list); 220 break; 221 } 222 for (i = 0; i < node->num_links; i++) { 223 struct icc_node *tmp = node->links[i]; 224 225 if (!tmp) { 226 path = ERR_PTR(-ENOENT); 227 goto out; 228 } 229 230 if (tmp->is_traversed) 231 continue; 232 233 tmp->is_traversed = true; 234 tmp->reverse = node; 235 list_add_tail(&tmp->search_list, &edge_list); 236 } 237 } 238 239 if (found) 240 break; 241 242 list_splice_init(&traverse_list, &visited_list); 243 list_splice_init(&edge_list, &traverse_list); 244 245 /* count the hops including the source */ 246 depth++; 247 248 } while (!list_empty(&traverse_list)); 249 250out: 251 252 /* reset the traversed state */ 253 list_for_each_entry_reverse(n, &visited_list, search_list) 254 n->is_traversed = false; 255 256 if (found) 257 path = path_init(dev, dst, depth); 258 259 return path; 260} 261 262/* 263 * We want the path to honor all bandwidth requests, so the average and peak 264 * bandwidth requirements from each consumer are aggregated at each node. 265 * The aggregation is platform specific, so each platform can customize it by 266 * implementing its own aggregate() function. 267 */ 268 269static int aggregate_requests(struct icc_node *node) 270{ 271 struct icc_provider *p = node->provider; 272 struct icc_req *r; 273 u32 avg_bw, peak_bw; 274 275 node->avg_bw = 0; 276 node->peak_bw = 0; 277 278 if (p->pre_aggregate) 279 p->pre_aggregate(node); 280 281 hlist_for_each_entry(r, &node->req_list, req_node) { 282 if (r->enabled) { 283 avg_bw = r->avg_bw; 284 peak_bw = r->peak_bw; 285 } else { 286 avg_bw = 0; 287 peak_bw = 0; 288 } 289 p->aggregate(node, r->tag, avg_bw, peak_bw, 290 &node->avg_bw, &node->peak_bw); 291 292 /* during boot use the initial bandwidth as a floor value */ 293 if (!synced_state) { 294 node->avg_bw = max(node->avg_bw, node->init_avg); 295 node->peak_bw = max(node->peak_bw, node->init_peak); 296 } 297 } 298 299 return 0; 300} 301 302static int apply_constraints(struct icc_path *path) 303{ 304 struct icc_node *next, *prev = NULL; 305 struct icc_provider *p; 306 int ret = -EINVAL; 307 int i; 308 309 for (i = 0; i < path->num_nodes; i++) { 310 next = path->reqs[i].node; 311 p = next->provider; 312 313 /* both endpoints should be valid master-slave pairs */ 314 if (!prev || (p != prev->provider && !p->inter_set)) { 315 prev = next; 316 continue; 317 } 318 319 /* set the constraints */ 320 ret = p->set(prev, next); 321 if (ret) 322 goto out; 323 324 prev = next; 325 } 326out: 327 return ret; 328} 329 330int icc_std_aggregate(struct icc_node *node, u32 tag, u32 avg_bw, 331 u32 peak_bw, u32 *agg_avg, u32 *agg_peak) 332{ 333 *agg_avg += avg_bw; 334 *agg_peak = max(*agg_peak, peak_bw); 335 336 return 0; 337} 338EXPORT_SYMBOL_GPL(icc_std_aggregate); 339 340/* of_icc_xlate_onecell() - Translate function using a single index. 341 * @spec: OF phandle args to map into an interconnect node. 342 * @data: private data (pointer to struct icc_onecell_data) 343 * 344 * This is a generic translate function that can be used to model simple 345 * interconnect providers that have one device tree node and provide 346 * multiple interconnect nodes. A single cell is used as an index into 347 * an array of icc nodes specified in the icc_onecell_data struct when 348 * registering the provider. 349 */ 350struct icc_node *of_icc_xlate_onecell(const struct of_phandle_args *spec, 351 void *data) 352{ 353 struct icc_onecell_data *icc_data = data; 354 unsigned int idx = spec->args[0]; 355 356 if (idx >= icc_data->num_nodes) { 357 pr_err("%s: invalid index %u\n", __func__, idx); 358 return ERR_PTR(-EINVAL); 359 } 360 361 return icc_data->nodes[idx]; 362} 363EXPORT_SYMBOL_GPL(of_icc_xlate_onecell); 364 365/** 366 * of_icc_get_from_provider() - Look-up interconnect node 367 * @spec: OF phandle args to use for look-up 368 * 369 * Looks for interconnect provider under the node specified by @spec and if 370 * found, uses xlate function of the provider to map phandle args to node. 371 * 372 * Returns a valid pointer to struct icc_node_data on success or ERR_PTR() 373 * on failure. 374 */ 375struct icc_node_data *of_icc_get_from_provider(const struct of_phandle_args *spec) 376{ 377 struct icc_node *node = ERR_PTR(-EPROBE_DEFER); 378 struct icc_node_data *data = NULL; 379 struct icc_provider *provider; 380 381 if (!spec) 382 return ERR_PTR(-EINVAL); 383 384 mutex_lock(&icc_lock); 385 list_for_each_entry(provider, &icc_providers, provider_list) { 386 if (provider->dev->of_node == spec->np) { 387 if (provider->xlate_extended) { 388 data = provider->xlate_extended(spec, provider->data); 389 if (!IS_ERR(data)) { 390 node = data->node; 391 break; 392 } 393 } else { 394 node = provider->xlate(spec, provider->data); 395 if (!IS_ERR(node)) 396 break; 397 } 398 } 399 } 400 mutex_unlock(&icc_lock); 401 402 if (!node) 403 return ERR_PTR(-EINVAL); 404 405 if (IS_ERR(node)) 406 return ERR_CAST(node); 407 408 if (!data) { 409 data = kzalloc(sizeof(*data), GFP_KERNEL); 410 if (!data) 411 return ERR_PTR(-ENOMEM); 412 data->node = node; 413 } 414 415 return data; 416} 417EXPORT_SYMBOL_GPL(of_icc_get_from_provider); 418 419static void devm_icc_release(struct device *dev, void *res) 420{ 421 icc_put(*(struct icc_path **)res); 422} 423 424struct icc_path *devm_of_icc_get(struct device *dev, const char *name) 425{ 426 struct icc_path **ptr, *path; 427 428 ptr = devres_alloc(devm_icc_release, sizeof(*ptr), GFP_KERNEL); 429 if (!ptr) 430 return ERR_PTR(-ENOMEM); 431 432 path = of_icc_get(dev, name); 433 if (!IS_ERR(path)) { 434 *ptr = path; 435 devres_add(dev, ptr); 436 } else { 437 devres_free(ptr); 438 } 439 440 return path; 441} 442EXPORT_SYMBOL_GPL(devm_of_icc_get); 443 444/** 445 * of_icc_get_by_index() - get a path handle from a DT node based on index 446 * @dev: device pointer for the consumer device 447 * @idx: interconnect path index 448 * 449 * This function will search for a path between two endpoints and return an 450 * icc_path handle on success. Use icc_put() to release constraints when they 451 * are not needed anymore. 452 * If the interconnect API is disabled, NULL is returned and the consumer 453 * drivers will still build. Drivers are free to handle this specifically, 454 * but they don't have to. 455 * 456 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned 457 * when the API is disabled or the "interconnects" DT property is missing. 458 */ 459struct icc_path *of_icc_get_by_index(struct device *dev, int idx) 460{ 461 struct icc_path *path; 462 struct icc_node_data *src_data, *dst_data; 463 struct device_node *np; 464 struct of_phandle_args src_args, dst_args; 465 int ret; 466 467 if (!dev || !dev->of_node) 468 return ERR_PTR(-ENODEV); 469 470 np = dev->of_node; 471 472 /* 473 * When the consumer DT node do not have "interconnects" property 474 * return a NULL path to skip setting constraints. 475 */ 476 if (!of_property_present(np, "interconnects")) 477 return NULL; 478 479 /* 480 * We use a combination of phandle and specifier for endpoint. For now 481 * lets support only global ids and extend this in the future if needed 482 * without breaking DT compatibility. 483 */ 484 ret = of_parse_phandle_with_args(np, "interconnects", 485 "#interconnect-cells", idx * 2, 486 &src_args); 487 if (ret) 488 return ERR_PTR(ret); 489 490 of_node_put(src_args.np); 491 492 ret = of_parse_phandle_with_args(np, "interconnects", 493 "#interconnect-cells", idx * 2 + 1, 494 &dst_args); 495 if (ret) 496 return ERR_PTR(ret); 497 498 of_node_put(dst_args.np); 499 500 src_data = of_icc_get_from_provider(&src_args); 501 502 if (IS_ERR(src_data)) { 503 dev_err_probe(dev, PTR_ERR(src_data), "error finding src node\n"); 504 return ERR_CAST(src_data); 505 } 506 507 dst_data = of_icc_get_from_provider(&dst_args); 508 509 if (IS_ERR(dst_data)) { 510 dev_err_probe(dev, PTR_ERR(dst_data), "error finding dst node\n"); 511 kfree(src_data); 512 return ERR_CAST(dst_data); 513 } 514 515 mutex_lock(&icc_lock); 516 path = path_find(dev, src_data->node, dst_data->node); 517 mutex_unlock(&icc_lock); 518 if (IS_ERR(path)) { 519 dev_err(dev, "%s: invalid path=%ld\n", __func__, PTR_ERR(path)); 520 goto free_icc_data; 521 } 522 523 if (src_data->tag && src_data->tag == dst_data->tag) 524 icc_set_tag(path, src_data->tag); 525 526 path->name = kasprintf(GFP_KERNEL, "%s-%s", 527 src_data->node->name, dst_data->node->name); 528 if (!path->name) { 529 kfree(path); 530 path = ERR_PTR(-ENOMEM); 531 } 532 533free_icc_data: 534 kfree(src_data); 535 kfree(dst_data); 536 return path; 537} 538EXPORT_SYMBOL_GPL(of_icc_get_by_index); 539 540/** 541 * of_icc_get() - get a path handle from a DT node based on name 542 * @dev: device pointer for the consumer device 543 * @name: interconnect path name 544 * 545 * This function will search for a path between two endpoints and return an 546 * icc_path handle on success. Use icc_put() to release constraints when they 547 * are not needed anymore. 548 * If the interconnect API is disabled, NULL is returned and the consumer 549 * drivers will still build. Drivers are free to handle this specifically, 550 * but they don't have to. 551 * 552 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned 553 * when the API is disabled or the "interconnects" DT property is missing. 554 */ 555struct icc_path *of_icc_get(struct device *dev, const char *name) 556{ 557 struct device_node *np; 558 int idx = 0; 559 560 if (!dev || !dev->of_node) 561 return ERR_PTR(-ENODEV); 562 563 np = dev->of_node; 564 565 /* 566 * When the consumer DT node do not have "interconnects" property 567 * return a NULL path to skip setting constraints. 568 */ 569 if (!of_property_present(np, "interconnects")) 570 return NULL; 571 572 /* 573 * We use a combination of phandle and specifier for endpoint. For now 574 * lets support only global ids and extend this in the future if needed 575 * without breaking DT compatibility. 576 */ 577 if (name) { 578 idx = of_property_match_string(np, "interconnect-names", name); 579 if (idx < 0) 580 return ERR_PTR(idx); 581 } 582 583 return of_icc_get_by_index(dev, idx); 584} 585EXPORT_SYMBOL_GPL(of_icc_get); 586 587/** 588 * icc_get() - get a path handle between two endpoints 589 * @dev: device pointer for the consumer device 590 * @src: source node name 591 * @dst: destination node name 592 * 593 * This function will search for a path between two endpoints and return an 594 * icc_path handle on success. Use icc_put() to release constraints when they 595 * are not needed anymore. 596 * 597 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned 598 * when the API is disabled. 599 */ 600struct icc_path *icc_get(struct device *dev, const char *src, const char *dst) 601{ 602 struct icc_node *src_node, *dst_node; 603 struct icc_path *path = ERR_PTR(-EPROBE_DEFER); 604 605 mutex_lock(&icc_lock); 606 607 src_node = node_find_by_name(src); 608 if (!src_node) { 609 dev_err(dev, "%s: invalid src=%s\n", __func__, src); 610 goto out; 611 } 612 613 dst_node = node_find_by_name(dst); 614 if (!dst_node) { 615 dev_err(dev, "%s: invalid dst=%s\n", __func__, dst); 616 goto out; 617 } 618 619 path = path_find(dev, src_node, dst_node); 620 if (IS_ERR(path)) { 621 dev_err(dev, "%s: invalid path=%ld\n", __func__, PTR_ERR(path)); 622 goto out; 623 } 624 625 path->name = kasprintf(GFP_KERNEL, "%s-%s", src_node->name, dst_node->name); 626 if (!path->name) { 627 kfree(path); 628 path = ERR_PTR(-ENOMEM); 629 } 630out: 631 mutex_unlock(&icc_lock); 632 return path; 633} 634 635/** 636 * icc_set_tag() - set an optional tag on a path 637 * @path: the path we want to tag 638 * @tag: the tag value 639 * 640 * This function allows consumers to append a tag to the requests associated 641 * with a path, so that a different aggregation could be done based on this tag. 642 */ 643void icc_set_tag(struct icc_path *path, u32 tag) 644{ 645 int i; 646 647 if (!path) 648 return; 649 650 mutex_lock(&icc_lock); 651 652 for (i = 0; i < path->num_nodes; i++) 653 path->reqs[i].tag = tag; 654 655 mutex_unlock(&icc_lock); 656} 657EXPORT_SYMBOL_GPL(icc_set_tag); 658 659/** 660 * icc_get_name() - Get name of the icc path 661 * @path: interconnect path 662 * 663 * This function is used by an interconnect consumer to get the name of the icc 664 * path. 665 * 666 * Returns a valid pointer on success, or NULL otherwise. 667 */ 668const char *icc_get_name(struct icc_path *path) 669{ 670 if (!path) 671 return NULL; 672 673 return path->name; 674} 675EXPORT_SYMBOL_GPL(icc_get_name); 676 677/** 678 * icc_set_bw() - set bandwidth constraints on an interconnect path 679 * @path: interconnect path 680 * @avg_bw: average bandwidth in kilobytes per second 681 * @peak_bw: peak bandwidth in kilobytes per second 682 * 683 * This function is used by an interconnect consumer to express its own needs 684 * in terms of bandwidth for a previously requested path between two endpoints. 685 * The requests are aggregated and each node is updated accordingly. The entire 686 * path is locked by a mutex to ensure that the set() is completed. 687 * The @path can be NULL when the "interconnects" DT properties is missing, 688 * which will mean that no constraints will be set. 689 * 690 * Returns 0 on success, or an appropriate error code otherwise. 691 */ 692int icc_set_bw(struct icc_path *path, u32 avg_bw, u32 peak_bw) 693{ 694 struct icc_node *node; 695 u32 old_avg, old_peak; 696 size_t i; 697 int ret; 698 699 if (!path) 700 return 0; 701 702 if (WARN_ON(IS_ERR(path) || !path->num_nodes)) 703 return -EINVAL; 704 705 mutex_lock(&icc_bw_lock); 706 707 old_avg = path->reqs[0].avg_bw; 708 old_peak = path->reqs[0].peak_bw; 709 710 for (i = 0; i < path->num_nodes; i++) { 711 node = path->reqs[i].node; 712 713 /* update the consumer request for this path */ 714 path->reqs[i].avg_bw = avg_bw; 715 path->reqs[i].peak_bw = peak_bw; 716 717 /* aggregate requests for this node */ 718 aggregate_requests(node); 719 720 trace_icc_set_bw(path, node, i, avg_bw, peak_bw); 721 } 722 723 ret = apply_constraints(path); 724 if (ret) { 725 pr_debug("interconnect: error applying constraints (%d)\n", 726 ret); 727 728 for (i = 0; i < path->num_nodes; i++) { 729 node = path->reqs[i].node; 730 path->reqs[i].avg_bw = old_avg; 731 path->reqs[i].peak_bw = old_peak; 732 aggregate_requests(node); 733 } 734 apply_constraints(path); 735 } 736 737 mutex_unlock(&icc_bw_lock); 738 739 trace_icc_set_bw_end(path, ret); 740 741 return ret; 742} 743EXPORT_SYMBOL_GPL(icc_set_bw); 744 745static int __icc_enable(struct icc_path *path, bool enable) 746{ 747 int i; 748 749 if (!path) 750 return 0; 751 752 if (WARN_ON(IS_ERR(path) || !path->num_nodes)) 753 return -EINVAL; 754 755 mutex_lock(&icc_lock); 756 757 for (i = 0; i < path->num_nodes; i++) 758 path->reqs[i].enabled = enable; 759 760 mutex_unlock(&icc_lock); 761 762 return icc_set_bw(path, path->reqs[0].avg_bw, 763 path->reqs[0].peak_bw); 764} 765 766int icc_enable(struct icc_path *path) 767{ 768 return __icc_enable(path, true); 769} 770EXPORT_SYMBOL_GPL(icc_enable); 771 772int icc_disable(struct icc_path *path) 773{ 774 return __icc_enable(path, false); 775} 776EXPORT_SYMBOL_GPL(icc_disable); 777 778/** 779 * icc_put() - release the reference to the icc_path 780 * @path: interconnect path 781 * 782 * Use this function to release the constraints on a path when the path is 783 * no longer needed. The constraints will be re-aggregated. 784 */ 785void icc_put(struct icc_path *path) 786{ 787 struct icc_node *node; 788 size_t i; 789 int ret; 790 791 if (!path || WARN_ON(IS_ERR(path))) 792 return; 793 794 ret = icc_set_bw(path, 0, 0); 795 if (ret) 796 pr_err("%s: error (%d)\n", __func__, ret); 797 798 mutex_lock(&icc_lock); 799 mutex_lock(&icc_bw_lock); 800 801 for (i = 0; i < path->num_nodes; i++) { 802 node = path->reqs[i].node; 803 hlist_del(&path->reqs[i].req_node); 804 if (!WARN_ON(!node->provider->users)) 805 node->provider->users--; 806 } 807 808 mutex_unlock(&icc_bw_lock); 809 mutex_unlock(&icc_lock); 810 811 kfree_const(path->name); 812 kfree(path); 813} 814EXPORT_SYMBOL_GPL(icc_put); 815 816static struct icc_node *icc_node_create_nolock(int id) 817{ 818 struct icc_node *node; 819 820 /* check if node already exists */ 821 node = node_find(id); 822 if (node) 823 return node; 824 825 node = kzalloc(sizeof(*node), GFP_KERNEL); 826 if (!node) 827 return ERR_PTR(-ENOMEM); 828 829 id = idr_alloc(&icc_idr, node, id, id + 1, GFP_KERNEL); 830 if (id < 0) { 831 WARN(1, "%s: couldn't get idr\n", __func__); 832 kfree(node); 833 return ERR_PTR(id); 834 } 835 836 node->id = id; 837 838 return node; 839} 840 841/** 842 * icc_node_create() - create a node 843 * @id: node id 844 * 845 * Return: icc_node pointer on success, or ERR_PTR() on error 846 */ 847struct icc_node *icc_node_create(int id) 848{ 849 struct icc_node *node; 850 851 mutex_lock(&icc_lock); 852 853 node = icc_node_create_nolock(id); 854 855 mutex_unlock(&icc_lock); 856 857 return node; 858} 859EXPORT_SYMBOL_GPL(icc_node_create); 860 861/** 862 * icc_node_destroy() - destroy a node 863 * @id: node id 864 */ 865void icc_node_destroy(int id) 866{ 867 struct icc_node *node; 868 869 mutex_lock(&icc_lock); 870 871 node = node_find(id); 872 if (node) { 873 idr_remove(&icc_idr, node->id); 874 WARN_ON(!hlist_empty(&node->req_list)); 875 } 876 877 mutex_unlock(&icc_lock); 878 879 if (!node) 880 return; 881 882 kfree(node->links); 883 kfree(node); 884} 885EXPORT_SYMBOL_GPL(icc_node_destroy); 886 887/** 888 * icc_link_create() - create a link between two nodes 889 * @node: source node id 890 * @dst_id: destination node id 891 * 892 * Create a link between two nodes. The nodes might belong to different 893 * interconnect providers and the @dst_id node might not exist (if the 894 * provider driver has not probed yet). So just create the @dst_id node 895 * and when the actual provider driver is probed, the rest of the node 896 * data is filled. 897 * 898 * Return: 0 on success, or an error code otherwise 899 */ 900int icc_link_create(struct icc_node *node, const int dst_id) 901{ 902 struct icc_node *dst; 903 struct icc_node **new; 904 int ret = 0; 905 906 if (!node->provider) 907 return -EINVAL; 908 909 mutex_lock(&icc_lock); 910 911 dst = node_find(dst_id); 912 if (!dst) { 913 dst = icc_node_create_nolock(dst_id); 914 915 if (IS_ERR(dst)) { 916 ret = PTR_ERR(dst); 917 goto out; 918 } 919 } 920 921 new = krealloc(node->links, 922 (node->num_links + 1) * sizeof(*node->links), 923 GFP_KERNEL); 924 if (!new) { 925 ret = -ENOMEM; 926 goto out; 927 } 928 929 node->links = new; 930 node->links[node->num_links++] = dst; 931 932out: 933 mutex_unlock(&icc_lock); 934 935 return ret; 936} 937EXPORT_SYMBOL_GPL(icc_link_create); 938 939/** 940 * icc_node_add() - add interconnect node to interconnect provider 941 * @node: pointer to the interconnect node 942 * @provider: pointer to the interconnect provider 943 */ 944void icc_node_add(struct icc_node *node, struct icc_provider *provider) 945{ 946 if (WARN_ON(node->provider)) 947 return; 948 949 mutex_lock(&icc_lock); 950 mutex_lock(&icc_bw_lock); 951 952 node->provider = provider; 953 list_add_tail(&node->node_list, &provider->nodes); 954 955 /* get the initial bandwidth values and sync them with hardware */ 956 if (provider->get_bw) { 957 provider->get_bw(node, &node->init_avg, &node->init_peak); 958 } else { 959 node->init_avg = INT_MAX; 960 node->init_peak = INT_MAX; 961 } 962 node->avg_bw = node->init_avg; 963 node->peak_bw = node->init_peak; 964 965 if (node->avg_bw || node->peak_bw) { 966 if (provider->pre_aggregate) 967 provider->pre_aggregate(node); 968 969 if (provider->aggregate) 970 provider->aggregate(node, 0, node->init_avg, node->init_peak, 971 &node->avg_bw, &node->peak_bw); 972 if (provider->set) 973 provider->set(node, node); 974 } 975 976 node->avg_bw = 0; 977 node->peak_bw = 0; 978 979 mutex_unlock(&icc_bw_lock); 980 mutex_unlock(&icc_lock); 981} 982EXPORT_SYMBOL_GPL(icc_node_add); 983 984/** 985 * icc_node_del() - delete interconnect node from interconnect provider 986 * @node: pointer to the interconnect node 987 */ 988void icc_node_del(struct icc_node *node) 989{ 990 mutex_lock(&icc_lock); 991 992 list_del(&node->node_list); 993 994 mutex_unlock(&icc_lock); 995} 996EXPORT_SYMBOL_GPL(icc_node_del); 997 998/** 999 * icc_nodes_remove() - remove all previously added nodes from provider 1000 * @provider: the interconnect provider we are removing nodes from 1001 * 1002 * Return: 0 on success, or an error code otherwise 1003 */ 1004int icc_nodes_remove(struct icc_provider *provider) 1005{ 1006 struct icc_node *n, *tmp; 1007 1008 if (WARN_ON(IS_ERR_OR_NULL(provider))) 1009 return -EINVAL; 1010 1011 list_for_each_entry_safe_reverse(n, tmp, &provider->nodes, node_list) { 1012 icc_node_del(n); 1013 icc_node_destroy(n->id); 1014 } 1015 1016 return 0; 1017} 1018EXPORT_SYMBOL_GPL(icc_nodes_remove); 1019 1020/** 1021 * icc_provider_init() - initialize a new interconnect provider 1022 * @provider: the interconnect provider to initialize 1023 * 1024 * Must be called before adding nodes to the provider. 1025 */ 1026void icc_provider_init(struct icc_provider *provider) 1027{ 1028 WARN_ON(!provider->set); 1029 1030 INIT_LIST_HEAD(&provider->nodes); 1031} 1032EXPORT_SYMBOL_GPL(icc_provider_init); 1033 1034/** 1035 * icc_provider_register() - register a new interconnect provider 1036 * @provider: the interconnect provider to register 1037 * 1038 * Return: 0 on success, or an error code otherwise 1039 */ 1040int icc_provider_register(struct icc_provider *provider) 1041{ 1042 if (WARN_ON(!provider->xlate && !provider->xlate_extended)) 1043 return -EINVAL; 1044 1045 mutex_lock(&icc_lock); 1046 list_add_tail(&provider->provider_list, &icc_providers); 1047 mutex_unlock(&icc_lock); 1048 1049 dev_dbg(provider->dev, "interconnect provider registered\n"); 1050 1051 return 0; 1052} 1053EXPORT_SYMBOL_GPL(icc_provider_register); 1054 1055/** 1056 * icc_provider_deregister() - deregister an interconnect provider 1057 * @provider: the interconnect provider to deregister 1058 */ 1059void icc_provider_deregister(struct icc_provider *provider) 1060{ 1061 mutex_lock(&icc_lock); 1062 WARN_ON(provider->users); 1063 1064 list_del(&provider->provider_list); 1065 mutex_unlock(&icc_lock); 1066} 1067EXPORT_SYMBOL_GPL(icc_provider_deregister); 1068 1069static const struct of_device_id __maybe_unused ignore_list[] = { 1070 { .compatible = "qcom,sc7180-ipa-virt" }, 1071 { .compatible = "qcom,sc8180x-ipa-virt" }, 1072 { .compatible = "qcom,sdx55-ipa-virt" }, 1073 { .compatible = "qcom,sm8150-ipa-virt" }, 1074 { .compatible = "qcom,sm8250-ipa-virt" }, 1075 {} 1076}; 1077 1078static int of_count_icc_providers(struct device_node *np) 1079{ 1080 struct device_node *child; 1081 int count = 0; 1082 1083 for_each_available_child_of_node(np, child) { 1084 if (of_property_read_bool(child, "#interconnect-cells") && 1085 likely(!of_match_node(ignore_list, child))) 1086 count++; 1087 count += of_count_icc_providers(child); 1088 } 1089 1090 return count; 1091} 1092 1093void icc_sync_state(struct device *dev) 1094{ 1095 struct icc_provider *p; 1096 struct icc_node *n; 1097 static int count; 1098 1099 count++; 1100 1101 if (count < providers_count) 1102 return; 1103 1104 mutex_lock(&icc_lock); 1105 mutex_lock(&icc_bw_lock); 1106 synced_state = true; 1107 list_for_each_entry(p, &icc_providers, provider_list) { 1108 dev_dbg(p->dev, "interconnect provider is in synced state\n"); 1109 list_for_each_entry(n, &p->nodes, node_list) { 1110 if (n->init_avg || n->init_peak) { 1111 n->init_avg = 0; 1112 n->init_peak = 0; 1113 aggregate_requests(n); 1114 p->set(n, n); 1115 } 1116 } 1117 } 1118 mutex_unlock(&icc_bw_lock); 1119 mutex_unlock(&icc_lock); 1120} 1121EXPORT_SYMBOL_GPL(icc_sync_state); 1122 1123static int __init icc_init(void) 1124{ 1125 struct device_node *root; 1126 1127 /* Teach lockdep about lock ordering wrt. shrinker: */ 1128 fs_reclaim_acquire(GFP_KERNEL); 1129 might_lock(&icc_bw_lock); 1130 fs_reclaim_release(GFP_KERNEL); 1131 1132 root = of_find_node_by_path("/"); 1133 1134 providers_count = of_count_icc_providers(root); 1135 of_node_put(root); 1136 1137 icc_debugfs_dir = debugfs_create_dir("interconnect", NULL); 1138 debugfs_create_file("interconnect_summary", 0444, 1139 icc_debugfs_dir, NULL, &icc_summary_fops); 1140 debugfs_create_file("interconnect_graph", 0444, 1141 icc_debugfs_dir, NULL, &icc_graph_fops); 1142 1143 icc_debugfs_client_init(icc_debugfs_dir); 1144 1145 return 0; 1146} 1147 1148device_initcall(icc_init); 1149