1/* 2 * (C) Copyright 2008 Intel Corporation 3 * Authors: 4 * Andy Henroid <andrew.d.henroid@intel.com> 5 * Venkatesh Pallipadi <venkatesh.pallipadi@intel.com> 6 */ 7 8/* 9 * Save DIMM power on Intel 7300-based platforms when all CPUs/cores 10 * are idle, using the DIMM thermal throttling capability. 11 * 12 * This driver depends on the Intel integrated DMA controller (I/O AT). 13 * If the driver for I/O AT (drivers/dma/ioatdma*) is also enabled, 14 * this driver should work cooperatively. 15 */ 16 17/* #define DEBUG */ 18 19#include <linux/module.h> 20#include <linux/pci.h> 21#include <linux/gfp.h> 22#include <linux/sched.h> 23#include <linux/notifier.h> 24#include <linux/cpumask.h> 25#include <linux/ktime.h> 26#include <linux/delay.h> 27#include <linux/debugfs.h> 28#include <linux/stop_machine.h> 29#include <linux/i7300_idle.h> 30 31#include <asm/idle.h> 32 33#include "../dma/ioat/hw.h" 34#include "../dma/ioat/registers.h" 35 36#define I7300_IDLE_DRIVER_VERSION "1.55" 37#define I7300_PRINT "i7300_idle:" 38 39#define MAX_STOP_RETRIES 10 40 41static int debug; 42module_param_named(debug, debug, uint, 0644); 43MODULE_PARM_DESC(debug, "Enable debug printks in this driver"); 44 45static int forceload; 46module_param_named(forceload, forceload, uint, 0644); 47MODULE_PARM_DESC(debug, "Enable driver testing on unvalidated i5000"); 48 49#define dprintk(fmt, arg...) \ 50 do { if (debug) printk(KERN_INFO I7300_PRINT fmt, ##arg); } while (0) 51 52/* 53 * Value to set THRTLOW to when initiating throttling 54 * 0 = No throttling 55 * 1 = Throttle when > 4 activations per eval window (Maximum throttling) 56 * 2 = Throttle when > 8 activations 57 * 168 = Throttle when > 672 activations (Minimum throttling) 58 */ 59#define MAX_THROTTLE_LOW_LIMIT 168 60static uint throttle_low_limit = 1; 61module_param_named(throttle_low_limit, throttle_low_limit, uint, 0644); 62MODULE_PARM_DESC(throttle_low_limit, 63 "Value for THRTLOWLM activation field " 64 "(0 = disable throttle, 1 = Max throttle, 168 = Min throttle)"); 65 66/* 67 * simple invocation and duration statistics 68 */ 69static unsigned long total_starts; 70static unsigned long total_us; 71 72#ifdef DEBUG 73static unsigned long past_skip; 74#endif 75 76static struct pci_dev *fbd_dev; 77 78static spinlock_t i7300_idle_lock; 79static int i7300_idle_active; 80 81static u8 i7300_idle_thrtctl_saved; 82static u8 i7300_idle_thrtlow_saved; 83static u32 i7300_idle_mc_saved; 84 85static cpumask_var_t idle_cpumask; 86static ktime_t start_ktime; 87static unsigned long avg_idle_us; 88 89static struct dentry *debugfs_dir; 90 91/* Begin: I/O AT Helper routines */ 92 93#define IOAT_CHANBASE(ioat_ctl, chan) (ioat_ctl + 0x80 + 0x80 * chan) 94/* Snoop control (disable snoops when coherency is not important) */ 95#define IOAT_DESC_SADDR_SNP_CTL (1UL << 1) 96#define IOAT_DESC_DADDR_SNP_CTL (1UL << 2) 97 98static struct pci_dev *ioat_dev; 99static struct ioat_dma_descriptor *ioat_desc; /* I/O AT desc & data (1 page) */ 100static unsigned long ioat_desc_phys; 101static u8 *ioat_iomap; /* I/O AT memory-mapped control regs (aka CB_BAR) */ 102static u8 *ioat_chanbase; 103 104/* Start I/O AT memory copy */ 105static int i7300_idle_ioat_start(void) 106{ 107 u32 err; 108 /* Clear error (due to circular descriptor pointer) */ 109 err = readl(ioat_chanbase + IOAT_CHANERR_OFFSET); 110 if (err) 111 writel(err, ioat_chanbase + IOAT_CHANERR_OFFSET); 112 113 writeb(IOAT_CHANCMD_START, ioat_chanbase + IOAT1_CHANCMD_OFFSET); 114 return 0; 115} 116 117/* Stop I/O AT memory copy */ 118static void i7300_idle_ioat_stop(void) 119{ 120 int i; 121 u64 sts; 122 123 for (i = 0; i < MAX_STOP_RETRIES; i++) { 124 writeb(IOAT_CHANCMD_RESET, 125 ioat_chanbase + IOAT1_CHANCMD_OFFSET); 126 127 udelay(10); 128 129 sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) & 130 IOAT_CHANSTS_STATUS; 131 132 if (sts != IOAT_CHANSTS_ACTIVE) 133 break; 134 135 } 136 137 if (i == MAX_STOP_RETRIES) { 138 dprintk("failed to stop I/O AT after %d retries\n", 139 MAX_STOP_RETRIES); 140 } 141} 142 143/* Test I/O AT by copying 1024 byte from 2k to 1k */ 144static int __init i7300_idle_ioat_selftest(u8 *ctl, 145 struct ioat_dma_descriptor *desc, unsigned long desc_phys) 146{ 147 u64 chan_sts; 148 149 memset(desc, 0, 2048); 150 memset((u8 *) desc + 2048, 0xab, 1024); 151 152 desc[0].size = 1024; 153 desc[0].ctl = 0; 154 desc[0].src_addr = desc_phys + 2048; 155 desc[0].dst_addr = desc_phys + 1024; 156 desc[0].next = 0; 157 158 writeb(IOAT_CHANCMD_RESET, ioat_chanbase + IOAT1_CHANCMD_OFFSET); 159 writeb(IOAT_CHANCMD_START, ioat_chanbase + IOAT1_CHANCMD_OFFSET); 160 161 udelay(1000); 162 163 chan_sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) & 164 IOAT_CHANSTS_STATUS; 165 166 if (chan_sts != IOAT_CHANSTS_DONE) { 167 /* Not complete, reset the channel */ 168 writeb(IOAT_CHANCMD_RESET, 169 ioat_chanbase + IOAT1_CHANCMD_OFFSET); 170 return -1; 171 } 172 173 if (*(u32 *) ((u8 *) desc + 3068) != 0xabababab || 174 *(u32 *) ((u8 *) desc + 2044) != 0xabababab) { 175 dprintk("Data values src 0x%x, dest 0x%x, memset 0x%x\n", 176 *(u32 *) ((u8 *) desc + 2048), 177 *(u32 *) ((u8 *) desc + 1024), 178 *(u32 *) ((u8 *) desc + 3072)); 179 return -1; 180 } 181 return 0; 182} 183 184static struct device dummy_dma_dev = { 185 .init_name = "fallback device", 186 .coherent_dma_mask = DMA_BIT_MASK(64), 187 .dma_mask = &dummy_dma_dev.coherent_dma_mask, 188}; 189 190/* Setup and initialize I/O AT */ 191/* This driver needs I/O AT as the throttling takes effect only when there is 192 * some memory activity. We use I/O AT to set up a dummy copy, while all CPUs 193 * go idle and memory is throttled. 194 */ 195static int __init i7300_idle_ioat_init(void) 196{ 197 u8 ver, chan_count, ioat_chan; 198 u16 chan_ctl; 199 200 ioat_iomap = (u8 *) ioremap_nocache(pci_resource_start(ioat_dev, 0), 201 pci_resource_len(ioat_dev, 0)); 202 203 if (!ioat_iomap) { 204 printk(KERN_ERR I7300_PRINT "failed to map I/O AT registers\n"); 205 goto err_ret; 206 } 207 208 ver = readb(ioat_iomap + IOAT_VER_OFFSET); 209 if (ver != IOAT_VER_1_2) { 210 printk(KERN_ERR I7300_PRINT "unknown I/O AT version (%u.%u)\n", 211 ver >> 4, ver & 0xf); 212 goto err_unmap; 213 } 214 215 chan_count = readb(ioat_iomap + IOAT_CHANCNT_OFFSET); 216 if (!chan_count) { 217 printk(KERN_ERR I7300_PRINT "unexpected # of I/O AT channels " 218 "(%u)\n", 219 chan_count); 220 goto err_unmap; 221 } 222 223 ioat_chan = chan_count - 1; 224 ioat_chanbase = IOAT_CHANBASE(ioat_iomap, ioat_chan); 225 226 chan_ctl = readw(ioat_chanbase + IOAT_CHANCTRL_OFFSET); 227 if (chan_ctl & IOAT_CHANCTRL_CHANNEL_IN_USE) { 228 printk(KERN_ERR I7300_PRINT "channel %d in use\n", ioat_chan); 229 goto err_unmap; 230 } 231 232 writew(IOAT_CHANCTRL_CHANNEL_IN_USE, 233 ioat_chanbase + IOAT_CHANCTRL_OFFSET); 234 235 ioat_desc = (struct ioat_dma_descriptor *)dma_alloc_coherent( 236 &dummy_dma_dev, 4096, 237 (dma_addr_t *)&ioat_desc_phys, GFP_KERNEL); 238 if (!ioat_desc) { 239 printk(KERN_ERR I7300_PRINT "failed to allocate I/O AT desc\n"); 240 goto err_mark_unused; 241 } 242 243 writel(ioat_desc_phys & 0xffffffffUL, 244 ioat_chanbase + IOAT1_CHAINADDR_OFFSET_LOW); 245 writel(ioat_desc_phys >> 32, 246 ioat_chanbase + IOAT1_CHAINADDR_OFFSET_HIGH); 247 248 if (i7300_idle_ioat_selftest(ioat_iomap, ioat_desc, ioat_desc_phys)) { 249 printk(KERN_ERR I7300_PRINT "I/O AT self-test failed\n"); 250 goto err_free; 251 } 252 253 /* Setup circular I/O AT descriptor chain */ 254 ioat_desc[0].ctl = IOAT_DESC_SADDR_SNP_CTL | IOAT_DESC_DADDR_SNP_CTL; 255 ioat_desc[0].src_addr = ioat_desc_phys + 2048; 256 ioat_desc[0].dst_addr = ioat_desc_phys + 3072; 257 ioat_desc[0].size = 128; 258 ioat_desc[0].next = ioat_desc_phys + sizeof(struct ioat_dma_descriptor); 259 260 ioat_desc[1].ctl = ioat_desc[0].ctl; 261 ioat_desc[1].src_addr = ioat_desc[0].src_addr; 262 ioat_desc[1].dst_addr = ioat_desc[0].dst_addr; 263 ioat_desc[1].size = ioat_desc[0].size; 264 ioat_desc[1].next = ioat_desc_phys; 265 266 return 0; 267 268err_free: 269 dma_free_coherent(&dummy_dma_dev, 4096, (void *)ioat_desc, 0); 270err_mark_unused: 271 writew(0, ioat_chanbase + IOAT_CHANCTRL_OFFSET); 272err_unmap: 273 iounmap(ioat_iomap); 274err_ret: 275 return -ENODEV; 276} 277 278/* Cleanup I/O AT */ 279static void __exit i7300_idle_ioat_exit(void) 280{ 281 int i; 282 u64 chan_sts; 283 284 i7300_idle_ioat_stop(); 285 286 /* Wait for a while for the channel to halt before releasing */ 287 for (i = 0; i < MAX_STOP_RETRIES; i++) { 288 writeb(IOAT_CHANCMD_RESET, 289 ioat_chanbase + IOAT1_CHANCMD_OFFSET); 290 291 chan_sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) & 292 IOAT_CHANSTS_STATUS; 293 294 if (chan_sts != IOAT_CHANSTS_ACTIVE) { 295 writew(0, ioat_chanbase + IOAT_CHANCTRL_OFFSET); 296 break; 297 } 298 udelay(1000); 299 } 300 301 chan_sts = readq(ioat_chanbase + IOAT1_CHANSTS_OFFSET) & 302 IOAT_CHANSTS_STATUS; 303 304 /* 305 * We tried to reset multiple times. If IO A/T channel is still active 306 * flag an error and return without cleanup. Memory leak is better 307 * than random corruption in that extreme error situation. 308 */ 309 if (chan_sts == IOAT_CHANSTS_ACTIVE) { 310 printk(KERN_ERR I7300_PRINT "Unable to stop IO A/T channels." 311 " Not freeing resources\n"); 312 return; 313 } 314 315 dma_free_coherent(&dummy_dma_dev, 4096, (void *)ioat_desc, 0); 316 iounmap(ioat_iomap); 317} 318 319/* End: I/O AT Helper routines */ 320 321#define DIMM_THRTLOW 0x64 322#define DIMM_THRTCTL 0x67 323#define DIMM_THRTCTL_THRMHUNT (1UL << 0) 324#define DIMM_MC 0x40 325#define DIMM_GTW_MODE (1UL << 17) 326#define DIMM_GBLACT 0x60 327 328/* 329 * Keep track of an exponential-decaying average of recent idle durations. 330 * The latest duration gets DURATION_WEIGHT_PCT percentage weight 331 * in this average, with the old average getting the remaining weight. 332 * 333 * High weights emphasize recent history, low weights include long history. 334 */ 335#define DURATION_WEIGHT_PCT 55 336 337/* 338 * When the decaying average of recent durations or the predicted duration 339 * of the next timer interrupt is shorter than duration_threshold, the 340 * driver will decline to throttle. 341 */ 342#define DURATION_THRESHOLD_US 100 343 344 345/* Store DIMM thermal throttle configuration */ 346static int i7300_idle_thrt_save(void) 347{ 348 u32 new_mc_val; 349 u8 gblactlm; 350 351 pci_read_config_byte(fbd_dev, DIMM_THRTCTL, &i7300_idle_thrtctl_saved); 352 pci_read_config_byte(fbd_dev, DIMM_THRTLOW, &i7300_idle_thrtlow_saved); 353 pci_read_config_dword(fbd_dev, DIMM_MC, &i7300_idle_mc_saved); 354 /* 355 * Make sure we have Global Throttling Window Mode set to have a 356 * "short" window. This (mostly) works around an issue where 357 * throttling persists until the end of the global throttling window 358 * size. On the tested system, this was resulting in a maximum of 359 * 64 ms to exit throttling (average 32 ms). The actual numbers 360 * depends on system frequencies. Setting the short window reduces 361 * this by a factor of 4096. 362 * 363 * We will only do this only if the system is set for 364 * unlimited-activations while in open-loop throttling (i.e., when 365 * Global Activation Throttle Limit is zero). 366 */ 367 pci_read_config_byte(fbd_dev, DIMM_GBLACT, &gblactlm); 368 dprintk("thrtctl_saved = 0x%02x, thrtlow_saved = 0x%02x\n", 369 i7300_idle_thrtctl_saved, 370 i7300_idle_thrtlow_saved); 371 dprintk("mc_saved = 0x%08x, gblactlm = 0x%02x\n", 372 i7300_idle_mc_saved, 373 gblactlm); 374 if (gblactlm == 0) { 375 new_mc_val = i7300_idle_mc_saved | DIMM_GTW_MODE; 376 pci_write_config_dword(fbd_dev, DIMM_MC, new_mc_val); 377 return 0; 378 } else { 379 dprintk("could not set GTW_MODE = 1 (OLTT enabled)\n"); 380 return -ENODEV; 381 } 382} 383 384/* Restore DIMM thermal throttle configuration */ 385static void i7300_idle_thrt_restore(void) 386{ 387 pci_write_config_dword(fbd_dev, DIMM_MC, i7300_idle_mc_saved); 388 pci_write_config_byte(fbd_dev, DIMM_THRTLOW, i7300_idle_thrtlow_saved); 389 pci_write_config_byte(fbd_dev, DIMM_THRTCTL, i7300_idle_thrtctl_saved); 390} 391 392/* Enable DIMM thermal throttling */ 393static void i7300_idle_start(void) 394{ 395 u8 new_ctl; 396 u8 limit; 397 398 new_ctl = i7300_idle_thrtctl_saved & ~DIMM_THRTCTL_THRMHUNT; 399 pci_write_config_byte(fbd_dev, DIMM_THRTCTL, new_ctl); 400 401 limit = throttle_low_limit; 402 if (unlikely(limit > MAX_THROTTLE_LOW_LIMIT)) 403 limit = MAX_THROTTLE_LOW_LIMIT; 404 405 pci_write_config_byte(fbd_dev, DIMM_THRTLOW, limit); 406 407 new_ctl = i7300_idle_thrtctl_saved | DIMM_THRTCTL_THRMHUNT; 408 pci_write_config_byte(fbd_dev, DIMM_THRTCTL, new_ctl); 409} 410 411/* Disable DIMM thermal throttling */ 412static void i7300_idle_stop(void) 413{ 414 u8 new_ctl; 415 u8 got_ctl; 416 417 new_ctl = i7300_idle_thrtctl_saved & ~DIMM_THRTCTL_THRMHUNT; 418 pci_write_config_byte(fbd_dev, DIMM_THRTCTL, new_ctl); 419 420 pci_write_config_byte(fbd_dev, DIMM_THRTLOW, i7300_idle_thrtlow_saved); 421 pci_write_config_byte(fbd_dev, DIMM_THRTCTL, i7300_idle_thrtctl_saved); 422 pci_read_config_byte(fbd_dev, DIMM_THRTCTL, &got_ctl); 423 WARN_ON_ONCE(got_ctl != i7300_idle_thrtctl_saved); 424} 425 426 427/* 428 * i7300_avg_duration_check() 429 * return 0 if the decaying average of recent idle durations is 430 * more than DURATION_THRESHOLD_US 431 */ 432static int i7300_avg_duration_check(void) 433{ 434 if (avg_idle_us >= DURATION_THRESHOLD_US) 435 return 0; 436 437#ifdef DEBUG 438 past_skip++; 439#endif 440 return 1; 441} 442 443/* Idle notifier to look at idle CPUs */ 444static int i7300_idle_notifier(struct notifier_block *nb, unsigned long val, 445 void *data) 446{ 447 unsigned long flags; 448 ktime_t now_ktime; 449 static ktime_t idle_begin_time; 450 static int time_init = 1; 451 452 if (!throttle_low_limit) 453 return 0; 454 455 if (unlikely(time_init)) { 456 time_init = 0; 457 idle_begin_time = ktime_get(); 458 } 459 460 spin_lock_irqsave(&i7300_idle_lock, flags); 461 if (val == IDLE_START) { 462 463 cpumask_set_cpu(smp_processor_id(), idle_cpumask); 464 465 if (cpumask_weight(idle_cpumask) != num_online_cpus()) 466 goto end; 467 468 now_ktime = ktime_get(); 469 idle_begin_time = now_ktime; 470 471 if (i7300_avg_duration_check()) 472 goto end; 473 474 i7300_idle_active = 1; 475 total_starts++; 476 start_ktime = now_ktime; 477 478 i7300_idle_start(); 479 i7300_idle_ioat_start(); 480 481 } else if (val == IDLE_END) { 482 cpumask_clear_cpu(smp_processor_id(), idle_cpumask); 483 if (cpumask_weight(idle_cpumask) == (num_online_cpus() - 1)) { 484 /* First CPU coming out of idle */ 485 u64 idle_duration_us; 486 487 now_ktime = ktime_get(); 488 489 idle_duration_us = ktime_to_us(ktime_sub 490 (now_ktime, idle_begin_time)); 491 492 avg_idle_us = 493 ((100 - DURATION_WEIGHT_PCT) * avg_idle_us + 494 DURATION_WEIGHT_PCT * idle_duration_us) / 100; 495 496 if (i7300_idle_active) { 497 ktime_t idle_ktime; 498 499 idle_ktime = ktime_sub(now_ktime, start_ktime); 500 total_us += ktime_to_us(idle_ktime); 501 502 i7300_idle_ioat_stop(); 503 i7300_idle_stop(); 504 i7300_idle_active = 0; 505 } 506 } 507 } 508end: 509 spin_unlock_irqrestore(&i7300_idle_lock, flags); 510 return 0; 511} 512 513static struct notifier_block i7300_idle_nb = { 514 .notifier_call = i7300_idle_notifier, 515}; 516 517MODULE_DEVICE_TABLE(pci, pci_tbl); 518 519int stats_open_generic(struct inode *inode, struct file *fp) 520{ 521 fp->private_data = inode->i_private; 522 return 0; 523} 524 525static ssize_t stats_read_ul(struct file *fp, char __user *ubuf, size_t count, 526 loff_t *off) 527{ 528 unsigned long *p = fp->private_data; 529 char buf[32]; 530 int len; 531 532 len = snprintf(buf, 32, "%lu\n", *p); 533 return simple_read_from_buffer(ubuf, count, off, buf, len); 534} 535 536static const struct file_operations idle_fops = { 537 .open = stats_open_generic, 538 .read = stats_read_ul, 539}; 540 541struct debugfs_file_info { 542 void *ptr; 543 char name[32]; 544 struct dentry *file; 545} debugfs_file_list[] = { 546 {&total_starts, "total_starts", NULL}, 547 {&total_us, "total_us", NULL}, 548#ifdef DEBUG 549 {&past_skip, "past_skip", NULL}, 550#endif 551 {NULL, "", NULL} 552 }; 553 554static int __init i7300_idle_init(void) 555{ 556 spin_lock_init(&i7300_idle_lock); 557 total_us = 0; 558 559 if (i7300_idle_platform_probe(&fbd_dev, &ioat_dev, forceload)) 560 return -ENODEV; 561 562 if (i7300_idle_thrt_save()) 563 return -ENODEV; 564 565 if (i7300_idle_ioat_init()) 566 return -ENODEV; 567 568 if (!zalloc_cpumask_var(&idle_cpumask, GFP_KERNEL)) 569 return -ENOMEM; 570 571 debugfs_dir = debugfs_create_dir("i7300_idle", NULL); 572 if (debugfs_dir) { 573 int i = 0; 574 575 while (debugfs_file_list[i].ptr != NULL) { 576 debugfs_file_list[i].file = debugfs_create_file( 577 debugfs_file_list[i].name, 578 S_IRUSR, 579 debugfs_dir, 580 debugfs_file_list[i].ptr, 581 &idle_fops); 582 i++; 583 } 584 } 585 586 idle_notifier_register(&i7300_idle_nb); 587 588 printk(KERN_INFO "i7300_idle: loaded v%s\n", I7300_IDLE_DRIVER_VERSION); 589 return 0; 590} 591 592static void __exit i7300_idle_exit(void) 593{ 594 idle_notifier_unregister(&i7300_idle_nb); 595 free_cpumask_var(idle_cpumask); 596 597 if (debugfs_dir) { 598 int i = 0; 599 600 while (debugfs_file_list[i].file != NULL) { 601 debugfs_remove(debugfs_file_list[i].file); 602 i++; 603 } 604 605 debugfs_remove(debugfs_dir); 606 } 607 i7300_idle_thrt_restore(); 608 i7300_idle_ioat_exit(); 609} 610 611module_init(i7300_idle_init); 612module_exit(i7300_idle_exit); 613 614MODULE_AUTHOR("Andy Henroid <andrew.d.henroid@intel.com>"); 615MODULE_DESCRIPTION("Intel Chipset DIMM Idle Power Saving Driver v" 616 I7300_IDLE_DRIVER_VERSION); 617MODULE_LICENSE("GPL"); 618