1/* $NetBSD: intel_gtt_subr.c,v 1.3 2021/12/19 12:37:36 riastradh Exp $ */ 2 3/*- 4 * Copyright (c) 2014 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Taylor R. Campbell. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32/* Intel GTT stubs */ 33 34#include <sys/cdefs.h> 35__KERNEL_RCSID(0, "$NetBSD: intel_gtt_subr.c,v 1.3 2021/12/19 12:37:36 riastradh Exp $"); 36 37#include <sys/types.h> 38#include <sys/bus.h> 39#include <sys/errno.h> 40#include <sys/systm.h> 41 42#include <machine/vmparam.h> 43 44#include <dev/pci/pcivar.h> /* XXX agpvar.h needs... */ 45#include <dev/pci/agpvar.h> 46#include <dev/pci/agp_i810var.h> 47 48#include <linux/pci.h> 49#include <linux/scatterlist.h> 50 51#include "drm/i915_drm.h" 52#include "drm/intel-gtt.h" 53 54static uint8_t 55pci_conf_read8(pci_chipset_tag_t pc, pcitag_t tag, bus_size_t reg) 56{ 57 uint32_t v; 58 59 v = pci_conf_read(pc, tag, reg & ~3); 60 61 return 0xff & (v >> (8 * (reg & 3))); 62} 63 64static uint8_t 65pci_read8(pci_chipset_tag_t pc, int bus, int dev, int func, bus_size_t reg) 66{ 67 pcitag_t tag = pci_make_tag(pc, bus, dev, func); 68 69 return pci_conf_read8(pc, tag, reg); 70} 71 72static uint16_t 73pci_conf_read16(pci_chipset_tag_t pc, pcitag_t tag, bus_size_t reg) 74{ 75 uint32_t v; 76 77 KASSERT((reg & 1) == 0); 78 79 v = pci_conf_read(pc, tag, reg & ~2); 80 81 return 0xffff & (v >> (8 * (reg & 2))); 82} 83 84static uint16_t 85pci_read16(pci_chipset_tag_t pc, int bus, int dev, int func, bus_size_t reg) 86{ 87 pcitag_t tag = pci_make_tag(pc, bus, dev, func); 88 89 return pci_conf_read16(pc, tag, reg); 90} 91 92/* Access to this should be single-threaded. */ 93static struct { 94 bus_dma_segment_t scratch_seg; 95 bus_dmamap_t scratch_map; 96} intel_gtt; 97 98/* XXX This logic should be merged with agp_i810.c. */ 99struct resource intel_graphics_stolen_res; 100 101static bus_size_t 102i830_tseg_size(pci_chipset_tag_t pc) 103{ 104 uint8_t esmramc = pci_read8(pc, 0, 0, 0, I830_ESMRAMC); 105 106 if ((esmramc & TSEG_ENABLE) == 0) 107 return 0; 108 109 return (esmramc & I830_TSEG_SIZE_1M) ? 1024*1024 : 512*1024; 110} 111 112static bus_size_t 113i845_tseg_size(pci_chipset_tag_t pc) 114{ 115 uint8_t esmramc = pci_read8(pc, 0, 0, 0, I845_ESMRAMC); 116 117 if ((esmramc & TSEG_ENABLE) == 0) 118 return 0; 119 120 switch (esmramc & I845_TSEG_SIZE_MASK) { 121 case I845_TSEG_SIZE_512K: 122 return 512*1024; 123 case I845_TSEG_SIZE_1M: 124 return 1024*1024; 125 default: 126 return 0; 127 } 128} 129 130static bus_size_t 131i85x_tseg_size(pci_chipset_tag_t pc) 132{ 133 uint8_t esmramc = pci_read8(pc, 0, 0, 0, I85X_ESMRAMC); 134 135 if ((esmramc & TSEG_ENABLE) == 0) 136 return 0; 137 138 return 1024*1024; 139} 140 141static bus_size_t 142i830_tom(pci_chipset_tag_t pc) 143{ 144 uint8_t drb3 = pci_read8(pc, 0, 0, 0, I830_DRB3); 145 146 return (bus_size_t)32*1024*1024 * drb3; 147} 148 149static bus_size_t 150i85x_tom(pci_chipset_tag_t pc) 151{ 152 uint8_t drb3 = pci_read8(pc, 0, 0, 1, I85X_DRB3); 153 154 return (bus_size_t)32*1024*1024 * drb3; 155} 156 157static bus_size_t 158i830_stolen_size(pci_chipset_tag_t pc, pcitag_t tag) 159{ 160 uint16_t gmch_ctrl = pci_read16(pc, 0, 0, 0, I830_GMCH_CTRL); 161 162 switch (gmch_ctrl & I830_GMCH_GMS_MASK) { 163 case I830_GMCH_GMS_STOLEN_512: 164 return 512*1024; 165 case I830_GMCH_GMS_STOLEN_1024: 166 return 1024*1024; 167 case I830_GMCH_GMS_STOLEN_8192: 168 return 8*1024*1024; 169 case I830_GMCH_GMS_LOCAL: 170 default: 171 aprint_error("%s: invalid gmch_ctrl 0x%04x\n", __func__, 172 gmch_ctrl); 173 return 0; 174 } 175} 176 177static bus_size_t 178gen3_stolen_size(pci_chipset_tag_t pc, pcitag_t tag) 179{ 180 uint16_t gmch_ctrl = pci_read16(pc, 0, 0, 0, I830_GMCH_CTRL); 181 182 switch (gmch_ctrl & I855_GMCH_GMS_MASK) { 183 case I855_GMCH_GMS_STOLEN_1M: 184 return 1024*1024; 185 case I855_GMCH_GMS_STOLEN_4M: 186 return 4*1024*1024; 187 case I855_GMCH_GMS_STOLEN_8M: 188 return 8*1024*1024; 189 case I855_GMCH_GMS_STOLEN_16M: 190 return 16*1024*1024; 191 case I855_GMCH_GMS_STOLEN_32M: 192 return 32*1024*1024; 193 case I915_GMCH_GMS_STOLEN_48M: 194 return 48*1024*1024; 195 case I915_GMCH_GMS_STOLEN_64M: 196 return 64*1024*1024; 197 case G33_GMCH_GMS_STOLEN_128M: 198 return 128*1024*1024; 199 case G33_GMCH_GMS_STOLEN_256M: 200 return 256*1024*1024; 201 case INTEL_GMCH_GMS_STOLEN_96M: 202 return 96*1024*1024; 203 case INTEL_GMCH_GMS_STOLEN_160M: 204 return 160*1024*1024; 205 case INTEL_GMCH_GMS_STOLEN_224M: 206 return 224*1024*1024; 207 case INTEL_GMCH_GMS_STOLEN_352M: 208 return 352*1024*1024; 209 default: 210 aprint_error("%s: invalid gmch_ctrl 0x%04x\n", __func__, 211 gmch_ctrl); 212 return 0; 213 } 214} 215 216static bus_size_t 217gen6_stolen_size(pci_chipset_tag_t pc, pcitag_t tag) 218{ 219 uint16_t gmch_ctrl = pci_conf_read16(pc, tag, SNB_GMCH_CTRL); 220 uint16_t gms = (gmch_ctrl >> SNB_GMCH_GMS_SHIFT) & SNB_GMCH_GMS_MASK; 221 222 return (bus_size_t)32*1024*1024 * gms; 223} 224 225static bus_size_t 226gen8_stolen_size(pci_chipset_tag_t pc, pcitag_t tag) 227{ 228 uint16_t gmch_ctrl = pci_conf_read16(pc, tag, SNB_GMCH_CTRL); 229 uint16_t gms = (gmch_ctrl >> BDW_GMCH_GMS_SHIFT) & BDW_GMCH_GMS_MASK; 230 231 return (bus_size_t)32*1024*1024 * gms; 232} 233 234static bus_size_t 235chv_stolen_size(pci_chipset_tag_t pc, pcitag_t tag) 236{ 237 uint16_t gmch_ctrl = pci_conf_read16(pc, tag, SNB_GMCH_CTRL); 238 uint16_t gms = (gmch_ctrl >> SNB_GMCH_GMS_SHIFT) & SNB_GMCH_GMS_MASK; 239 240 if (gms <= 0x10) 241 return (bus_size_t)32*1024*1024 * gms; 242 else if (gms <= 0x16) 243 return (bus_size_t)(8 + 4*(gms - 0x11))*1024*1024; 244 else 245 return (bus_size_t)(36 + 4*(gms - 0x17))*1024*1024; 246} 247 248static bus_size_t 249gen9_stolen_size(pci_chipset_tag_t pc, pcitag_t tag) 250{ 251 uint16_t gmch_ctrl = pci_conf_read16(pc, tag, SNB_GMCH_CTRL); 252 uint16_t gms = (gmch_ctrl >> BDW_GMCH_GMS_SHIFT) & BDW_GMCH_GMS_MASK; 253 254 if (gms <= 0xef) 255 return (bus_size_t)32*1024*1024 * gms; 256 else 257 return (bus_size_t)(4 + 4*(gms - 0xf0))*1024*1024; 258} 259 260static bus_addr_t 261i830_stolen_base(pci_chipset_tag_t pc, pcitag_t tag, bus_size_t stolen_size) 262{ 263 264 return i830_tom(pc) - i830_tseg_size(pc) - stolen_size; 265} 266 267static bus_addr_t 268i845_stolen_base(pci_chipset_tag_t pc, pcitag_t tag, bus_size_t stolen_size) 269{ 270 271 return i830_tom(pc) - i845_tseg_size(pc) - stolen_size; 272} 273 274static bus_addr_t 275i85x_stolen_base(pci_chipset_tag_t pc, pcitag_t tag, bus_size_t stolen_size) 276{ 277 278 return i85x_tom(pc) - i85x_tseg_size(pc) - stolen_size; 279} 280 281static bus_addr_t 282i865_stolen_base(pci_chipset_tag_t pc, pcitag_t tag, bus_size_t stolen_size) 283{ 284 uint16_t toud = pci_read16(pc, 0, 0, 0, I865_TOUD); 285 286 return i845_tseg_size(pc) + (64*1024 * toud); 287} 288 289static bus_addr_t 290gen3_stolen_base(pci_chipset_tag_t pc, pcitag_t tag, bus_size_t stolen_size) 291{ 292 uint32_t bsm = pci_conf_read(pc, tag, INTEL_BSM); 293 294 return bsm & INTEL_BSM_MASK; 295} 296 297static bus_addr_t 298gen11_stolen_base(pci_chipset_tag_t pc, pcitag_t tag, bus_size_t stolen_size) 299{ 300 uint32_t bsm; 301 302 bsm = pci_conf_read(pc, tag, INTEL_GEN11_BSM_DW0) & INTEL_BSM_MASK; 303 bsm |= (uint64_t)pci_conf_read(pc, tag, INTEL_GEN11_BSM_DW1) << 32; 304 305 return bsm; 306} 307 308struct intel_stolen_ops { 309 bus_size_t (*size)(pci_chipset_tag_t, pcitag_t); 310 bus_addr_t (*base)(pci_chipset_tag_t, pcitag_t, bus_size_t); 311}; 312 313static const struct intel_stolen_ops i830_stolen_ops = { 314 .size = i830_stolen_size, 315 .base = i830_stolen_base, 316}; 317 318static const struct intel_stolen_ops i845_stolen_ops = { 319 .size = i830_stolen_size, 320 .base = i845_stolen_base, 321}; 322 323static const struct intel_stolen_ops i85x_stolen_ops = { 324 .size = gen3_stolen_size, 325 .base = i85x_stolen_base, 326}; 327 328static const struct intel_stolen_ops i865_stolen_ops = { 329 .size = gen3_stolen_size, 330 .base = i865_stolen_base, 331}; 332 333static const struct intel_stolen_ops gen3_stolen_ops = { 334 .size = gen3_stolen_size, 335 .base = gen3_stolen_base, 336}; 337 338static const struct intel_stolen_ops gen6_stolen_ops = { 339 .size = gen6_stolen_size, 340 .base = gen3_stolen_base, 341}; 342 343static const struct intel_stolen_ops gen8_stolen_ops = { 344 .size = gen8_stolen_size, 345 .base = gen3_stolen_base, 346}; 347 348static const struct intel_stolen_ops gen9_stolen_ops = { 349 .size = gen9_stolen_size, 350 .base = gen3_stolen_base, 351}; 352 353static const struct intel_stolen_ops chv_stolen_ops = { 354 .size = chv_stolen_size, 355 .base = gen3_stolen_base, 356}; 357 358static const struct intel_stolen_ops gen11_stolen_ops = { 359 .size = gen9_stolen_size, 360 .base = gen11_stolen_base, 361}; 362 363static const struct pci_device_id intel_stolen_ids[] = { 364 INTEL_I830_IDS(&i830_stolen_ops), 365 INTEL_I845G_IDS(&i845_stolen_ops), 366 INTEL_I85X_IDS(&i85x_stolen_ops), 367 INTEL_I865G_IDS(&i865_stolen_ops), 368 INTEL_I915G_IDS(&gen3_stolen_ops), 369 INTEL_I915GM_IDS(&gen3_stolen_ops), 370 INTEL_I945G_IDS(&gen3_stolen_ops), 371 INTEL_I945GM_IDS(&gen3_stolen_ops), 372 INTEL_VLV_IDS(&gen6_stolen_ops), 373 INTEL_PINEVIEW_G_IDS(&gen3_stolen_ops), 374 INTEL_PINEVIEW_M_IDS(&gen3_stolen_ops), 375 INTEL_I965G_IDS(&gen3_stolen_ops), 376 INTEL_G33_IDS(&gen3_stolen_ops), 377 INTEL_I965GM_IDS(&gen3_stolen_ops), 378 INTEL_GM45_IDS(&gen3_stolen_ops), 379 INTEL_G45_IDS(&gen3_stolen_ops), 380 INTEL_IRONLAKE_D_IDS(&gen3_stolen_ops), 381 INTEL_IRONLAKE_M_IDS(&gen3_stolen_ops), 382 INTEL_SNB_D_IDS(&gen6_stolen_ops), 383 INTEL_SNB_M_IDS(&gen6_stolen_ops), 384 INTEL_IVB_M_IDS(&gen6_stolen_ops), 385 INTEL_IVB_D_IDS(&gen6_stolen_ops), 386 INTEL_HSW_IDS(&gen6_stolen_ops), 387 INTEL_BDW_IDS(&gen8_stolen_ops), 388 INTEL_CHV_IDS(&chv_stolen_ops), 389 INTEL_SKL_IDS(&gen9_stolen_ops), 390 INTEL_BXT_IDS(&gen9_stolen_ops), 391 INTEL_KBL_IDS(&gen9_stolen_ops), 392 INTEL_CFL_IDS(&gen9_stolen_ops), 393 INTEL_GLK_IDS(&gen9_stolen_ops), 394 INTEL_CNL_IDS(&gen9_stolen_ops), 395 INTEL_ICL_11_IDS(&gen11_stolen_ops), 396 INTEL_EHL_IDS(&gen11_stolen_ops), 397 INTEL_TGL_12_IDS(&gen11_stolen_ops), 398}; 399 400void 401intel_gtt_get(uint64_t *va_size, bus_addr_t *aper_base, 402 resource_size_t *aper_size) 403{ 404 struct agp_softc *sc; 405 pci_chipset_tag_t pc; 406 pcitag_t tag; 407 struct agp_i810_softc *isc; 408 const struct intel_stolen_ops *ops; 409 bus_addr_t stolen_base; 410 bus_size_t stolen_size; 411 unsigned i; 412 413 if ((sc = agp_i810_sc) == NULL) { 414 *va_size = 0; 415 *aper_base = 0; 416 *aper_size = 0; 417 return; 418 } 419 420 pc = sc->as_pc; 421 tag = sc->as_tag; 422 423 isc = sc->as_chipc; 424 *va_size = ((size_t)(isc->gtt_size/sizeof(uint32_t)) << PAGE_SHIFT); 425 *aper_base = sc->as_apaddr; 426 *aper_size = sc->as_apsize; 427 428 for (i = 0; i < __arraycount(intel_stolen_ids); i++) { 429 if (intel_stolen_ids[i].device == PCI_PRODUCT(sc->as_id)) { 430 ops = (const struct intel_stolen_ops *) 431 intel_stolen_ids[i].driver_data; 432 stolen_size = (*ops->size)(pc, tag); 433 stolen_base = (*ops->base)(pc, tag, stolen_size); 434 intel_graphics_stolen_res.start = stolen_base; 435 intel_graphics_stolen_res.end = 436 stolen_base + stolen_size - 1; 437 break; 438 } 439 } 440} 441 442int 443intel_gmch_probe(struct pci_dev *bridge_pci __unused, 444 struct pci_dev *gpu __unused, struct agp_bridge_data *bridge_agp __unused) 445{ 446 struct agp_softc *const sc = agp_i810_sc; 447 int nsegs; 448 int error; 449 450 if (sc == NULL) 451 return 0; 452 453 error = bus_dmamem_alloc(sc->as_dmat, PAGE_SIZE, PAGE_SIZE, 0, 454 &intel_gtt.scratch_seg, 1, &nsegs, BUS_DMA_WAITOK); 455 if (error) 456 goto fail0; 457 KASSERT(nsegs == 1); 458 459 error = bus_dmamap_create(sc->as_dmat, PAGE_SIZE, 1, PAGE_SIZE, 0, 460 BUS_DMA_WAITOK, &intel_gtt.scratch_map); 461 if (error) 462 goto fail1; 463 464 error = bus_dmamap_load_raw(sc->as_dmat, intel_gtt.scratch_map, 465 &intel_gtt.scratch_seg, 1, PAGE_SIZE, BUS_DMA_WAITOK); 466 if (error) 467 goto fail2; 468 469 /* Success! */ 470 return 1; 471 472fail3: __unused 473 bus_dmamap_unload(sc->as_dmat, intel_gtt.scratch_map); 474fail2: bus_dmamap_destroy(sc->as_dmat, intel_gtt.scratch_map); 475fail1: bus_dmamem_free(sc->as_dmat, &intel_gtt.scratch_seg, 1); 476fail0: KASSERT(error); 477 return 0; 478} 479 480void 481intel_gmch_remove(void) 482{ 483 struct agp_softc *const sc = agp_i810_sc; 484 485 bus_dmamap_unload(sc->as_dmat, intel_gtt.scratch_map); 486 bus_dmamap_destroy(sc->as_dmat, intel_gtt.scratch_map); 487 bus_dmamem_free(sc->as_dmat, &intel_gtt.scratch_seg, 1); 488} 489 490bool 491intel_enable_gtt(void) 492{ 493 struct agp_softc *sc = agp_i810_sc; 494 struct agp_i810_softc *isc; 495 496 if (sc == NULL) 497 return false; 498 isc = sc->as_chipc; 499 agp_i810_reset(isc); 500 return true; 501} 502 503void 504intel_gtt_chipset_flush(void) 505{ 506 507 KASSERT(agp_i810_sc != NULL); 508 agp_i810_chipset_flush(agp_i810_sc->as_chipc); 509} 510 511static int 512intel_gtt_flags(unsigned flags) 513{ 514 int gtt_flags = AGP_I810_GTT_VALID; 515 516 switch (flags) { 517 case AGP_USER_MEMORY: 518 break; 519 case AGP_USER_CACHED_MEMORY: 520 gtt_flags |= AGP_I810_GTT_CACHED; 521 break; 522 default: 523 panic("invalid intel gtt flags: %x", flags); 524 } 525 526 return gtt_flags; 527} 528 529void 530intel_gtt_insert_page(bus_addr_t addr, unsigned va_page, unsigned flags) 531{ 532 struct agp_i810_softc *const isc = agp_i810_sc->as_chipc; 533 off_t va = (off_t)va_page << PAGE_SHIFT; 534 int gtt_flags = intel_gtt_flags(flags); 535 int error; 536 537 error = agp_i810_write_gtt_entry(isc, va, addr, gtt_flags); 538 if (error) 539 device_printf(agp_i810_sc->as_dev, 540 "write gtt entry" 541 " %"PRIxMAX" -> %"PRIxMAX" (flags=%x) failed: %d\n", 542 (uintmax_t)va, (uintmax_t)addr, flags, 543 error); 544 agp_i810_post_gtt_entry(isc, va); 545 intel_gtt_chipset_flush(); 546} 547 548void 549intel_gtt_insert_sg_entries(struct sg_table *sg, unsigned va_page, 550 unsigned flags) 551{ 552 bus_dmamap_t dmamap = sg->sgl[0].sg_dmamap; 553 struct agp_i810_softc *const isc = agp_i810_sc->as_chipc; 554 off_t va = (off_t)va_page << PAGE_SHIFT; 555 unsigned seg; 556 int gtt_flags = intel_gtt_flags(flags); 557 int error; 558 559 KASSERT(0 <= va); 560 KASSERT((va >> PAGE_SHIFT) == va_page); 561 KASSERT(0 < dmamap->dm_nsegs); 562 563 for (seg = 0; seg < dmamap->dm_nsegs; seg++) { 564 const bus_addr_t addr = dmamap->dm_segs[seg].ds_addr; 565 bus_size_t len; 566 567 for (len = dmamap->dm_segs[seg].ds_len; 568 len >= PAGE_SIZE; 569 len -= PAGE_SIZE, va += PAGE_SIZE) { 570 error = agp_i810_write_gtt_entry(isc, va, addr, 571 gtt_flags); 572 if (error) 573 device_printf(agp_i810_sc->as_dev, 574 "write gtt entry" 575 " %"PRIxMAX" -> %"PRIxMAX" (flags=%x)" 576 " failed: %d\n", 577 (uintmax_t)va, (uintmax_t)addr, flags, 578 error); 579 } 580 KASSERTMSG(len == 0, 581 "segment length not divisible by PAGE_SIZE: %jx", 582 (uintmax_t)dmamap->dm_segs[seg].ds_len); 583 } 584 agp_i810_post_gtt_entry(isc, (va - PAGE_SIZE)); 585 intel_gtt_chipset_flush(); 586} 587 588void 589intel_gtt_clear_range(unsigned va_page, unsigned npages) 590{ 591 struct agp_i810_softc *const isc = agp_i810_sc->as_chipc; 592 const bus_addr_t addr = intel_gtt.scratch_map->dm_segs[0].ds_addr; 593 const int gtt_flags = AGP_I810_GTT_VALID; 594 off_t va = (va_page << PAGE_SHIFT); 595 596 KASSERT(0 <= va); 597 KASSERT((va >> PAGE_SHIFT) == va_page); 598 KASSERT(0 < npages); 599 600 while (npages--) { 601 agp_i810_write_gtt_entry(isc, va, addr, gtt_flags); 602 va += PAGE_SIZE; 603 } 604 agp_i810_post_gtt_entry(isc, va - PAGE_SIZE); 605} 606