1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2006-2008 Stanislav Sedov <stas@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 */ 29 30#include <sys/cdefs.h> 31__FBSDID("$FreeBSD$"); 32 33#include <sys/param.h> 34#include <sys/systm.h> 35#include <sys/conf.h> 36#include <sys/fcntl.h> 37#include <sys/ioccom.h> 38#include <sys/malloc.h> 39#include <sys/module.h> 40#include <sys/mutex.h> 41#include <sys/priv.h> 42#include <sys/proc.h> 43#include <sys/queue.h> 44#include <sys/sched.h> 45#include <sys/kernel.h> 46#include <sys/sysctl.h> 47#include <sys/uio.h> 48#include <sys/pcpu.h> 49#include <sys/smp.h> 50#include <sys/pmckern.h> 51#include <sys/cpuctl.h> 52 53#include <vm/vm.h> 54#include <vm/vm_param.h> 55#include <vm/pmap.h> 56 57#include <machine/cpufunc.h> 58#include <machine/md_var.h> 59#include <machine/specialreg.h> 60#include <x86/ucode.h> 61 62static d_open_t cpuctl_open; 63static d_ioctl_t cpuctl_ioctl; 64 65#define CPUCTL_VERSION 1 66 67#ifdef CPUCTL_DEBUG 68# define DPRINTF(format,...) printf(format, __VA_ARGS__); 69#else 70# define DPRINTF(...) 71#endif 72 73#define UCODE_SIZE_MAX (4 * 1024 * 1024) 74 75static int cpuctl_do_msr(int cpu, cpuctl_msr_args_t *data, u_long cmd, 76 struct thread *td); 77static int cpuctl_do_cpuid(int cpu, cpuctl_cpuid_args_t *data, 78 struct thread *td); 79static int cpuctl_do_cpuid_count(int cpu, cpuctl_cpuid_count_args_t *data, 80 struct thread *td); 81static int cpuctl_do_eval_cpu_features(int cpu, struct thread *td); 82static int cpuctl_do_update(int cpu, cpuctl_update_args_t *data, 83 struct thread *td); 84static int update_intel(int cpu, cpuctl_update_args_t *args, 85 struct thread *td); 86static int update_amd(int cpu, cpuctl_update_args_t *args, struct thread *td); 87static int update_via(int cpu, cpuctl_update_args_t *args, 88 struct thread *td); 89 90static struct cdev **cpuctl_devs; 91static MALLOC_DEFINE(M_CPUCTL, "cpuctl", "CPUCTL buffer"); 92 93static struct cdevsw cpuctl_cdevsw = { 94 .d_version = D_VERSION, 95 .d_open = cpuctl_open, 96 .d_ioctl = cpuctl_ioctl, 97 .d_name = "cpuctl", 98}; 99 100/* 101 * This function checks if specified cpu enabled or not. 102 */ 103static int 104cpu_enabled(int cpu) 105{ 106 107 return (pmc_cpu_is_disabled(cpu) == 0); 108} 109 110/* 111 * Check if the current thread is bound to a specific cpu. 112 */ 113static int 114cpu_sched_is_bound(struct thread *td) 115{ 116 int ret; 117 118 thread_lock(td); 119 ret = sched_is_bound(td); 120 thread_unlock(td); 121 return (ret); 122} 123 124/* 125 * Switch to target cpu to run. 126 */ 127static void 128set_cpu(int cpu, struct thread *td) 129{ 130 131 KASSERT(cpu >= 0 && cpu <= mp_maxid && cpu_enabled(cpu), 132 ("[cpuctl,%d]: bad cpu number %d", __LINE__, cpu)); 133 thread_lock(td); 134 sched_bind(td, cpu); 135 thread_unlock(td); 136 KASSERT(td->td_oncpu == cpu, 137 ("[cpuctl,%d]: cannot bind to target cpu %d on cpu %d", __LINE__, 138 cpu, td->td_oncpu)); 139} 140 141static void 142restore_cpu(int oldcpu, int is_bound, struct thread *td) 143{ 144 145 KASSERT(oldcpu >= 0 && oldcpu <= mp_maxid && cpu_enabled(oldcpu), 146 ("[cpuctl,%d]: bad cpu number %d", __LINE__, oldcpu)); 147 thread_lock(td); 148 if (is_bound == 0) 149 sched_unbind(td); 150 else 151 sched_bind(td, oldcpu); 152 thread_unlock(td); 153} 154 155int 156cpuctl_ioctl(struct cdev *dev, u_long cmd, caddr_t data, 157 int flags, struct thread *td) 158{ 159 int cpu, ret; 160 161 cpu = dev2unit(dev); 162 if (cpu > mp_maxid || !cpu_enabled(cpu)) { 163 DPRINTF("[cpuctl,%d]: bad cpu number %d\n", __LINE__, cpu); 164 return (ENXIO); 165 } 166 /* Require write flag for "write" requests. */ 167 if ((cmd == CPUCTL_MSRCBIT || cmd == CPUCTL_MSRSBIT || 168 cmd == CPUCTL_UPDATE || cmd == CPUCTL_WRMSR || 169 cmd == CPUCTL_EVAL_CPU_FEATURES) && 170 (flags & FWRITE) == 0) 171 return (EPERM); 172 switch (cmd) { 173 case CPUCTL_RDMSR: 174 ret = cpuctl_do_msr(cpu, (cpuctl_msr_args_t *)data, cmd, td); 175 break; 176 case CPUCTL_MSRSBIT: 177 case CPUCTL_MSRCBIT: 178 case CPUCTL_WRMSR: 179 ret = priv_check(td, PRIV_CPUCTL_WRMSR); 180 if (ret != 0) 181 goto fail; 182 ret = cpuctl_do_msr(cpu, (cpuctl_msr_args_t *)data, cmd, td); 183 break; 184 case CPUCTL_CPUID: 185 ret = cpuctl_do_cpuid(cpu, (cpuctl_cpuid_args_t *)data, td); 186 break; 187 case CPUCTL_UPDATE: 188 ret = priv_check(td, PRIV_CPUCTL_UPDATE); 189 if (ret != 0) 190 goto fail; 191 ret = cpuctl_do_update(cpu, (cpuctl_update_args_t *)data, td); 192 break; 193 case CPUCTL_CPUID_COUNT: 194 ret = cpuctl_do_cpuid_count(cpu, 195 (cpuctl_cpuid_count_args_t *)data, td); 196 break; 197 case CPUCTL_EVAL_CPU_FEATURES: 198 ret = cpuctl_do_eval_cpu_features(cpu, td); 199 break; 200 default: 201 ret = EINVAL; 202 break; 203 } 204fail: 205 return (ret); 206} 207 208/* 209 * Actually perform cpuid operation. 210 */ 211static int 212cpuctl_do_cpuid_count(int cpu, cpuctl_cpuid_count_args_t *data, 213 struct thread *td) 214{ 215 int is_bound = 0; 216 int oldcpu; 217 218 KASSERT(cpu >= 0 && cpu <= mp_maxid, 219 ("[cpuctl,%d]: bad cpu number %d", __LINE__, cpu)); 220 221 /* Explicitly clear cpuid data to avoid returning stale info. */ 222 bzero(data->data, sizeof(data->data)); 223 DPRINTF("[cpuctl,%d]: retrieving cpuid lev %#0x type %#0x for %d cpu\n", 224 __LINE__, data->level, data->level_type, cpu); 225#ifdef __i386__ 226 if (cpu_id == 0) 227 return (ENODEV); 228#endif 229 oldcpu = td->td_oncpu; 230 is_bound = cpu_sched_is_bound(td); 231 set_cpu(cpu, td); 232 cpuid_count(data->level, data->level_type, data->data); 233 restore_cpu(oldcpu, is_bound, td); 234 return (0); 235} 236 237static int 238cpuctl_do_cpuid(int cpu, cpuctl_cpuid_args_t *data, struct thread *td) 239{ 240 cpuctl_cpuid_count_args_t cdata; 241 int error; 242 243 cdata.level = data->level; 244 /* Override the level type. */ 245 cdata.level_type = 0; 246 error = cpuctl_do_cpuid_count(cpu, &cdata, td); 247 bcopy(cdata.data, data->data, sizeof(data->data)); /* Ignore error */ 248 return (error); 249} 250 251/* 252 * Actually perform MSR operations. 253 */ 254static int 255cpuctl_do_msr(int cpu, cpuctl_msr_args_t *data, u_long cmd, struct thread *td) 256{ 257 uint64_t reg; 258 int is_bound = 0; 259 int oldcpu; 260 int ret; 261 262 KASSERT(cpu >= 0 && cpu <= mp_maxid, 263 ("[cpuctl,%d]: bad cpu number %d", __LINE__, cpu)); 264 265 /* 266 * Explicitly clear cpuid data to avoid returning stale 267 * info 268 */ 269 DPRINTF("[cpuctl,%d]: operating on MSR %#0x for %d cpu\n", __LINE__, 270 data->msr, cpu); 271#ifdef __i386__ 272 if ((cpu_feature & CPUID_MSR) == 0) 273 return (ENODEV); 274#endif 275 oldcpu = td->td_oncpu; 276 is_bound = cpu_sched_is_bound(td); 277 set_cpu(cpu, td); 278 if (cmd == CPUCTL_RDMSR) { 279 data->data = 0; 280 ret = rdmsr_safe(data->msr, &data->data); 281 } else if (cmd == CPUCTL_WRMSR) { 282 ret = wrmsr_safe(data->msr, data->data); 283 } else if (cmd == CPUCTL_MSRSBIT) { 284 critical_enter(); 285 ret = rdmsr_safe(data->msr, ®); 286 if (ret == 0) 287 ret = wrmsr_safe(data->msr, reg | data->data); 288 critical_exit(); 289 } else if (cmd == CPUCTL_MSRCBIT) { 290 critical_enter(); 291 ret = rdmsr_safe(data->msr, ®); 292 if (ret == 0) 293 ret = wrmsr_safe(data->msr, reg & ~data->data); 294 critical_exit(); 295 } else 296 panic("[cpuctl,%d]: unknown operation requested: %lu", 297 __LINE__, cmd); 298 restore_cpu(oldcpu, is_bound, td); 299 return (ret); 300} 301 302/* 303 * Actually perform microcode update. 304 */ 305static int 306cpuctl_do_update(int cpu, cpuctl_update_args_t *data, struct thread *td) 307{ 308 cpuctl_cpuid_args_t args = { 309 .level = 0, 310 }; 311 char vendor[13]; 312 int ret; 313 314 KASSERT(cpu >= 0 && cpu <= mp_maxid, 315 ("[cpuctl,%d]: bad cpu number %d", __LINE__, cpu)); 316 DPRINTF("[cpuctl,%d]: XXX %d", __LINE__, cpu); 317 318 ret = cpuctl_do_cpuid(cpu, &args, td); 319 if (ret != 0) 320 return (ret); 321 ((uint32_t *)vendor)[0] = args.data[1]; 322 ((uint32_t *)vendor)[1] = args.data[3]; 323 ((uint32_t *)vendor)[2] = args.data[2]; 324 vendor[12] = '\0'; 325 if (strncmp(vendor, INTEL_VENDOR_ID, sizeof(INTEL_VENDOR_ID)) == 0) 326 ret = update_intel(cpu, data, td); 327 else if(strncmp(vendor, AMD_VENDOR_ID, sizeof(AMD_VENDOR_ID)) == 0) 328 ret = update_amd(cpu, data, td); 329 else if(strncmp(vendor, CENTAUR_VENDOR_ID, sizeof(CENTAUR_VENDOR_ID)) 330 == 0) 331 ret = update_via(cpu, data, td); 332 else 333 ret = ENXIO; 334 return (ret); 335} 336 337struct ucode_update_data { 338 void *ptr; 339 int cpu; 340 int ret; 341}; 342 343static void 344ucode_intel_load_rv(void *arg) 345{ 346 struct ucode_update_data *d; 347 348 d = arg; 349 if (PCPU_GET(cpuid) == d->cpu) 350 d->ret = ucode_intel_load(d->ptr, true, NULL, NULL); 351} 352 353static int 354update_intel(int cpu, cpuctl_update_args_t *args, struct thread *td) 355{ 356 struct ucode_update_data d; 357 void *ptr; 358 int is_bound, oldcpu, ret; 359 360 if (args->size == 0 || args->data == NULL) { 361 DPRINTF("[cpuctl,%d]: zero-sized firmware image", __LINE__); 362 return (EINVAL); 363 } 364 if (args->size > UCODE_SIZE_MAX) { 365 DPRINTF("[cpuctl,%d]: firmware image too large", __LINE__); 366 return (EINVAL); 367 } 368 369 /* 370 * 16 byte alignment required. Rely on the fact that 371 * malloc(9) always returns the pointer aligned at least on 372 * the size of the allocation. 373 */ 374 ptr = malloc(args->size + 16, M_CPUCTL, M_WAITOK); 375 if (copyin(args->data, ptr, args->size) != 0) { 376 DPRINTF("[cpuctl,%d]: copyin %p->%p of %zd bytes failed", 377 __LINE__, args->data, ptr, args->size); 378 ret = EFAULT; 379 goto out; 380 } 381 oldcpu = td->td_oncpu; 382 is_bound = cpu_sched_is_bound(td); 383 set_cpu(cpu, td); 384 d.ptr = ptr; 385 d.cpu = cpu; 386 smp_rendezvous(NULL, ucode_intel_load_rv, NULL, &d); 387 restore_cpu(oldcpu, is_bound, td); 388 ret = d.ret; 389 390 /* 391 * Replace any existing update. This ensures that the new update 392 * will be reloaded automatically during ACPI resume. 393 */ 394 if (ret == 0) 395 ptr = ucode_update(ptr); 396 397out: 398 free(ptr, M_CPUCTL); 399 return (ret); 400} 401 402/* 403 * NB: MSR 0xc0010020, MSR_K8_UCODE_UPDATE, is not documented by AMD. 404 * Coreboot, illumos and Linux source code was used to understand 405 * its workings. 406 */ 407static void 408amd_ucode_wrmsr(void *ucode_ptr) 409{ 410 uint32_t tmp[4]; 411 412 wrmsr_safe(MSR_K8_UCODE_UPDATE, (uintptr_t)ucode_ptr); 413 do_cpuid(0, tmp); 414} 415 416static int 417update_amd(int cpu, cpuctl_update_args_t *args, struct thread *td) 418{ 419 void *ptr; 420 int ret; 421 422 if (args->size == 0 || args->data == NULL) { 423 DPRINTF("[cpuctl,%d]: zero-sized firmware image", __LINE__); 424 return (EINVAL); 425 } 426 if (args->size > UCODE_SIZE_MAX) { 427 DPRINTF("[cpuctl,%d]: firmware image too large", __LINE__); 428 return (EINVAL); 429 } 430 431 /* 432 * 16 byte alignment required. Rely on the fact that 433 * malloc(9) always returns the pointer aligned at least on 434 * the size of the allocation. 435 */ 436 ptr = malloc(args->size + 16, M_CPUCTL, M_ZERO | M_WAITOK); 437 if (copyin(args->data, ptr, args->size) != 0) { 438 DPRINTF("[cpuctl,%d]: copyin %p->%p of %zd bytes failed", 439 __LINE__, args->data, ptr, args->size); 440 ret = EFAULT; 441 goto fail; 442 } 443 smp_rendezvous(NULL, amd_ucode_wrmsr, NULL, ptr); 444 ret = 0; 445fail: 446 free(ptr, M_CPUCTL); 447 return (ret); 448} 449 450static int 451update_via(int cpu, cpuctl_update_args_t *args, struct thread *td) 452{ 453 void *ptr; 454 uint64_t rev0, rev1, res; 455 uint32_t tmp[4]; 456 int is_bound; 457 int oldcpu; 458 int ret; 459 460 if (args->size == 0 || args->data == NULL) { 461 DPRINTF("[cpuctl,%d]: zero-sized firmware image", __LINE__); 462 return (EINVAL); 463 } 464 if (args->size > UCODE_SIZE_MAX) { 465 DPRINTF("[cpuctl,%d]: firmware image too large", __LINE__); 466 return (EINVAL); 467 } 468 469 /* 470 * 4 byte alignment required. 471 */ 472 ptr = malloc(args->size, M_CPUCTL, M_WAITOK); 473 if (copyin(args->data, ptr, args->size) != 0) { 474 DPRINTF("[cpuctl,%d]: copyin %p->%p of %zd bytes failed", 475 __LINE__, args->data, ptr, args->size); 476 ret = EFAULT; 477 goto fail; 478 } 479 oldcpu = td->td_oncpu; 480 is_bound = cpu_sched_is_bound(td); 481 set_cpu(cpu, td); 482 critical_enter(); 483 rdmsr_safe(MSR_BIOS_SIGN, &rev0); /* Get current microcode revision. */ 484 485 /* 486 * Perform update. 487 */ 488 wrmsr_safe(MSR_BIOS_UPDT_TRIG, (uintptr_t)(ptr)); 489 do_cpuid(1, tmp); 490 491 /* 492 * Result are in low byte of MSR FCR5: 493 * 0x00: No update has been attempted since RESET. 494 * 0x01: The last attempted update was successful. 495 * 0x02: The last attempted update was unsuccessful due to a bad 496 * environment. No update was loaded and any preexisting 497 * patches are still active. 498 * 0x03: The last attempted update was not applicable to this processor. 499 * No update was loaded and any preexisting patches are still 500 * active. 501 * 0x04: The last attempted update was not successful due to an invalid 502 * update data block. No update was loaded and any preexisting 503 * patches are still active 504 */ 505 rdmsr_safe(0x1205, &res); 506 res &= 0xff; 507 critical_exit(); 508 rdmsr_safe(MSR_BIOS_SIGN, &rev1); /* Get new microcode revision. */ 509 restore_cpu(oldcpu, is_bound, td); 510 511 DPRINTF("[cpu,%d]: rev0=%x rev1=%x res=%x\n", __LINE__, 512 (unsigned)(rev0 >> 32), (unsigned)(rev1 >> 32), (unsigned)res); 513 514 if (res != 0x01) 515 ret = EINVAL; 516 else 517 ret = 0; 518fail: 519 free(ptr, M_CPUCTL); 520 return (ret); 521} 522 523static int 524cpuctl_do_eval_cpu_features(int cpu, struct thread *td) 525{ 526 int is_bound = 0; 527 int oldcpu; 528 529 KASSERT(cpu >= 0 && cpu <= mp_maxid, 530 ("[cpuctl,%d]: bad cpu number %d", __LINE__, cpu)); 531 532#ifdef __i386__ 533 if (cpu_id == 0) 534 return (ENODEV); 535#endif 536 oldcpu = td->td_oncpu; 537 is_bound = cpu_sched_is_bound(td); 538 set_cpu(cpu, td); 539 identify_cpu1(); 540 identify_cpu2(); 541 restore_cpu(oldcpu, is_bound, td); 542 hw_ibrs_recalculate(true); 543 hw_ssb_recalculate(true); 544#ifdef __amd64__ 545 amd64_syscall_ret_flush_l1d_recalc(); 546 pmap_allow_2m_x_ept_recalculate(); 547#endif 548 hw_mds_recalculate(); 549 x86_taa_recalculate(); 550 x86_rngds_mitg_recalculate(true); 551 printcpuinfo(); 552 return (0); 553} 554 555int 556cpuctl_open(struct cdev *dev, int flags, int fmt __unused, struct thread *td) 557{ 558 int ret = 0; 559 int cpu; 560 561 cpu = dev2unit(dev); 562 if (cpu > mp_maxid || !cpu_enabled(cpu)) { 563 DPRINTF("[cpuctl,%d]: incorrect cpu number %d\n", __LINE__, 564 cpu); 565 return (ENXIO); 566 } 567 if (flags & FWRITE) 568 ret = securelevel_gt(td->td_ucred, 0); 569 return (ret); 570} 571 572static int 573cpuctl_modevent(module_t mod __unused, int type, void *data __unused) 574{ 575 int cpu; 576 577 switch(type) { 578 case MOD_LOAD: 579 if (bootverbose) 580 printf("cpuctl: access to MSR registers/cpuid info.\n"); 581 cpuctl_devs = malloc(sizeof(*cpuctl_devs) * (mp_maxid + 1), M_CPUCTL, 582 M_WAITOK | M_ZERO); 583 CPU_FOREACH(cpu) 584 if (cpu_enabled(cpu)) 585 cpuctl_devs[cpu] = make_dev(&cpuctl_cdevsw, cpu, 586 UID_ROOT, GID_KMEM, 0640, "cpuctl%d", cpu); 587 break; 588 case MOD_UNLOAD: 589 CPU_FOREACH(cpu) { 590 if (cpuctl_devs[cpu] != NULL) 591 destroy_dev(cpuctl_devs[cpu]); 592 } 593 free(cpuctl_devs, M_CPUCTL); 594 break; 595 case MOD_SHUTDOWN: 596 break; 597 default: 598 return (EOPNOTSUPP); 599 } 600 return (0); 601} 602 603DEV_MODULE(cpuctl, cpuctl_modevent, NULL); 604MODULE_VERSION(cpuctl, CPUCTL_VERSION); 605