1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 26 /* All Rights Reserved */ 27 28 #include <sys/param.h> 29 #include <sys/types.h> 30 #include <sys/vmparam.h> 31 #include <sys/systm.h> 32 #include <sys/signal.h> 33 #include <sys/stack.h> 34 #include <sys/regset.h> 35 #include <sys/privregs.h> 36 #include <sys/frame.h> 37 #include <sys/proc.h> 38 #include <sys/psw.h> 39 #include <sys/siginfo.h> 40 #include <sys/cpuvar.h> 41 #include <sys/asm_linkage.h> 42 #include <sys/kmem.h> 43 #include <sys/errno.h> 44 #include <sys/bootconf.h> 45 #include <sys/archsystm.h> 46 #include <sys/debug.h> 47 #include <sys/elf.h> 48 #include <sys/spl.h> 49 #include <sys/time.h> 50 #include <sys/atomic.h> 51 #include <sys/sysmacros.h> 52 #include <sys/cmn_err.h> 53 #include <sys/modctl.h> 54 #include <sys/kobj.h> 55 #include <sys/panic.h> 56 #include <sys/reboot.h> 57 #include <sys/time.h> 58 #include <sys/fp.h> 59 #include <sys/x86_archext.h> 60 #include <sys/auxv.h> 61 #include <sys/auxv_386.h> 62 #include <sys/dtrace.h> 63 #include <sys/brand.h> 64 #include <sys/machbrand.h> 65 #include <sys/cmn_err.h> 66 67 extern const struct fnsave_state x87_initial; 68 extern const struct fxsave_state sse_initial; 69 70 /* 71 * Map an fnsave-formatted save area into an fxsave-formatted save area. 72 * 73 * Most fields are the same width, content and semantics. However 74 * the tag word is compressed. 75 */ 76 static void 77 fnsave_to_fxsave(const struct fnsave_state *fn, struct fxsave_state *fx) 78 { 79 uint_t i, tagbits; 80 81 fx->fx_fcw = fn->f_fcw; 82 fx->fx_fsw = fn->f_fsw; 83 84 /* 85 * copy element by element (because of holes) 86 */ 87 for (i = 0; i < 8; i++) 88 bcopy(&fn->f_st[i].fpr_16[0], &fx->fx_st[i].fpr_16[0], 89 sizeof (fn->f_st[0].fpr_16)); /* 80-bit x87-style floats */ 90 91 /* 92 * synthesize compressed tag bits 93 */ 94 fx->fx_fctw = 0; 95 for (tagbits = fn->f_ftw, i = 0; i < 8; i++, tagbits >>= 2) 96 if ((tagbits & 3) != 3) 97 fx->fx_fctw |= (1 << i); 98 99 fx->fx_fop = fn->f_fop; 100 101 #if defined(__amd64) 102 fx->fx_rip = (uint64_t)fn->f_eip; 103 fx->fx_rdp = (uint64_t)fn->f_dp; 104 #else 105 fx->fx_eip = fn->f_eip; 106 fx->fx_cs = fn->f_cs; 107 fx->__fx_ign0 = 0; 108 fx->fx_dp = fn->f_dp; 109 fx->fx_ds = fn->f_ds; 110 fx->__fx_ign1 = 0; 111 #endif 112 } 113 114 /* 115 * Map from an fxsave-format save area to an fnsave-format save area. 116 */ 117 static void 118 fxsave_to_fnsave(const struct fxsave_state *fx, struct fnsave_state *fn) 119 { 120 uint_t i, top, tagbits; 121 122 fn->f_fcw = fx->fx_fcw; 123 fn->__f_ign0 = 0; 124 fn->f_fsw = fx->fx_fsw; 125 fn->__f_ign1 = 0; 126 127 top = (fx->fx_fsw & FPS_TOP) >> 11; 128 129 /* 130 * copy element by element (because of holes) 131 */ 132 for (i = 0; i < 8; i++) 133 bcopy(&fx->fx_st[i].fpr_16[0], &fn->f_st[i].fpr_16[0], 134 sizeof (fn->f_st[0].fpr_16)); /* 80-bit x87-style floats */ 135 136 /* 137 * synthesize uncompressed tag bits 138 */ 139 fn->f_ftw = 0; 140 for (tagbits = fx->fx_fctw, i = 0; i < 8; i++, tagbits >>= 1) { 141 uint_t ibit, expo; 142 const uint16_t *fpp; 143 static const uint16_t zero[5] = { 0, 0, 0, 0, 0 }; 144 145 if ((tagbits & 1) == 0) { 146 fn->f_ftw |= 3 << (i << 1); /* empty */ 147 continue; 148 } 149 150 /* 151 * (tags refer to *physical* registers) 152 */ 153 fpp = &fx->fx_st[(i - top + 8) & 7].fpr_16[0]; 154 ibit = fpp[3] >> 15; 155 expo = fpp[4] & 0x7fff; 156 157 if (ibit && expo != 0 && expo != 0x7fff) 158 continue; /* valid fp number */ 159 160 if (bcmp(fpp, &zero, sizeof (zero))) 161 fn->f_ftw |= 2 << (i << 1); /* NaN */ 162 else 163 fn->f_ftw |= 1 << (i << 1); /* fp zero */ 164 } 165 166 fn->f_fop = fx->fx_fop; 167 168 fn->__f_ign2 = 0; 169 #if defined(__amd64) 170 fn->f_eip = (uint32_t)fx->fx_rip; 171 fn->f_cs = U32CS_SEL; 172 fn->f_dp = (uint32_t)fx->fx_rdp; 173 fn->f_ds = UDS_SEL; 174 #else 175 fn->f_eip = fx->fx_eip; 176 fn->f_cs = fx->fx_cs; 177 fn->f_dp = fx->fx_dp; 178 fn->f_ds = fx->fx_ds; 179 #endif 180 fn->__f_ign3 = 0; 181 } 182 183 /* 184 * Map from an fpregset_t into an fxsave-format save area 185 */ 186 static void 187 fpregset_to_fxsave(const fpregset_t *fp, struct fxsave_state *fx) 188 { 189 #if defined(__amd64) 190 bcopy(fp, fx, sizeof (*fx)); 191 #else 192 const struct fpchip_state *fc = &fp->fp_reg_set.fpchip_state; 193 194 fnsave_to_fxsave((const struct fnsave_state *)fc, fx); 195 fx->fx_mxcsr = fc->mxcsr; 196 bcopy(&fc->xmm[0], &fx->fx_xmm[0], sizeof (fc->xmm)); 197 #endif 198 /* 199 * avoid useless #gp exceptions - mask reserved bits 200 */ 201 fx->fx_mxcsr &= sse_mxcsr_mask; 202 } 203 204 /* 205 * Map from an fxsave-format save area into a fpregset_t 206 */ 207 static void 208 fxsave_to_fpregset(const struct fxsave_state *fx, fpregset_t *fp) 209 { 210 #if defined(__amd64) 211 bcopy(fx, fp, sizeof (*fx)); 212 #else 213 struct fpchip_state *fc = &fp->fp_reg_set.fpchip_state; 214 215 fxsave_to_fnsave(fx, (struct fnsave_state *)fc); 216 fc->mxcsr = fx->fx_mxcsr; 217 bcopy(&fx->fx_xmm[0], &fc->xmm[0], sizeof (fc->xmm)); 218 #endif 219 } 220 221 #if defined(_SYSCALL32_IMPL) 222 static void 223 fpregset32_to_fxsave(const fpregset32_t *fp, struct fxsave_state *fx) 224 { 225 const struct fpchip32_state *fc = &fp->fp_reg_set.fpchip_state; 226 227 fnsave_to_fxsave((const struct fnsave_state *)fc, fx); 228 /* 229 * avoid useless #gp exceptions - mask reserved bits 230 */ 231 fx->fx_mxcsr = sse_mxcsr_mask & fc->mxcsr; 232 bcopy(&fc->xmm[0], &fx->fx_xmm[0], sizeof (fc->xmm)); 233 } 234 235 static void 236 fxsave_to_fpregset32(const struct fxsave_state *fx, fpregset32_t *fp) 237 { 238 struct fpchip32_state *fc = &fp->fp_reg_set.fpchip_state; 239 240 fxsave_to_fnsave(fx, (struct fnsave_state *)fc); 241 fc->mxcsr = fx->fx_mxcsr; 242 bcopy(&fx->fx_xmm[0], &fc->xmm[0], sizeof (fc->xmm)); 243 } 244 245 static void 246 fpregset_nto32(const fpregset_t *src, fpregset32_t *dst) 247 { 248 fxsave_to_fpregset32((struct fxsave_state *)src, dst); 249 dst->fp_reg_set.fpchip_state.status = 250 src->fp_reg_set.fpchip_state.status; 251 dst->fp_reg_set.fpchip_state.xstatus = 252 src->fp_reg_set.fpchip_state.xstatus; 253 } 254 255 static void 256 fpregset_32ton(const fpregset32_t *src, fpregset_t *dst) 257 { 258 fpregset32_to_fxsave(src, (struct fxsave_state *)dst); 259 dst->fp_reg_set.fpchip_state.status = 260 src->fp_reg_set.fpchip_state.status; 261 dst->fp_reg_set.fpchip_state.xstatus = 262 src->fp_reg_set.fpchip_state.xstatus; 263 } 264 #endif 265 266 /* 267 * Set floating-point registers from a native fpregset_t. 268 */ 269 void 270 setfpregs(klwp_t *lwp, fpregset_t *fp) 271 { 272 struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu; 273 274 if (fpu->fpu_flags & FPU_EN) { 275 if (!(fpu->fpu_flags & FPU_VALID)) { 276 /* 277 * FPU context is still active, release the 278 * ownership. 279 */ 280 fp_free(fpu, 0); 281 } 282 } 283 /* 284 * Else: if we are trying to change the FPU state of a thread which 285 * hasn't yet initialized floating point, store the state in 286 * the pcb and indicate that the state is valid. When the 287 * thread enables floating point, it will use this state instead 288 * of the default state. 289 */ 290 291 switch (fp_save_mech) { 292 #if defined(__i386) 293 case FP_FNSAVE: 294 bcopy(fp, &fpu->fpu_regs.kfpu_u.kfpu_fn, 295 sizeof (fpu->fpu_regs.kfpu_u.kfpu_fn)); 296 break; 297 #endif 298 case FP_FXSAVE: 299 fpregset_to_fxsave(fp, &fpu->fpu_regs.kfpu_u.kfpu_fx); 300 fpu->fpu_regs.kfpu_xstatus = 301 fp->fp_reg_set.fpchip_state.xstatus; 302 break; 303 304 case FP_XSAVE: 305 fpregset_to_fxsave(fp, 306 &fpu->fpu_regs.kfpu_u.kfpu_xs.xs_fxsave); 307 fpu->fpu_regs.kfpu_xstatus = 308 fp->fp_reg_set.fpchip_state.xstatus; 309 fpu->fpu_regs.kfpu_u.kfpu_xs.xs_xstate_bv |= 310 (XFEATURE_LEGACY_FP | XFEATURE_SSE); 311 break; 312 default: 313 panic("Invalid fp_save_mech"); 314 /*NOTREACHED*/ 315 } 316 317 fpu->fpu_regs.kfpu_status = fp->fp_reg_set.fpchip_state.status; 318 fpu->fpu_flags |= FPU_VALID; 319 } 320 321 /* 322 * Get floating-point registers into a native fpregset_t. 323 */ 324 void 325 getfpregs(klwp_t *lwp, fpregset_t *fp) 326 { 327 struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu; 328 329 kpreempt_disable(); 330 if (fpu->fpu_flags & FPU_EN) { 331 /* 332 * If we have FPU hw and the thread's pcb doesn't have 333 * a valid FPU state then get the state from the hw. 334 */ 335 if (fpu_exists && ttolwp(curthread) == lwp && 336 !(fpu->fpu_flags & FPU_VALID)) 337 fp_save(fpu); /* get the current FPU state */ 338 } 339 340 /* 341 * There are 3 possible cases we have to be aware of here: 342 * 343 * 1. FPU is enabled. FPU state is stored in the current LWP. 344 * 345 * 2. FPU is not enabled, and there have been no intervening /proc 346 * modifications. Return initial FPU state. 347 * 348 * 3. FPU is not enabled, but a /proc consumer has modified FPU state. 349 * FPU state is stored in the current LWP. 350 */ 351 if ((fpu->fpu_flags & FPU_EN) || (fpu->fpu_flags & FPU_VALID)) { 352 /* 353 * Cases 1 and 3. 354 */ 355 switch (fp_save_mech) { 356 #if defined(__i386) 357 case FP_FNSAVE: 358 bcopy(&fpu->fpu_regs.kfpu_u.kfpu_fn, fp, 359 sizeof (fpu->fpu_regs.kfpu_u.kfpu_fn)); 360 break; 361 #endif 362 case FP_FXSAVE: 363 fxsave_to_fpregset(&fpu->fpu_regs.kfpu_u.kfpu_fx, fp); 364 fp->fp_reg_set.fpchip_state.xstatus = 365 fpu->fpu_regs.kfpu_xstatus; 366 break; 367 case FP_XSAVE: 368 fxsave_to_fpregset( 369 &fpu->fpu_regs.kfpu_u.kfpu_xs.xs_fxsave, fp); 370 fp->fp_reg_set.fpchip_state.xstatus = 371 fpu->fpu_regs.kfpu_xstatus; 372 break; 373 default: 374 panic("Invalid fp_save_mech"); 375 /*NOTREACHED*/ 376 } 377 fp->fp_reg_set.fpchip_state.status = fpu->fpu_regs.kfpu_status; 378 } else { 379 /* 380 * Case 2. 381 */ 382 switch (fp_save_mech) { 383 #if defined(__i386) 384 case FP_FNSAVE: 385 bcopy(&x87_initial, fp, sizeof (x87_initial)); 386 break; 387 #endif 388 case FP_FXSAVE: 389 case FP_XSAVE: 390 /* 391 * For now, we don't have any AVX specific field in ABI. 392 * If we add any in the future, we need to initial them 393 * as well. 394 */ 395 fxsave_to_fpregset(&sse_initial, fp); 396 fp->fp_reg_set.fpchip_state.xstatus = 397 fpu->fpu_regs.kfpu_xstatus; 398 break; 399 default: 400 panic("Invalid fp_save_mech"); 401 /*NOTREACHED*/ 402 } 403 fp->fp_reg_set.fpchip_state.status = fpu->fpu_regs.kfpu_status; 404 } 405 kpreempt_enable(); 406 } 407 408 #if defined(_SYSCALL32_IMPL) 409 410 /* 411 * Set floating-point registers from an fpregset32_t. 412 */ 413 void 414 setfpregs32(klwp_t *lwp, fpregset32_t *fp) 415 { 416 fpregset_t fpregs; 417 418 fpregset_32ton(fp, &fpregs); 419 setfpregs(lwp, &fpregs); 420 } 421 422 /* 423 * Get floating-point registers into an fpregset32_t. 424 */ 425 void 426 getfpregs32(klwp_t *lwp, fpregset32_t *fp) 427 { 428 fpregset_t fpregs; 429 430 getfpregs(lwp, &fpregs); 431 fpregset_nto32(&fpregs, fp); 432 } 433 434 #endif /* _SYSCALL32_IMPL */ 435 436 /* 437 * Return the general registers 438 */ 439 void 440 getgregs(klwp_t *lwp, gregset_t grp) 441 { 442 struct regs *rp = lwptoregs(lwp); 443 #if defined(__amd64) 444 struct pcb *pcb = &lwp->lwp_pcb; 445 int thisthread = lwptot(lwp) == curthread; 446 447 grp[REG_RDI] = rp->r_rdi; 448 grp[REG_RSI] = rp->r_rsi; 449 grp[REG_RDX] = rp->r_rdx; 450 grp[REG_RCX] = rp->r_rcx; 451 grp[REG_R8] = rp->r_r8; 452 grp[REG_R9] = rp->r_r9; 453 grp[REG_RAX] = rp->r_rax; 454 grp[REG_RBX] = rp->r_rbx; 455 grp[REG_RBP] = rp->r_rbp; 456 grp[REG_R10] = rp->r_r10; 457 grp[REG_R11] = rp->r_r11; 458 grp[REG_R12] = rp->r_r12; 459 grp[REG_R13] = rp->r_r13; 460 grp[REG_R14] = rp->r_r14; 461 grp[REG_R15] = rp->r_r15; 462 grp[REG_FSBASE] = pcb->pcb_fsbase; 463 grp[REG_GSBASE] = pcb->pcb_gsbase; 464 if (thisthread) 465 kpreempt_disable(); 466 if (pcb->pcb_rupdate == 1) { 467 grp[REG_DS] = pcb->pcb_ds; 468 grp[REG_ES] = pcb->pcb_es; 469 grp[REG_FS] = pcb->pcb_fs; 470 grp[REG_GS] = pcb->pcb_gs; 471 } else { 472 grp[REG_DS] = rp->r_ds; 473 grp[REG_ES] = rp->r_es; 474 grp[REG_FS] = rp->r_fs; 475 grp[REG_GS] = rp->r_gs; 476 } 477 if (thisthread) 478 kpreempt_enable(); 479 grp[REG_TRAPNO] = rp->r_trapno; 480 grp[REG_ERR] = rp->r_err; 481 grp[REG_RIP] = rp->r_rip; 482 grp[REG_CS] = rp->r_cs; 483 grp[REG_SS] = rp->r_ss; 484 grp[REG_RFL] = rp->r_rfl; 485 grp[REG_RSP] = rp->r_rsp; 486 #else 487 bcopy(&rp->r_gs, grp, sizeof (gregset_t)); 488 #endif 489 } 490 491 #if defined(_SYSCALL32_IMPL) 492 493 void 494 getgregs32(klwp_t *lwp, gregset32_t grp) 495 { 496 struct regs *rp = lwptoregs(lwp); 497 struct pcb *pcb = &lwp->lwp_pcb; 498 int thisthread = lwptot(lwp) == curthread; 499 500 if (thisthread) 501 kpreempt_disable(); 502 if (pcb->pcb_rupdate == 1) { 503 grp[GS] = (uint16_t)pcb->pcb_gs; 504 grp[FS] = (uint16_t)pcb->pcb_fs; 505 grp[DS] = (uint16_t)pcb->pcb_ds; 506 grp[ES] = (uint16_t)pcb->pcb_es; 507 } else { 508 grp[GS] = (uint16_t)rp->r_gs; 509 grp[FS] = (uint16_t)rp->r_fs; 510 grp[DS] = (uint16_t)rp->r_ds; 511 grp[ES] = (uint16_t)rp->r_es; 512 } 513 if (thisthread) 514 kpreempt_enable(); 515 grp[EDI] = (greg32_t)rp->r_rdi; 516 grp[ESI] = (greg32_t)rp->r_rsi; 517 grp[EBP] = (greg32_t)rp->r_rbp; 518 grp[ESP] = 0; 519 grp[EBX] = (greg32_t)rp->r_rbx; 520 grp[EDX] = (greg32_t)rp->r_rdx; 521 grp[ECX] = (greg32_t)rp->r_rcx; 522 grp[EAX] = (greg32_t)rp->r_rax; 523 grp[TRAPNO] = (greg32_t)rp->r_trapno; 524 grp[ERR] = (greg32_t)rp->r_err; 525 grp[EIP] = (greg32_t)rp->r_rip; 526 grp[CS] = (uint16_t)rp->r_cs; 527 grp[EFL] = (greg32_t)rp->r_rfl; 528 grp[UESP] = (greg32_t)rp->r_rsp; 529 grp[SS] = (uint16_t)rp->r_ss; 530 } 531 532 void 533 ucontext_32ton(const ucontext32_t *src, ucontext_t *dst) 534 { 535 mcontext_t *dmc = &dst->uc_mcontext; 536 const mcontext32_t *smc = &src->uc_mcontext; 537 538 bzero(dst, sizeof (*dst)); 539 dst->uc_flags = src->uc_flags; 540 dst->uc_link = (ucontext_t *)(uintptr_t)src->uc_link; 541 542 bcopy(&src->uc_sigmask, &dst->uc_sigmask, sizeof (dst->uc_sigmask)); 543 544 dst->uc_stack.ss_sp = (void *)(uintptr_t)src->uc_stack.ss_sp; 545 dst->uc_stack.ss_size = (size_t)src->uc_stack.ss_size; 546 dst->uc_stack.ss_flags = src->uc_stack.ss_flags; 547 548 dmc->gregs[REG_GS] = (greg_t)(uint32_t)smc->gregs[GS]; 549 dmc->gregs[REG_FS] = (greg_t)(uint32_t)smc->gregs[FS]; 550 dmc->gregs[REG_ES] = (greg_t)(uint32_t)smc->gregs[ES]; 551 dmc->gregs[REG_DS] = (greg_t)(uint32_t)smc->gregs[DS]; 552 dmc->gregs[REG_RDI] = (greg_t)(uint32_t)smc->gregs[EDI]; 553 dmc->gregs[REG_RSI] = (greg_t)(uint32_t)smc->gregs[ESI]; 554 dmc->gregs[REG_RBP] = (greg_t)(uint32_t)smc->gregs[EBP]; 555 dmc->gregs[REG_RBX] = (greg_t)(uint32_t)smc->gregs[EBX]; 556 dmc->gregs[REG_RDX] = (greg_t)(uint32_t)smc->gregs[EDX]; 557 dmc->gregs[REG_RCX] = (greg_t)(uint32_t)smc->gregs[ECX]; 558 dmc->gregs[REG_RAX] = (greg_t)(uint32_t)smc->gregs[EAX]; 559 dmc->gregs[REG_TRAPNO] = (greg_t)(uint32_t)smc->gregs[TRAPNO]; 560 dmc->gregs[REG_ERR] = (greg_t)(uint32_t)smc->gregs[ERR]; 561 dmc->gregs[REG_RIP] = (greg_t)(uint32_t)smc->gregs[EIP]; 562 dmc->gregs[REG_CS] = (greg_t)(uint32_t)smc->gregs[CS]; 563 dmc->gregs[REG_RFL] = (greg_t)(uint32_t)smc->gregs[EFL]; 564 dmc->gregs[REG_RSP] = (greg_t)(uint32_t)smc->gregs[UESP]; 565 dmc->gregs[REG_SS] = (greg_t)(uint32_t)smc->gregs[SS]; 566 567 /* 568 * A valid fpregs is only copied in if uc.uc_flags has UC_FPU set 569 * otherwise there is no guarantee that anything in fpregs is valid. 570 */ 571 if (src->uc_flags & UC_FPU) 572 fpregset_32ton(&src->uc_mcontext.fpregs, 573 &dst->uc_mcontext.fpregs); 574 } 575 576 #endif /* _SYSCALL32_IMPL */ 577 578 /* 579 * Return the user-level PC. 580 * If in a system call, return the address of the syscall trap. 581 */ 582 greg_t 583 getuserpc() 584 { 585 greg_t upc = lwptoregs(ttolwp(curthread))->r_pc; 586 uint32_t insn; 587 588 if (curthread->t_sysnum == 0) 589 return (upc); 590 591 /* 592 * We might've gotten here from sysenter (0xf 0x34), 593 * syscall (0xf 0x5) or lcall (0x9a 0 0 0 0 0x27 0). 594 * 595 * Go peek at the binary to figure it out.. 596 */ 597 if (fuword32((void *)(upc - 2), &insn) != -1 && 598 (insn & 0xffff) == 0x340f || (insn & 0xffff) == 0x050f) 599 return (upc - 2); 600 return (upc - 7); 601 } 602 603 /* 604 * Protect segment registers from non-user privilege levels and GDT selectors 605 * other than USER_CS, USER_DS and lwp FS and GS values. If the segment 606 * selector is non-null and not USER_CS/USER_DS, we make sure that the 607 * TI bit is set to point into the LDT and that the RPL is set to 3. 608 * 609 * Since struct regs stores each 16-bit segment register as a 32-bit greg_t, we 610 * also explicitly zero the top 16 bits since they may be coming from the 611 * user's address space via setcontext(2) or /proc. 612 * 613 * Note about null selector. When running on the hypervisor if we allow a 614 * process to set its %cs to null selector with RPL of 0 the hypervisor will 615 * crash the domain. If running on bare metal we would get a #gp fault and 616 * be able to kill the process and continue on. Therefore we make sure to 617 * force RPL to SEL_UPL even for null selector when setting %cs. 618 */ 619 620 #if defined(IS_CS) || defined(IS_NOT_CS) 621 #error "IS_CS and IS_NOT_CS already defined" 622 #endif 623 624 #define IS_CS 1 625 #define IS_NOT_CS 0 626 627 /*ARGSUSED*/ 628 static greg_t 629 fix_segreg(greg_t sr, int iscs, model_t datamodel) 630 { 631 kthread_t *t = curthread; 632 633 switch (sr &= 0xffff) { 634 635 case 0: 636 if (iscs == IS_CS) 637 return (0 | SEL_UPL); 638 else 639 return (0); 640 641 #if defined(__amd64) 642 /* 643 * If lwp attempts to switch data model then force their 644 * code selector to be null selector. 645 */ 646 case U32CS_SEL: 647 if (datamodel == DATAMODEL_NATIVE) 648 return (0 | SEL_UPL); 649 else 650 return (sr); 651 652 case UCS_SEL: 653 if (datamodel == DATAMODEL_ILP32) 654 return (0 | SEL_UPL); 655 #elif defined(__i386) 656 case UCS_SEL: 657 #endif 658 /*FALLTHROUGH*/ 659 case UDS_SEL: 660 case LWPFS_SEL: 661 case LWPGS_SEL: 662 case SEL_UPL: 663 return (sr); 664 default: 665 break; 666 } 667 668 /* 669 * Allow this process's brand to do any necessary segment register 670 * manipulation. 671 */ 672 if (PROC_IS_BRANDED(t->t_procp) && BRMOP(t->t_procp)->b_fixsegreg) { 673 greg_t bsr = BRMOP(t->t_procp)->b_fixsegreg(sr, datamodel); 674 675 if (bsr == 0 && iscs == IS_CS) 676 return (0 | SEL_UPL); 677 else 678 return (bsr); 679 } 680 681 /* 682 * Force it into the LDT in ring 3 for 32-bit processes, which by 683 * default do not have an LDT, so that any attempt to use an invalid 684 * selector will reference the (non-existant) LDT, and cause a #gp 685 * fault for the process. 686 * 687 * 64-bit processes get the null gdt selector since they 688 * are not allowed to have a private LDT. 689 */ 690 #if defined(__amd64) 691 if (datamodel == DATAMODEL_ILP32) { 692 return (sr | SEL_TI_LDT | SEL_UPL); 693 } else { 694 if (iscs == IS_CS) 695 return (0 | SEL_UPL); 696 else 697 return (0); 698 } 699 700 #elif defined(__i386) 701 return (sr | SEL_TI_LDT | SEL_UPL); 702 #endif 703 } 704 705 /* 706 * Set general registers. 707 */ 708 void 709 setgregs(klwp_t *lwp, gregset_t grp) 710 { 711 struct regs *rp = lwptoregs(lwp); 712 model_t datamodel = lwp_getdatamodel(lwp); 713 714 #if defined(__amd64) 715 struct pcb *pcb = &lwp->lwp_pcb; 716 int thisthread = lwptot(lwp) == curthread; 717 718 if (datamodel == DATAMODEL_NATIVE) { 719 720 if (thisthread) 721 (void) save_syscall_args(); /* copy the args */ 722 723 rp->r_rdi = grp[REG_RDI]; 724 rp->r_rsi = grp[REG_RSI]; 725 rp->r_rdx = grp[REG_RDX]; 726 rp->r_rcx = grp[REG_RCX]; 727 rp->r_r8 = grp[REG_R8]; 728 rp->r_r9 = grp[REG_R9]; 729 rp->r_rax = grp[REG_RAX]; 730 rp->r_rbx = grp[REG_RBX]; 731 rp->r_rbp = grp[REG_RBP]; 732 rp->r_r10 = grp[REG_R10]; 733 rp->r_r11 = grp[REG_R11]; 734 rp->r_r12 = grp[REG_R12]; 735 rp->r_r13 = grp[REG_R13]; 736 rp->r_r14 = grp[REG_R14]; 737 rp->r_r15 = grp[REG_R15]; 738 rp->r_trapno = grp[REG_TRAPNO]; 739 rp->r_err = grp[REG_ERR]; 740 rp->r_rip = grp[REG_RIP]; 741 /* 742 * Setting %cs or %ss to anything else is quietly but 743 * quite definitely forbidden! 744 */ 745 rp->r_cs = UCS_SEL; 746 rp->r_ss = UDS_SEL; 747 rp->r_rsp = grp[REG_RSP]; 748 749 if (thisthread) 750 kpreempt_disable(); 751 752 pcb->pcb_ds = UDS_SEL; 753 pcb->pcb_es = UDS_SEL; 754 755 /* 756 * 64-bit processes -are- allowed to set their fsbase/gsbase 757 * values directly, but only if they're using the segment 758 * selectors that allow that semantic. 759 * 760 * (32-bit processes must use lwp_set_private().) 761 */ 762 pcb->pcb_fsbase = grp[REG_FSBASE]; 763 pcb->pcb_gsbase = grp[REG_GSBASE]; 764 pcb->pcb_fs = fix_segreg(grp[REG_FS], IS_NOT_CS, datamodel); 765 pcb->pcb_gs = fix_segreg(grp[REG_GS], IS_NOT_CS, datamodel); 766 767 /* 768 * Ensure that we go out via update_sregs 769 */ 770 pcb->pcb_rupdate = 1; 771 lwptot(lwp)->t_post_sys = 1; 772 if (thisthread) 773 kpreempt_enable(); 774 #if defined(_SYSCALL32_IMPL) 775 } else { 776 rp->r_rdi = (uint32_t)grp[REG_RDI]; 777 rp->r_rsi = (uint32_t)grp[REG_RSI]; 778 rp->r_rdx = (uint32_t)grp[REG_RDX]; 779 rp->r_rcx = (uint32_t)grp[REG_RCX]; 780 rp->r_rax = (uint32_t)grp[REG_RAX]; 781 rp->r_rbx = (uint32_t)grp[REG_RBX]; 782 rp->r_rbp = (uint32_t)grp[REG_RBP]; 783 rp->r_trapno = (uint32_t)grp[REG_TRAPNO]; 784 rp->r_err = (uint32_t)grp[REG_ERR]; 785 rp->r_rip = (uint32_t)grp[REG_RIP]; 786 787 rp->r_cs = fix_segreg(grp[REG_CS], IS_CS, datamodel); 788 rp->r_ss = fix_segreg(grp[REG_DS], IS_NOT_CS, datamodel); 789 790 rp->r_rsp = (uint32_t)grp[REG_RSP]; 791 792 if (thisthread) 793 kpreempt_disable(); 794 795 pcb->pcb_ds = fix_segreg(grp[REG_DS], IS_NOT_CS, datamodel); 796 pcb->pcb_es = fix_segreg(grp[REG_ES], IS_NOT_CS, datamodel); 797 798 /* 799 * (See fsbase/gsbase commentary above) 800 */ 801 pcb->pcb_fs = fix_segreg(grp[REG_FS], IS_NOT_CS, datamodel); 802 pcb->pcb_gs = fix_segreg(grp[REG_GS], IS_NOT_CS, datamodel); 803 804 /* 805 * Ensure that we go out via update_sregs 806 */ 807 pcb->pcb_rupdate = 1; 808 lwptot(lwp)->t_post_sys = 1; 809 if (thisthread) 810 kpreempt_enable(); 811 #endif 812 } 813 814 /* 815 * Only certain bits of the flags register can be modified. 816 */ 817 rp->r_rfl = (rp->r_rfl & ~PSL_USERMASK) | 818 (grp[REG_RFL] & PSL_USERMASK); 819 820 #elif defined(__i386) 821 822 /* 823 * Only certain bits of the flags register can be modified. 824 */ 825 grp[EFL] = (rp->r_efl & ~PSL_USERMASK) | (grp[EFL] & PSL_USERMASK); 826 827 /* 828 * Copy saved registers from user stack. 829 */ 830 bcopy(grp, &rp->r_gs, sizeof (gregset_t)); 831 832 rp->r_cs = fix_segreg(rp->r_cs, IS_CS, datamodel); 833 rp->r_ss = fix_segreg(rp->r_ss, IS_NOT_CS, datamodel); 834 rp->r_ds = fix_segreg(rp->r_ds, IS_NOT_CS, datamodel); 835 rp->r_es = fix_segreg(rp->r_es, IS_NOT_CS, datamodel); 836 rp->r_fs = fix_segreg(rp->r_fs, IS_NOT_CS, datamodel); 837 rp->r_gs = fix_segreg(rp->r_gs, IS_NOT_CS, datamodel); 838 839 #endif /* __i386 */ 840 } 841 842 /* 843 * Determine whether eip is likely to have an interrupt frame 844 * on the stack. We do this by comparing the address to the 845 * range of addresses spanned by several well-known routines. 846 */ 847 extern void _interrupt(); 848 extern void _allsyscalls(); 849 extern void _cmntrap(); 850 extern void fakesoftint(); 851 852 extern size_t _interrupt_size; 853 extern size_t _allsyscalls_size; 854 extern size_t _cmntrap_size; 855 extern size_t _fakesoftint_size; 856 857 /* 858 * Get a pc-only stacktrace. Used for kmem_alloc() buffer ownership tracking. 859 * Returns MIN(current stack depth, pcstack_limit). 860 */ 861 int 862 getpcstack(pc_t *pcstack, int pcstack_limit) 863 { 864 struct frame *fp = (struct frame *)getfp(); 865 struct frame *nextfp, *minfp, *stacktop; 866 int depth = 0; 867 int on_intr; 868 uintptr_t pc; 869 870 if ((on_intr = CPU_ON_INTR(CPU)) != 0) 871 stacktop = (struct frame *)(CPU->cpu_intr_stack + SA(MINFRAME)); 872 else 873 stacktop = (struct frame *)curthread->t_stk; 874 minfp = fp; 875 876 pc = ((struct regs *)fp)->r_pc; 877 878 while (depth < pcstack_limit) { 879 nextfp = (struct frame *)fp->fr_savfp; 880 pc = fp->fr_savpc; 881 if (nextfp <= minfp || nextfp >= stacktop) { 882 if (on_intr) { 883 /* 884 * Hop from interrupt stack to thread stack. 885 */ 886 stacktop = (struct frame *)curthread->t_stk; 887 minfp = (struct frame *)curthread->t_stkbase; 888 on_intr = 0; 889 continue; 890 } 891 break; 892 } 893 pcstack[depth++] = (pc_t)pc; 894 fp = nextfp; 895 minfp = fp; 896 } 897 return (depth); 898 } 899 900 /* 901 * The following ELF header fields are defined as processor-specific 902 * in the V8 ABI: 903 * 904 * e_ident[EI_DATA] encoding of the processor-specific 905 * data in the object file 906 * e_machine processor identification 907 * e_flags processor-specific flags associated 908 * with the file 909 */ 910 911 /* 912 * The value of at_flags reflects a platform's cpu module support. 913 * at_flags is used to check for allowing a binary to execute and 914 * is passed as the value of the AT_FLAGS auxiliary vector. 915 */ 916 int at_flags = 0; 917 918 /* 919 * Check the processor-specific fields of an ELF header. 920 * 921 * returns 1 if the fields are valid, 0 otherwise 922 */ 923 /*ARGSUSED2*/ 924 int 925 elfheadcheck( 926 unsigned char e_data, 927 Elf32_Half e_machine, 928 Elf32_Word e_flags) 929 { 930 if (e_data != ELFDATA2LSB) 931 return (0); 932 #if defined(__amd64) 933 if (e_machine == EM_AMD64) 934 return (1); 935 #endif 936 return (e_machine == EM_386); 937 } 938 939 uint_t auxv_hwcap_include = 0; /* patch to enable unrecognized features */ 940 uint_t auxv_hwcap_exclude = 0; /* patch for broken cpus, debugging */ 941 #if defined(_SYSCALL32_IMPL) 942 uint_t auxv_hwcap32_include = 0; /* ditto for 32-bit apps */ 943 uint_t auxv_hwcap32_exclude = 0; /* ditto for 32-bit apps */ 944 #endif 945 946 /* 947 * Gather information about the processor and place it into auxv_hwcap 948 * so that it can be exported to the linker via the aux vector. 949 * 950 * We use this seemingly complicated mechanism so that we can ensure 951 * that /etc/system can be used to override what the system can or 952 * cannot discover for itself. 953 */ 954 void 955 bind_hwcap(void) 956 { 957 uint_t cpu_hwcap_flags = cpuid_pass4(NULL); 958 959 auxv_hwcap = (auxv_hwcap_include | cpu_hwcap_flags) & 960 ~auxv_hwcap_exclude; 961 962 #if defined(__amd64) 963 /* 964 * On AMD processors, sysenter just doesn't work at all 965 * when the kernel is in long mode. On IA-32e processors 966 * it does, but there's no real point in all the alternate 967 * mechanism when syscall works on both. 968 * 969 * Besides, the kernel's sysenter handler is expecting a 970 * 32-bit lwp ... 971 */ 972 auxv_hwcap &= ~AV_386_SEP; 973 #else 974 /* 975 * 32-bit processes can -always- use the lahf/sahf instructions 976 */ 977 auxv_hwcap |= AV_386_AHF; 978 #endif 979 980 if (auxv_hwcap_include || auxv_hwcap_exclude) 981 cmn_err(CE_CONT, "?user ABI extensions: %b\n", 982 auxv_hwcap, FMT_AV_386); 983 984 #if defined(_SYSCALL32_IMPL) 985 auxv_hwcap32 = (auxv_hwcap32_include | cpu_hwcap_flags) & 986 ~auxv_hwcap32_exclude; 987 988 #if defined(__amd64) 989 /* 990 * If this is an amd64 architecture machine from Intel, then 991 * syscall -doesn't- work in compatibility mode, only sysenter does. 992 * 993 * Sigh. 994 */ 995 if (!cpuid_syscall32_insn(NULL)) 996 auxv_hwcap32 &= ~AV_386_AMD_SYSC; 997 998 /* 999 * 32-bit processes can -always- use the lahf/sahf instructions 1000 */ 1001 auxv_hwcap32 |= AV_386_AHF; 1002 #endif 1003 1004 if (auxv_hwcap32_include || auxv_hwcap32_exclude) 1005 cmn_err(CE_CONT, "?32-bit user ABI extensions: %b\n", 1006 auxv_hwcap32, FMT_AV_386); 1007 #endif 1008 } 1009 1010 /* 1011 * sync_icache() - this is called 1012 * in proc/fs/prusrio.c. x86 has an unified cache and therefore 1013 * this is a nop. 1014 */ 1015 /* ARGSUSED */ 1016 void 1017 sync_icache(caddr_t addr, uint_t len) 1018 { 1019 /* Do nothing for now */ 1020 } 1021 1022 /*ARGSUSED*/ 1023 void 1024 sync_data_memory(caddr_t va, size_t len) 1025 { 1026 /* Not implemented for this platform */ 1027 } 1028 1029 int 1030 __ipltospl(int ipl) 1031 { 1032 return (ipltospl(ipl)); 1033 } 1034 1035 /* 1036 * The panic code invokes panic_saveregs() to record the contents of a 1037 * regs structure into the specified panic_data structure for debuggers. 1038 */ 1039 void 1040 panic_saveregs(panic_data_t *pdp, struct regs *rp) 1041 { 1042 panic_nv_t *pnv = PANICNVGET(pdp); 1043 1044 struct cregs creg; 1045 1046 getcregs(&creg); 1047 1048 #if defined(__amd64) 1049 PANICNVADD(pnv, "rdi", rp->r_rdi); 1050 PANICNVADD(pnv, "rsi", rp->r_rsi); 1051 PANICNVADD(pnv, "rdx", rp->r_rdx); 1052 PANICNVADD(pnv, "rcx", rp->r_rcx); 1053 PANICNVADD(pnv, "r8", rp->r_r8); 1054 PANICNVADD(pnv, "r9", rp->r_r9); 1055 PANICNVADD(pnv, "rax", rp->r_rax); 1056 PANICNVADD(pnv, "rbx", rp->r_rbx); 1057 PANICNVADD(pnv, "rbp", rp->r_rbp); 1058 PANICNVADD(pnv, "r10", rp->r_r10); 1059 PANICNVADD(pnv, "r10", rp->r_r10); 1060 PANICNVADD(pnv, "r11", rp->r_r11); 1061 PANICNVADD(pnv, "r12", rp->r_r12); 1062 PANICNVADD(pnv, "r13", rp->r_r13); 1063 PANICNVADD(pnv, "r14", rp->r_r14); 1064 PANICNVADD(pnv, "r15", rp->r_r15); 1065 PANICNVADD(pnv, "fsbase", rdmsr(MSR_AMD_FSBASE)); 1066 PANICNVADD(pnv, "gsbase", rdmsr(MSR_AMD_GSBASE)); 1067 PANICNVADD(pnv, "ds", rp->r_ds); 1068 PANICNVADD(pnv, "es", rp->r_es); 1069 PANICNVADD(pnv, "fs", rp->r_fs); 1070 PANICNVADD(pnv, "gs", rp->r_gs); 1071 PANICNVADD(pnv, "trapno", rp->r_trapno); 1072 PANICNVADD(pnv, "err", rp->r_err); 1073 PANICNVADD(pnv, "rip", rp->r_rip); 1074 PANICNVADD(pnv, "cs", rp->r_cs); 1075 PANICNVADD(pnv, "rflags", rp->r_rfl); 1076 PANICNVADD(pnv, "rsp", rp->r_rsp); 1077 PANICNVADD(pnv, "ss", rp->r_ss); 1078 PANICNVADD(pnv, "gdt_hi", (uint64_t)(creg.cr_gdt._l[3])); 1079 PANICNVADD(pnv, "gdt_lo", (uint64_t)(creg.cr_gdt._l[0])); 1080 PANICNVADD(pnv, "idt_hi", (uint64_t)(creg.cr_idt._l[3])); 1081 PANICNVADD(pnv, "idt_lo", (uint64_t)(creg.cr_idt._l[0])); 1082 #elif defined(__i386) 1083 PANICNVADD(pnv, "gs", (uint32_t)rp->r_gs); 1084 PANICNVADD(pnv, "fs", (uint32_t)rp->r_fs); 1085 PANICNVADD(pnv, "es", (uint32_t)rp->r_es); 1086 PANICNVADD(pnv, "ds", (uint32_t)rp->r_ds); 1087 PANICNVADD(pnv, "edi", (uint32_t)rp->r_edi); 1088 PANICNVADD(pnv, "esi", (uint32_t)rp->r_esi); 1089 PANICNVADD(pnv, "ebp", (uint32_t)rp->r_ebp); 1090 PANICNVADD(pnv, "esp", (uint32_t)rp->r_esp); 1091 PANICNVADD(pnv, "ebx", (uint32_t)rp->r_ebx); 1092 PANICNVADD(pnv, "edx", (uint32_t)rp->r_edx); 1093 PANICNVADD(pnv, "ecx", (uint32_t)rp->r_ecx); 1094 PANICNVADD(pnv, "eax", (uint32_t)rp->r_eax); 1095 PANICNVADD(pnv, "trapno", (uint32_t)rp->r_trapno); 1096 PANICNVADD(pnv, "err", (uint32_t)rp->r_err); 1097 PANICNVADD(pnv, "eip", (uint32_t)rp->r_eip); 1098 PANICNVADD(pnv, "cs", (uint32_t)rp->r_cs); 1099 PANICNVADD(pnv, "eflags", (uint32_t)rp->r_efl); 1100 PANICNVADD(pnv, "uesp", (uint32_t)rp->r_uesp); 1101 PANICNVADD(pnv, "ss", (uint32_t)rp->r_ss); 1102 PANICNVADD(pnv, "gdt", creg.cr_gdt); 1103 PANICNVADD(pnv, "idt", creg.cr_idt); 1104 #endif /* __i386 */ 1105 1106 PANICNVADD(pnv, "ldt", creg.cr_ldt); 1107 PANICNVADD(pnv, "task", creg.cr_task); 1108 PANICNVADD(pnv, "cr0", creg.cr_cr0); 1109 PANICNVADD(pnv, "cr2", creg.cr_cr2); 1110 PANICNVADD(pnv, "cr3", creg.cr_cr3); 1111 if (creg.cr_cr4) 1112 PANICNVADD(pnv, "cr4", creg.cr_cr4); 1113 1114 PANICNVSET(pdp, pnv); 1115 } 1116 1117 #define TR_ARG_MAX 6 /* Max args to print, same as SPARC */ 1118 1119 #if !defined(__amd64) 1120 1121 /* 1122 * Given a return address (%eip), determine the likely number of arguments 1123 * that were pushed on the stack prior to its execution. We do this by 1124 * expecting that a typical call sequence consists of pushing arguments on 1125 * the stack, executing a call instruction, and then performing an add 1126 * on %esp to restore it to the value prior to pushing the arguments for 1127 * the call. We attempt to detect such an add, and divide the addend 1128 * by the size of a word to determine the number of pushed arguments. 1129 * 1130 * If we do not find such an add, we punt and return TR_ARG_MAX. It is not 1131 * possible to reliably determine if a function took no arguments (i.e. was 1132 * void) because assembler routines do not reliably perform an add on %esp 1133 * immediately upon returning (eg. _sys_call()), so returning TR_ARG_MAX is 1134 * safer than returning 0. 1135 */ 1136 static ulong_t 1137 argcount(uintptr_t eip) 1138 { 1139 const uint8_t *ins = (const uint8_t *)eip; 1140 ulong_t n; 1141 1142 enum { 1143 M_MODRM_ESP = 0xc4, /* Mod/RM byte indicates %esp */ 1144 M_ADD_IMM32 = 0x81, /* ADD imm32 to r/m32 */ 1145 M_ADD_IMM8 = 0x83 /* ADD imm8 to r/m32 */ 1146 }; 1147 1148 if (eip < KERNELBASE || ins[1] != M_MODRM_ESP) 1149 return (TR_ARG_MAX); 1150 1151 switch (ins[0]) { 1152 case M_ADD_IMM32: 1153 n = ins[2] + (ins[3] << 8) + (ins[4] << 16) + (ins[5] << 24); 1154 break; 1155 1156 case M_ADD_IMM8: 1157 n = ins[2]; 1158 break; 1159 1160 default: 1161 return (TR_ARG_MAX); 1162 } 1163 1164 n /= sizeof (long); 1165 return (MIN(n, TR_ARG_MAX)); 1166 } 1167 1168 #endif /* !__amd64 */ 1169 1170 /* 1171 * Print a stack backtrace using the specified frame pointer. We delay two 1172 * seconds before continuing, unless this is the panic traceback. 1173 * If we are in the process of panicking, we also attempt to write the 1174 * stack backtrace to a staticly assigned buffer, to allow the panic 1175 * code to find it and write it in to uncompressed pages within the 1176 * system crash dump. 1177 * Note that the frame for the starting stack pointer value is omitted because 1178 * the corresponding %eip is not known. 1179 */ 1180 1181 extern char *dump_stack_scratch; 1182 1183 #if defined(__amd64) 1184 1185 void 1186 traceback(caddr_t fpreg) 1187 { 1188 struct frame *fp = (struct frame *)fpreg; 1189 struct frame *nextfp; 1190 uintptr_t pc, nextpc; 1191 ulong_t off; 1192 char args[TR_ARG_MAX * 2 + 16], *sym; 1193 uint_t offset = 0; 1194 uint_t next_offset = 0; 1195 char stack_buffer[1024]; 1196 1197 if (!panicstr) 1198 printf("traceback: %%fp = %p\n", (void *)fp); 1199 1200 if (panicstr && !dump_stack_scratch) { 1201 printf("Warning - stack not written to the dump buffer\n"); 1202 } 1203 1204 fp = (struct frame *)plat_traceback(fpreg); 1205 if ((uintptr_t)fp < KERNELBASE) 1206 goto out; 1207 1208 pc = fp->fr_savpc; 1209 fp = (struct frame *)fp->fr_savfp; 1210 1211 while ((uintptr_t)fp >= KERNELBASE) { 1212 /* 1213 * XX64 Until port is complete tolerate 8-byte aligned 1214 * frame pointers but flag with a warning so they can 1215 * be fixed. 1216 */ 1217 if (((uintptr_t)fp & (STACK_ALIGN - 1)) != 0) { 1218 if (((uintptr_t)fp & (8 - 1)) == 0) { 1219 printf(" >> warning! 8-byte" 1220 " aligned %%fp = %p\n", (void *)fp); 1221 } else { 1222 printf( 1223 " >> mis-aligned %%fp = %p\n", (void *)fp); 1224 break; 1225 } 1226 } 1227 1228 args[0] = '\0'; 1229 nextpc = (uintptr_t)fp->fr_savpc; 1230 nextfp = (struct frame *)fp->fr_savfp; 1231 if ((sym = kobj_getsymname(pc, &off)) != NULL) { 1232 printf("%016lx %s:%s+%lx (%s)\n", (uintptr_t)fp, 1233 mod_containing_pc((caddr_t)pc), sym, off, args); 1234 (void) snprintf(stack_buffer, sizeof (stack_buffer), 1235 "%s:%s+%lx (%s) | ", 1236 mod_containing_pc((caddr_t)pc), sym, off, args); 1237 } else { 1238 printf("%016lx %lx (%s)\n", 1239 (uintptr_t)fp, pc, args); 1240 (void) snprintf(stack_buffer, sizeof (stack_buffer), 1241 "%lx (%s) | ", pc, args); 1242 } 1243 1244 if (panicstr && dump_stack_scratch) { 1245 next_offset = offset + strlen(stack_buffer); 1246 if (next_offset < STACK_BUF_SIZE) { 1247 bcopy(stack_buffer, dump_stack_scratch + offset, 1248 strlen(stack_buffer)); 1249 offset = next_offset; 1250 } else { 1251 /* 1252 * In attempting to save the panic stack 1253 * to the dumpbuf we have overflowed that area. 1254 * Print a warning and continue to printf the 1255 * stack to the msgbuf 1256 */ 1257 printf("Warning: stack in the dump buffer" 1258 " may be incomplete\n"); 1259 offset = next_offset; 1260 } 1261 } 1262 1263 pc = nextpc; 1264 fp = nextfp; 1265 } 1266 out: 1267 if (!panicstr) { 1268 printf("end of traceback\n"); 1269 DELAY(2 * MICROSEC); 1270 } else if (dump_stack_scratch) { 1271 dump_stack_scratch[offset] = '\0'; 1272 } 1273 } 1274 1275 #elif defined(__i386) 1276 1277 void 1278 traceback(caddr_t fpreg) 1279 { 1280 struct frame *fp = (struct frame *)fpreg; 1281 struct frame *nextfp, *minfp, *stacktop; 1282 uintptr_t pc, nextpc; 1283 uint_t offset = 0; 1284 uint_t next_offset = 0; 1285 char stack_buffer[1024]; 1286 1287 cpu_t *cpu; 1288 1289 /* 1290 * args[] holds TR_ARG_MAX hex long args, plus ", " or '\0'. 1291 */ 1292 char args[TR_ARG_MAX * 2 + 8], *p; 1293 1294 int on_intr; 1295 ulong_t off; 1296 char *sym; 1297 1298 if (!panicstr) 1299 printf("traceback: %%fp = %p\n", (void *)fp); 1300 1301 if (panicstr && !dump_stack_scratch) { 1302 printf("Warning - stack not written to the dumpbuf\n"); 1303 } 1304 1305 /* 1306 * If we are panicking, all high-level interrupt information in 1307 * CPU was overwritten. panic_cpu has the correct values. 1308 */ 1309 kpreempt_disable(); /* prevent migration */ 1310 1311 cpu = (panicstr && CPU->cpu_id == panic_cpu.cpu_id)? &panic_cpu : CPU; 1312 1313 if ((on_intr = CPU_ON_INTR(cpu)) != 0) 1314 stacktop = (struct frame *)(cpu->cpu_intr_stack + SA(MINFRAME)); 1315 else 1316 stacktop = (struct frame *)curthread->t_stk; 1317 1318 kpreempt_enable(); 1319 1320 fp = (struct frame *)plat_traceback(fpreg); 1321 if ((uintptr_t)fp < KERNELBASE) 1322 goto out; 1323 1324 minfp = fp; /* Baseline minimum frame pointer */ 1325 pc = fp->fr_savpc; 1326 fp = (struct frame *)fp->fr_savfp; 1327 1328 while ((uintptr_t)fp >= KERNELBASE) { 1329 ulong_t argc; 1330 long *argv; 1331 1332 if (fp <= minfp || fp >= stacktop) { 1333 if (on_intr) { 1334 /* 1335 * Hop from interrupt stack to thread stack. 1336 */ 1337 stacktop = (struct frame *)curthread->t_stk; 1338 minfp = (struct frame *)curthread->t_stkbase; 1339 on_intr = 0; 1340 continue; 1341 } 1342 break; /* we're outside of the expected range */ 1343 } 1344 1345 if ((uintptr_t)fp & (STACK_ALIGN - 1)) { 1346 printf(" >> mis-aligned %%fp = %p\n", (void *)fp); 1347 break; 1348 } 1349 1350 nextpc = fp->fr_savpc; 1351 nextfp = (struct frame *)fp->fr_savfp; 1352 argc = argcount(nextpc); 1353 argv = (long *)((char *)fp + sizeof (struct frame)); 1354 1355 args[0] = '\0'; 1356 p = args; 1357 while (argc-- > 0 && argv < (long *)stacktop) { 1358 p += snprintf(p, args + sizeof (args) - p, 1359 "%s%lx", (p == args) ? "" : ", ", *argv++); 1360 } 1361 1362 if ((sym = kobj_getsymname(pc, &off)) != NULL) { 1363 printf("%08lx %s:%s+%lx (%s)\n", (uintptr_t)fp, 1364 mod_containing_pc((caddr_t)pc), sym, off, args); 1365 (void) snprintf(stack_buffer, sizeof (stack_buffer), 1366 "%s:%s+%lx (%s) | ", 1367 mod_containing_pc((caddr_t)pc), sym, off, args); 1368 1369 } else { 1370 printf("%08lx %lx (%s)\n", 1371 (uintptr_t)fp, pc, args); 1372 (void) snprintf(stack_buffer, sizeof (stack_buffer), 1373 "%lx (%s) | ", pc, args); 1374 1375 } 1376 1377 if (panicstr && dump_stack_scratch) { 1378 next_offset = offset + strlen(stack_buffer); 1379 if (next_offset < STACK_BUF_SIZE) { 1380 bcopy(stack_buffer, dump_stack_scratch + offset, 1381 strlen(stack_buffer)); 1382 offset = next_offset; 1383 } else { 1384 /* 1385 * In attempting to save the panic stack 1386 * to the dumpbuf we have overflowed that area. 1387 * Print a warning and continue to printf the 1388 * stack to the msgbuf 1389 */ 1390 printf("Warning: stack in the dumpbuf" 1391 " may be incomplete\n"); 1392 offset = next_offset; 1393 } 1394 } 1395 1396 minfp = fp; 1397 pc = nextpc; 1398 fp = nextfp; 1399 } 1400 out: 1401 if (!panicstr) { 1402 printf("end of traceback\n"); 1403 DELAY(2 * MICROSEC); 1404 } else if (dump_stack_scratch) { 1405 dump_stack_scratch[offset] = '\0'; 1406 } 1407 1408 } 1409 1410 #endif /* __i386 */ 1411 1412 /* 1413 * Generate a stack backtrace from a saved register set. 1414 */ 1415 void 1416 traceregs(struct regs *rp) 1417 { 1418 traceback((caddr_t)rp->r_fp); 1419 } 1420 1421 void 1422 exec_set_sp(size_t stksize) 1423 { 1424 klwp_t *lwp = ttolwp(curthread); 1425 1426 lwptoregs(lwp)->r_sp = (uintptr_t)curproc->p_usrstack - stksize; 1427 } 1428 1429 hrtime_t 1430 gethrtime_waitfree(void) 1431 { 1432 return (dtrace_gethrtime()); 1433 } 1434 1435 hrtime_t 1436 gethrtime(void) 1437 { 1438 return (gethrtimef()); 1439 } 1440 1441 hrtime_t 1442 gethrtime_unscaled(void) 1443 { 1444 return (gethrtimeunscaledf()); 1445 } 1446 1447 void 1448 scalehrtime(hrtime_t *hrt) 1449 { 1450 scalehrtimef(hrt); 1451 } 1452 1453 uint64_t 1454 unscalehrtime(hrtime_t nsecs) 1455 { 1456 return (unscalehrtimef(nsecs)); 1457 } 1458 1459 void 1460 gethrestime(timespec_t *tp) 1461 { 1462 gethrestimef(tp); 1463 } 1464 1465 #if defined(__amd64) 1466 /* 1467 * Part of the implementation of hres_tick(); this routine is 1468 * easier in C than assembler .. called with the hres_lock held. 1469 * 1470 * XX64 Many of these timekeeping variables need to be extern'ed in a header 1471 */ 1472 1473 #include <sys/time.h> 1474 #include <sys/machlock.h> 1475 1476 extern int one_sec; 1477 extern int max_hres_adj; 1478 1479 void 1480 __adj_hrestime(void) 1481 { 1482 long long adj; 1483 1484 if (hrestime_adj == 0) 1485 adj = 0; 1486 else if (hrestime_adj > 0) { 1487 if (hrestime_adj < max_hres_adj) 1488 adj = hrestime_adj; 1489 else 1490 adj = max_hres_adj; 1491 } else { 1492 if (hrestime_adj < -max_hres_adj) 1493 adj = -max_hres_adj; 1494 else 1495 adj = hrestime_adj; 1496 } 1497 1498 timedelta -= adj; 1499 hrestime_adj = timedelta; 1500 hrestime.tv_nsec += adj; 1501 1502 while (hrestime.tv_nsec >= NANOSEC) { 1503 one_sec++; 1504 hrestime.tv_sec++; 1505 hrestime.tv_nsec -= NANOSEC; 1506 } 1507 } 1508 #endif 1509 1510 /* 1511 * Wrapper functions to maintain backwards compability 1512 */ 1513 int 1514 xcopyin(const void *uaddr, void *kaddr, size_t count) 1515 { 1516 return (xcopyin_nta(uaddr, kaddr, count, UIO_COPY_CACHED)); 1517 } 1518 1519 int 1520 xcopyout(const void *kaddr, void *uaddr, size_t count) 1521 { 1522 return (xcopyout_nta(kaddr, uaddr, count, UIO_COPY_CACHED)); 1523 }