1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24 
  25 /*      Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T     */
  26 /*        All Rights Reserved   */
  27 
  28 #include <sys/param.h>
  29 #include <sys/types.h>
  30 #include <sys/vmparam.h>
  31 #include <sys/systm.h>
  32 #include <sys/signal.h>
  33 #include <sys/stack.h>
  34 #include <sys/regset.h>
  35 #include <sys/privregs.h>
  36 #include <sys/frame.h>
  37 #include <sys/proc.h>
  38 #include <sys/psw.h>
  39 #include <sys/siginfo.h>
  40 #include <sys/cpuvar.h>
  41 #include <sys/asm_linkage.h>
  42 #include <sys/kmem.h>
  43 #include <sys/errno.h>
  44 #include <sys/bootconf.h>
  45 #include <sys/archsystm.h>
  46 #include <sys/debug.h>
  47 #include <sys/elf.h>
  48 #include <sys/spl.h>
  49 #include <sys/time.h>
  50 #include <sys/atomic.h>
  51 #include <sys/sysmacros.h>
  52 #include <sys/cmn_err.h>
  53 #include <sys/modctl.h>
  54 #include <sys/kobj.h>
  55 #include <sys/panic.h>
  56 #include <sys/reboot.h>
  57 #include <sys/time.h>
  58 #include <sys/fp.h>
  59 #include <sys/x86_archext.h>
  60 #include <sys/auxv.h>
  61 #include <sys/auxv_386.h>
  62 #include <sys/dtrace.h>
  63 #include <sys/brand.h>
  64 #include <sys/machbrand.h>
  65 #include <sys/cmn_err.h>
  66 
  67 extern const struct fnsave_state x87_initial;
  68 extern const struct fxsave_state sse_initial;
  69 
  70 /*
  71  * Map an fnsave-formatted save area into an fxsave-formatted save area.
  72  *
  73  * Most fields are the same width, content and semantics.  However
  74  * the tag word is compressed.
  75  */
  76 static void
  77 fnsave_to_fxsave(const struct fnsave_state *fn, struct fxsave_state *fx)
  78 {
  79         uint_t i, tagbits;
  80 
  81         fx->fx_fcw = fn->f_fcw;
  82         fx->fx_fsw = fn->f_fsw;
  83 
  84         /*
  85          * copy element by element (because of holes)
  86          */
  87         for (i = 0; i < 8; i++)
  88                 bcopy(&fn->f_st[i].fpr_16[0], &fx->fx_st[i].fpr_16[0],
  89                     sizeof (fn->f_st[0].fpr_16)); /* 80-bit x87-style floats */
  90 
  91         /*
  92          * synthesize compressed tag bits
  93          */
  94         fx->fx_fctw = 0;
  95         for (tagbits = fn->f_ftw, i = 0; i < 8; i++, tagbits >>= 2)
  96                 if ((tagbits & 3) != 3)
  97                         fx->fx_fctw |= (1 << i);
  98 
  99         fx->fx_fop = fn->f_fop;
 100 
 101 #if defined(__amd64)
 102         fx->fx_rip = (uint64_t)fn->f_eip;
 103         fx->fx_rdp = (uint64_t)fn->f_dp;
 104 #else
 105         fx->fx_eip = fn->f_eip;
 106         fx->fx_cs = fn->f_cs;
 107         fx->__fx_ign0 = 0;
 108         fx->fx_dp = fn->f_dp;
 109         fx->fx_ds = fn->f_ds;
 110         fx->__fx_ign1 = 0;
 111 #endif
 112 }
 113 
 114 /*
 115  * Map from an fxsave-format save area to an fnsave-format save area.
 116  */
 117 static void
 118 fxsave_to_fnsave(const struct fxsave_state *fx, struct fnsave_state *fn)
 119 {
 120         uint_t i, top, tagbits;
 121 
 122         fn->f_fcw = fx->fx_fcw;
 123         fn->__f_ign0 = 0;
 124         fn->f_fsw = fx->fx_fsw;
 125         fn->__f_ign1 = 0;
 126 
 127         top = (fx->fx_fsw & FPS_TOP) >> 11;
 128 
 129         /*
 130          * copy element by element (because of holes)
 131          */
 132         for (i = 0; i < 8; i++)
 133                 bcopy(&fx->fx_st[i].fpr_16[0], &fn->f_st[i].fpr_16[0],
 134                     sizeof (fn->f_st[0].fpr_16)); /* 80-bit x87-style floats */
 135 
 136         /*
 137          * synthesize uncompressed tag bits
 138          */
 139         fn->f_ftw = 0;
 140         for (tagbits = fx->fx_fctw, i = 0; i < 8; i++, tagbits >>= 1) {
 141                 uint_t ibit, expo;
 142                 const uint16_t *fpp;
 143                 static const uint16_t zero[5] = { 0, 0, 0, 0, 0 };
 144 
 145                 if ((tagbits & 1) == 0) {
 146                         fn->f_ftw |= 3 << (i << 1);      /* empty */
 147                         continue;
 148                 }
 149 
 150                 /*
 151                  * (tags refer to *physical* registers)
 152                  */
 153                 fpp = &fx->fx_st[(i - top + 8) & 7].fpr_16[0];
 154                 ibit = fpp[3] >> 15;
 155                 expo = fpp[4] & 0x7fff;
 156 
 157                 if (ibit && expo != 0 && expo != 0x7fff)
 158                         continue;                       /* valid fp number */
 159 
 160                 if (bcmp(fpp, &zero, sizeof (zero)))
 161                         fn->f_ftw |= 2 << (i << 1);      /* NaN */
 162                 else
 163                         fn->f_ftw |= 1 << (i << 1);      /* fp zero */
 164         }
 165 
 166         fn->f_fop = fx->fx_fop;
 167 
 168         fn->__f_ign2 = 0;
 169 #if defined(__amd64)
 170         fn->f_eip = (uint32_t)fx->fx_rip;
 171         fn->f_cs = U32CS_SEL;
 172         fn->f_dp = (uint32_t)fx->fx_rdp;
 173         fn->f_ds = UDS_SEL;
 174 #else
 175         fn->f_eip = fx->fx_eip;
 176         fn->f_cs = fx->fx_cs;
 177         fn->f_dp = fx->fx_dp;
 178         fn->f_ds = fx->fx_ds;
 179 #endif
 180         fn->__f_ign3 = 0;
 181 }
 182 
 183 /*
 184  * Map from an fpregset_t into an fxsave-format save area
 185  */
 186 static void
 187 fpregset_to_fxsave(const fpregset_t *fp, struct fxsave_state *fx)
 188 {
 189 #if defined(__amd64)
 190         bcopy(fp, fx, sizeof (*fx));
 191 #else
 192         const struct fpchip_state *fc = &fp->fp_reg_set.fpchip_state;
 193 
 194         fnsave_to_fxsave((const struct fnsave_state *)fc, fx);
 195         fx->fx_mxcsr = fc->mxcsr;
 196         bcopy(&fc->xmm[0], &fx->fx_xmm[0], sizeof (fc->xmm));
 197 #endif
 198         /*
 199          * avoid useless #gp exceptions - mask reserved bits
 200          */
 201         fx->fx_mxcsr &= sse_mxcsr_mask;
 202 }
 203 
 204 /*
 205  * Map from an fxsave-format save area into a fpregset_t
 206  */
 207 static void
 208 fxsave_to_fpregset(const struct fxsave_state *fx, fpregset_t *fp)
 209 {
 210 #if defined(__amd64)
 211         bcopy(fx, fp, sizeof (*fx));
 212 #else
 213         struct fpchip_state *fc = &fp->fp_reg_set.fpchip_state;
 214 
 215         fxsave_to_fnsave(fx, (struct fnsave_state *)fc);
 216         fc->mxcsr = fx->fx_mxcsr;
 217         bcopy(&fx->fx_xmm[0], &fc->xmm[0], sizeof (fc->xmm));
 218 #endif
 219 }
 220 
 221 #if defined(_SYSCALL32_IMPL)
 222 static void
 223 fpregset32_to_fxsave(const fpregset32_t *fp, struct fxsave_state *fx)
 224 {
 225         const struct fpchip32_state *fc = &fp->fp_reg_set.fpchip_state;
 226 
 227         fnsave_to_fxsave((const struct fnsave_state *)fc, fx);
 228         /*
 229          * avoid useless #gp exceptions - mask reserved bits
 230          */
 231         fx->fx_mxcsr = sse_mxcsr_mask & fc->mxcsr;
 232         bcopy(&fc->xmm[0], &fx->fx_xmm[0], sizeof (fc->xmm));
 233 }
 234 
 235 static void
 236 fxsave_to_fpregset32(const struct fxsave_state *fx, fpregset32_t *fp)
 237 {
 238         struct fpchip32_state *fc = &fp->fp_reg_set.fpchip_state;
 239 
 240         fxsave_to_fnsave(fx, (struct fnsave_state *)fc);
 241         fc->mxcsr = fx->fx_mxcsr;
 242         bcopy(&fx->fx_xmm[0], &fc->xmm[0], sizeof (fc->xmm));
 243 }
 244 
 245 static void
 246 fpregset_nto32(const fpregset_t *src, fpregset32_t *dst)
 247 {
 248         fxsave_to_fpregset32((struct fxsave_state *)src, dst);
 249         dst->fp_reg_set.fpchip_state.status =
 250             src->fp_reg_set.fpchip_state.status;
 251         dst->fp_reg_set.fpchip_state.xstatus =
 252             src->fp_reg_set.fpchip_state.xstatus;
 253 }
 254 
 255 static void
 256 fpregset_32ton(const fpregset32_t *src, fpregset_t *dst)
 257 {
 258         fpregset32_to_fxsave(src, (struct fxsave_state *)dst);
 259         dst->fp_reg_set.fpchip_state.status =
 260             src->fp_reg_set.fpchip_state.status;
 261         dst->fp_reg_set.fpchip_state.xstatus =
 262             src->fp_reg_set.fpchip_state.xstatus;
 263 }
 264 #endif
 265 
 266 /*
 267  * Set floating-point registers from a native fpregset_t.
 268  */
 269 void
 270 setfpregs(klwp_t *lwp, fpregset_t *fp)
 271 {
 272         struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;
 273 
 274         if (fpu->fpu_flags & FPU_EN) {
 275                 if (!(fpu->fpu_flags & FPU_VALID)) {
 276                         /*
 277                          * FPU context is still active, release the
 278                          * ownership.
 279                          */
 280                         fp_free(fpu, 0);
 281                 }
 282         }
 283         /*
 284          * Else: if we are trying to change the FPU state of a thread which
 285          * hasn't yet initialized floating point, store the state in
 286          * the pcb and indicate that the state is valid.  When the
 287          * thread enables floating point, it will use this state instead
 288          * of the default state.
 289          */
 290 
 291         switch (fp_save_mech) {
 292 #if defined(__i386)
 293         case FP_FNSAVE:
 294                 bcopy(fp, &fpu->fpu_regs.kfpu_u.kfpu_fn,
 295                     sizeof (fpu->fpu_regs.kfpu_u.kfpu_fn));
 296                 break;
 297 #endif
 298         case FP_FXSAVE:
 299                 fpregset_to_fxsave(fp, &fpu->fpu_regs.kfpu_u.kfpu_fx);
 300                 fpu->fpu_regs.kfpu_xstatus =
 301                     fp->fp_reg_set.fpchip_state.xstatus;
 302                 break;
 303 
 304         case FP_XSAVE:
 305                 fpregset_to_fxsave(fp,
 306                     &fpu->fpu_regs.kfpu_u.kfpu_xs.xs_fxsave);
 307                 fpu->fpu_regs.kfpu_xstatus =
 308                     fp->fp_reg_set.fpchip_state.xstatus;
 309                 fpu->fpu_regs.kfpu_u.kfpu_xs.xs_xstate_bv |=
 310                     (XFEATURE_LEGACY_FP | XFEATURE_SSE);
 311                 break;
 312         default:
 313                 panic("Invalid fp_save_mech");
 314                 /*NOTREACHED*/
 315         }
 316 
 317         fpu->fpu_regs.kfpu_status = fp->fp_reg_set.fpchip_state.status;
 318         fpu->fpu_flags |= FPU_VALID;
 319 }
 320 
 321 /*
 322  * Get floating-point registers into a native fpregset_t.
 323  */
 324 void
 325 getfpregs(klwp_t *lwp, fpregset_t *fp)
 326 {
 327         struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;
 328 
 329         kpreempt_disable();
 330         if (fpu->fpu_flags & FPU_EN) {
 331                 /*
 332                  * If we have FPU hw and the thread's pcb doesn't have
 333                  * a valid FPU state then get the state from the hw.
 334                  */
 335                 if (fpu_exists && ttolwp(curthread) == lwp &&
 336                     !(fpu->fpu_flags & FPU_VALID))
 337                         fp_save(fpu); /* get the current FPU state */
 338         }
 339 
 340         /*
 341          * There are 3 possible cases we have to be aware of here:
 342          *
 343          * 1. FPU is enabled.  FPU state is stored in the current LWP.
 344          *
 345          * 2. FPU is not enabled, and there have been no intervening /proc
 346          *    modifications.  Return initial FPU state.
 347          *
 348          * 3. FPU is not enabled, but a /proc consumer has modified FPU state.
 349          *    FPU state is stored in the current LWP.
 350          */
 351         if ((fpu->fpu_flags & FPU_EN) || (fpu->fpu_flags & FPU_VALID)) {
 352                 /*
 353                  * Cases 1 and 3.
 354                  */
 355                 switch (fp_save_mech) {
 356 #if defined(__i386)
 357                 case FP_FNSAVE:
 358                         bcopy(&fpu->fpu_regs.kfpu_u.kfpu_fn, fp,
 359                             sizeof (fpu->fpu_regs.kfpu_u.kfpu_fn));
 360                         break;
 361 #endif
 362                 case FP_FXSAVE:
 363                         fxsave_to_fpregset(&fpu->fpu_regs.kfpu_u.kfpu_fx, fp);
 364                         fp->fp_reg_set.fpchip_state.xstatus =
 365                             fpu->fpu_regs.kfpu_xstatus;
 366                         break;
 367                 case FP_XSAVE:
 368                         fxsave_to_fpregset(
 369                             &fpu->fpu_regs.kfpu_u.kfpu_xs.xs_fxsave, fp);
 370                         fp->fp_reg_set.fpchip_state.xstatus =
 371                             fpu->fpu_regs.kfpu_xstatus;
 372                         break;
 373                 default:
 374                         panic("Invalid fp_save_mech");
 375                         /*NOTREACHED*/
 376                 }
 377                 fp->fp_reg_set.fpchip_state.status = fpu->fpu_regs.kfpu_status;
 378         } else {
 379                 /*
 380                  * Case 2.
 381                  */
 382                 switch (fp_save_mech) {
 383 #if defined(__i386)
 384                 case FP_FNSAVE:
 385                         bcopy(&x87_initial, fp, sizeof (x87_initial));
 386                         break;
 387 #endif
 388                 case FP_FXSAVE:
 389                 case FP_XSAVE:
 390                         /*
 391                          * For now, we don't have any AVX specific field in ABI.
 392                          * If we add any in the future, we need to initial them
 393                          * as well.
 394                          */
 395                         fxsave_to_fpregset(&sse_initial, fp);
 396                         fp->fp_reg_set.fpchip_state.xstatus =
 397                             fpu->fpu_regs.kfpu_xstatus;
 398                         break;
 399                 default:
 400                         panic("Invalid fp_save_mech");
 401                         /*NOTREACHED*/
 402                 }
 403                 fp->fp_reg_set.fpchip_state.status = fpu->fpu_regs.kfpu_status;
 404         }
 405         kpreempt_enable();
 406 }
 407 
 408 #if defined(_SYSCALL32_IMPL)
 409 
 410 /*
 411  * Set floating-point registers from an fpregset32_t.
 412  */
 413 void
 414 setfpregs32(klwp_t *lwp, fpregset32_t *fp)
 415 {
 416         fpregset_t fpregs;
 417 
 418         fpregset_32ton(fp, &fpregs);
 419         setfpregs(lwp, &fpregs);
 420 }
 421 
 422 /*
 423  * Get floating-point registers into an fpregset32_t.
 424  */
 425 void
 426 getfpregs32(klwp_t *lwp, fpregset32_t *fp)
 427 {
 428         fpregset_t fpregs;
 429 
 430         getfpregs(lwp, &fpregs);
 431         fpregset_nto32(&fpregs, fp);
 432 }
 433 
 434 #endif  /* _SYSCALL32_IMPL */
 435 
 436 /*
 437  * Return the general registers
 438  */
 439 void
 440 getgregs(klwp_t *lwp, gregset_t grp)
 441 {
 442         struct regs *rp = lwptoregs(lwp);
 443 #if defined(__amd64)
 444         struct pcb *pcb = &lwp->lwp_pcb;
 445         int thisthread = lwptot(lwp) == curthread;
 446 
 447         grp[REG_RDI] = rp->r_rdi;
 448         grp[REG_RSI] = rp->r_rsi;
 449         grp[REG_RDX] = rp->r_rdx;
 450         grp[REG_RCX] = rp->r_rcx;
 451         grp[REG_R8] = rp->r_r8;
 452         grp[REG_R9] = rp->r_r9;
 453         grp[REG_RAX] = rp->r_rax;
 454         grp[REG_RBX] = rp->r_rbx;
 455         grp[REG_RBP] = rp->r_rbp;
 456         grp[REG_R10] = rp->r_r10;
 457         grp[REG_R11] = rp->r_r11;
 458         grp[REG_R12] = rp->r_r12;
 459         grp[REG_R13] = rp->r_r13;
 460         grp[REG_R14] = rp->r_r14;
 461         grp[REG_R15] = rp->r_r15;
 462         grp[REG_FSBASE] = pcb->pcb_fsbase;
 463         grp[REG_GSBASE] = pcb->pcb_gsbase;
 464         if (thisthread)
 465                 kpreempt_disable();
 466         if (pcb->pcb_rupdate == 1) {
 467                 grp[REG_DS] = pcb->pcb_ds;
 468                 grp[REG_ES] = pcb->pcb_es;
 469                 grp[REG_FS] = pcb->pcb_fs;
 470                 grp[REG_GS] = pcb->pcb_gs;
 471         } else {
 472                 grp[REG_DS] = rp->r_ds;
 473                 grp[REG_ES] = rp->r_es;
 474                 grp[REG_FS] = rp->r_fs;
 475                 grp[REG_GS] = rp->r_gs;
 476         }
 477         if (thisthread)
 478                 kpreempt_enable();
 479         grp[REG_TRAPNO] = rp->r_trapno;
 480         grp[REG_ERR] = rp->r_err;
 481         grp[REG_RIP] = rp->r_rip;
 482         grp[REG_CS] = rp->r_cs;
 483         grp[REG_SS] = rp->r_ss;
 484         grp[REG_RFL] = rp->r_rfl;
 485         grp[REG_RSP] = rp->r_rsp;
 486 #else
 487         bcopy(&rp->r_gs, grp, sizeof (gregset_t));
 488 #endif
 489 }
 490 
 491 #if defined(_SYSCALL32_IMPL)
 492 
 493 void
 494 getgregs32(klwp_t *lwp, gregset32_t grp)
 495 {
 496         struct regs *rp = lwptoregs(lwp);
 497         struct pcb *pcb = &lwp->lwp_pcb;
 498         int thisthread = lwptot(lwp) == curthread;
 499 
 500         if (thisthread)
 501                 kpreempt_disable();
 502         if (pcb->pcb_rupdate == 1) {
 503                 grp[GS] = (uint16_t)pcb->pcb_gs;
 504                 grp[FS] = (uint16_t)pcb->pcb_fs;
 505                 grp[DS] = (uint16_t)pcb->pcb_ds;
 506                 grp[ES] = (uint16_t)pcb->pcb_es;
 507         } else {
 508                 grp[GS] = (uint16_t)rp->r_gs;
 509                 grp[FS] = (uint16_t)rp->r_fs;
 510                 grp[DS] = (uint16_t)rp->r_ds;
 511                 grp[ES] = (uint16_t)rp->r_es;
 512         }
 513         if (thisthread)
 514                 kpreempt_enable();
 515         grp[EDI] = (greg32_t)rp->r_rdi;
 516         grp[ESI] = (greg32_t)rp->r_rsi;
 517         grp[EBP] = (greg32_t)rp->r_rbp;
 518         grp[ESP] = 0;
 519         grp[EBX] = (greg32_t)rp->r_rbx;
 520         grp[EDX] = (greg32_t)rp->r_rdx;
 521         grp[ECX] = (greg32_t)rp->r_rcx;
 522         grp[EAX] = (greg32_t)rp->r_rax;
 523         grp[TRAPNO] = (greg32_t)rp->r_trapno;
 524         grp[ERR] = (greg32_t)rp->r_err;
 525         grp[EIP] = (greg32_t)rp->r_rip;
 526         grp[CS] = (uint16_t)rp->r_cs;
 527         grp[EFL] = (greg32_t)rp->r_rfl;
 528         grp[UESP] = (greg32_t)rp->r_rsp;
 529         grp[SS] = (uint16_t)rp->r_ss;
 530 }
 531 
 532 void
 533 ucontext_32ton(const ucontext32_t *src, ucontext_t *dst)
 534 {
 535         mcontext_t *dmc = &dst->uc_mcontext;
 536         const mcontext32_t *smc = &src->uc_mcontext;
 537 
 538         bzero(dst, sizeof (*dst));
 539         dst->uc_flags = src->uc_flags;
 540         dst->uc_link = (ucontext_t *)(uintptr_t)src->uc_link;
 541 
 542         bcopy(&src->uc_sigmask, &dst->uc_sigmask, sizeof (dst->uc_sigmask));
 543 
 544         dst->uc_stack.ss_sp = (void *)(uintptr_t)src->uc_stack.ss_sp;
 545         dst->uc_stack.ss_size = (size_t)src->uc_stack.ss_size;
 546         dst->uc_stack.ss_flags = src->uc_stack.ss_flags;
 547 
 548         dmc->gregs[REG_GS] = (greg_t)(uint32_t)smc->gregs[GS];
 549         dmc->gregs[REG_FS] = (greg_t)(uint32_t)smc->gregs[FS];
 550         dmc->gregs[REG_ES] = (greg_t)(uint32_t)smc->gregs[ES];
 551         dmc->gregs[REG_DS] = (greg_t)(uint32_t)smc->gregs[DS];
 552         dmc->gregs[REG_RDI] = (greg_t)(uint32_t)smc->gregs[EDI];
 553         dmc->gregs[REG_RSI] = (greg_t)(uint32_t)smc->gregs[ESI];
 554         dmc->gregs[REG_RBP] = (greg_t)(uint32_t)smc->gregs[EBP];
 555         dmc->gregs[REG_RBX] = (greg_t)(uint32_t)smc->gregs[EBX];
 556         dmc->gregs[REG_RDX] = (greg_t)(uint32_t)smc->gregs[EDX];
 557         dmc->gregs[REG_RCX] = (greg_t)(uint32_t)smc->gregs[ECX];
 558         dmc->gregs[REG_RAX] = (greg_t)(uint32_t)smc->gregs[EAX];
 559         dmc->gregs[REG_TRAPNO] = (greg_t)(uint32_t)smc->gregs[TRAPNO];
 560         dmc->gregs[REG_ERR] = (greg_t)(uint32_t)smc->gregs[ERR];
 561         dmc->gregs[REG_RIP] = (greg_t)(uint32_t)smc->gregs[EIP];
 562         dmc->gregs[REG_CS] = (greg_t)(uint32_t)smc->gregs[CS];
 563         dmc->gregs[REG_RFL] = (greg_t)(uint32_t)smc->gregs[EFL];
 564         dmc->gregs[REG_RSP] = (greg_t)(uint32_t)smc->gregs[UESP];
 565         dmc->gregs[REG_SS] = (greg_t)(uint32_t)smc->gregs[SS];
 566 
 567         /*
 568          * A valid fpregs is only copied in if uc.uc_flags has UC_FPU set
 569          * otherwise there is no guarantee that anything in fpregs is valid.
 570          */
 571         if (src->uc_flags & UC_FPU)
 572                 fpregset_32ton(&src->uc_mcontext.fpregs,
 573                     &dst->uc_mcontext.fpregs);
 574 }
 575 
 576 #endif  /* _SYSCALL32_IMPL */
 577 
 578 /*
 579  * Return the user-level PC.
 580  * If in a system call, return the address of the syscall trap.
 581  */
 582 greg_t
 583 getuserpc()
 584 {
 585         greg_t upc = lwptoregs(ttolwp(curthread))->r_pc;
 586         uint32_t insn;
 587 
 588         if (curthread->t_sysnum == 0)
 589                 return (upc);
 590 
 591         /*
 592          * We might've gotten here from sysenter (0xf 0x34),
 593          * syscall (0xf 0x5) or lcall (0x9a 0 0 0 0 0x27 0).
 594          *
 595          * Go peek at the binary to figure it out..
 596          */
 597         if (fuword32((void *)(upc - 2), &insn) != -1 &&
 598             (insn & 0xffff) == 0x340f || (insn & 0xffff) == 0x050f)
 599                 return (upc - 2);
 600         return (upc - 7);
 601 }
 602 
 603 /*
 604  * Protect segment registers from non-user privilege levels and GDT selectors
 605  * other than USER_CS, USER_DS and lwp FS and GS values.  If the segment
 606  * selector is non-null and not USER_CS/USER_DS, we make sure that the
 607  * TI bit is set to point into the LDT and that the RPL is set to 3.
 608  *
 609  * Since struct regs stores each 16-bit segment register as a 32-bit greg_t, we
 610  * also explicitly zero the top 16 bits since they may be coming from the
 611  * user's address space via setcontext(2) or /proc.
 612  *
 613  * Note about null selector. When running on the hypervisor if we allow a
 614  * process to set its %cs to null selector with RPL of 0 the hypervisor will
 615  * crash the domain. If running on bare metal we would get a #gp fault and
 616  * be able to kill the process and continue on. Therefore we make sure to
 617  * force RPL to SEL_UPL even for null selector when setting %cs.
 618  */
 619 
 620 #if defined(IS_CS) || defined(IS_NOT_CS)
 621 #error  "IS_CS and IS_NOT_CS already defined"
 622 #endif
 623 
 624 #define IS_CS           1
 625 #define IS_NOT_CS       0
 626 
 627 /*ARGSUSED*/
 628 static greg_t
 629 fix_segreg(greg_t sr, int iscs, model_t datamodel)
 630 {
 631         kthread_t *t = curthread;
 632 
 633         switch (sr &= 0xffff) {
 634 
 635         case 0:
 636                 if (iscs == IS_CS)
 637                         return (0 | SEL_UPL);
 638                 else
 639                         return (0);
 640 
 641 #if defined(__amd64)
 642         /*
 643          * If lwp attempts to switch data model then force their
 644          * code selector to be null selector.
 645          */
 646         case U32CS_SEL:
 647                 if (datamodel == DATAMODEL_NATIVE)
 648                         return (0 | SEL_UPL);
 649                 else
 650                         return (sr);
 651 
 652         case UCS_SEL:
 653                 if (datamodel == DATAMODEL_ILP32)
 654                         return (0 | SEL_UPL);
 655 #elif defined(__i386)
 656         case UCS_SEL:
 657 #endif
 658         /*FALLTHROUGH*/
 659         case UDS_SEL:
 660         case LWPFS_SEL:
 661         case LWPGS_SEL:
 662         case SEL_UPL:
 663                 return (sr);
 664         default:
 665                 break;
 666         }
 667 
 668         /*
 669          * Allow this process's brand to do any necessary segment register
 670          * manipulation.
 671          */
 672         if (PROC_IS_BRANDED(t->t_procp) && BRMOP(t->t_procp)->b_fixsegreg) {
 673                 greg_t bsr = BRMOP(t->t_procp)->b_fixsegreg(sr, datamodel);
 674 
 675                 if (bsr == 0 && iscs == IS_CS)
 676                         return (0 | SEL_UPL);
 677                 else
 678                         return (bsr);
 679         }
 680 
 681         /*
 682          * Force it into the LDT in ring 3 for 32-bit processes, which by
 683          * default do not have an LDT, so that any attempt to use an invalid
 684          * selector will reference the (non-existant) LDT, and cause a #gp
 685          * fault for the process.
 686          *
 687          * 64-bit processes get the null gdt selector since they
 688          * are not allowed to have a private LDT.
 689          */
 690 #if defined(__amd64)
 691         if (datamodel == DATAMODEL_ILP32) {
 692                 return (sr | SEL_TI_LDT | SEL_UPL);
 693         } else {
 694                 if (iscs == IS_CS)
 695                         return (0 | SEL_UPL);
 696                 else
 697                         return (0);
 698         }
 699 
 700 #elif defined(__i386)
 701         return (sr | SEL_TI_LDT | SEL_UPL);
 702 #endif
 703 }
 704 
 705 /*
 706  * Set general registers.
 707  */
 708 void
 709 setgregs(klwp_t *lwp, gregset_t grp)
 710 {
 711         struct regs *rp = lwptoregs(lwp);
 712         model_t datamodel = lwp_getdatamodel(lwp);
 713 
 714 #if defined(__amd64)
 715         struct pcb *pcb = &lwp->lwp_pcb;
 716         int thisthread = lwptot(lwp) == curthread;
 717 
 718         if (datamodel == DATAMODEL_NATIVE) {
 719 
 720                 if (thisthread)
 721                         (void) save_syscall_args();     /* copy the args */
 722 
 723                 rp->r_rdi = grp[REG_RDI];
 724                 rp->r_rsi = grp[REG_RSI];
 725                 rp->r_rdx = grp[REG_RDX];
 726                 rp->r_rcx = grp[REG_RCX];
 727                 rp->r_r8 = grp[REG_R8];
 728                 rp->r_r9 = grp[REG_R9];
 729                 rp->r_rax = grp[REG_RAX];
 730                 rp->r_rbx = grp[REG_RBX];
 731                 rp->r_rbp = grp[REG_RBP];
 732                 rp->r_r10 = grp[REG_R10];
 733                 rp->r_r11 = grp[REG_R11];
 734                 rp->r_r12 = grp[REG_R12];
 735                 rp->r_r13 = grp[REG_R13];
 736                 rp->r_r14 = grp[REG_R14];
 737                 rp->r_r15 = grp[REG_R15];
 738                 rp->r_trapno = grp[REG_TRAPNO];
 739                 rp->r_err = grp[REG_ERR];
 740                 rp->r_rip = grp[REG_RIP];
 741                 /*
 742                  * Setting %cs or %ss to anything else is quietly but
 743                  * quite definitely forbidden!
 744                  */
 745                 rp->r_cs = UCS_SEL;
 746                 rp->r_ss = UDS_SEL;
 747                 rp->r_rsp = grp[REG_RSP];
 748 
 749                 if (thisthread)
 750                         kpreempt_disable();
 751 
 752                 pcb->pcb_ds = UDS_SEL;
 753                 pcb->pcb_es = UDS_SEL;
 754 
 755                 /*
 756                  * 64-bit processes -are- allowed to set their fsbase/gsbase
 757                  * values directly, but only if they're using the segment
 758                  * selectors that allow that semantic.
 759                  *
 760                  * (32-bit processes must use lwp_set_private().)
 761                  */
 762                 pcb->pcb_fsbase = grp[REG_FSBASE];
 763                 pcb->pcb_gsbase = grp[REG_GSBASE];
 764                 pcb->pcb_fs = fix_segreg(grp[REG_FS], IS_NOT_CS, datamodel);
 765                 pcb->pcb_gs = fix_segreg(grp[REG_GS], IS_NOT_CS, datamodel);
 766 
 767                 /*
 768                  * Ensure that we go out via update_sregs
 769                  */
 770                 pcb->pcb_rupdate = 1;
 771                 lwptot(lwp)->t_post_sys = 1;
 772                 if (thisthread)
 773                         kpreempt_enable();
 774 #if defined(_SYSCALL32_IMPL)
 775         } else {
 776                 rp->r_rdi = (uint32_t)grp[REG_RDI];
 777                 rp->r_rsi = (uint32_t)grp[REG_RSI];
 778                 rp->r_rdx = (uint32_t)grp[REG_RDX];
 779                 rp->r_rcx = (uint32_t)grp[REG_RCX];
 780                 rp->r_rax = (uint32_t)grp[REG_RAX];
 781                 rp->r_rbx = (uint32_t)grp[REG_RBX];
 782                 rp->r_rbp = (uint32_t)grp[REG_RBP];
 783                 rp->r_trapno = (uint32_t)grp[REG_TRAPNO];
 784                 rp->r_err = (uint32_t)grp[REG_ERR];
 785                 rp->r_rip = (uint32_t)grp[REG_RIP];
 786 
 787                 rp->r_cs = fix_segreg(grp[REG_CS], IS_CS, datamodel);
 788                 rp->r_ss = fix_segreg(grp[REG_DS], IS_NOT_CS, datamodel);
 789 
 790                 rp->r_rsp = (uint32_t)grp[REG_RSP];
 791 
 792                 if (thisthread)
 793                         kpreempt_disable();
 794 
 795                 pcb->pcb_ds = fix_segreg(grp[REG_DS], IS_NOT_CS, datamodel);
 796                 pcb->pcb_es = fix_segreg(grp[REG_ES], IS_NOT_CS, datamodel);
 797 
 798                 /*
 799                  * (See fsbase/gsbase commentary above)
 800                  */
 801                 pcb->pcb_fs = fix_segreg(grp[REG_FS], IS_NOT_CS, datamodel);
 802                 pcb->pcb_gs = fix_segreg(grp[REG_GS], IS_NOT_CS, datamodel);
 803 
 804                 /*
 805                  * Ensure that we go out via update_sregs
 806                  */
 807                 pcb->pcb_rupdate = 1;
 808                 lwptot(lwp)->t_post_sys = 1;
 809                 if (thisthread)
 810                         kpreempt_enable();
 811 #endif
 812         }
 813 
 814         /*
 815          * Only certain bits of the flags register can be modified.
 816          */
 817         rp->r_rfl = (rp->r_rfl & ~PSL_USERMASK) |
 818             (grp[REG_RFL] & PSL_USERMASK);
 819 
 820 #elif defined(__i386)
 821 
 822         /*
 823          * Only certain bits of the flags register can be modified.
 824          */
 825         grp[EFL] = (rp->r_efl & ~PSL_USERMASK) | (grp[EFL] & PSL_USERMASK);
 826 
 827         /*
 828          * Copy saved registers from user stack.
 829          */
 830         bcopy(grp, &rp->r_gs, sizeof (gregset_t));
 831 
 832         rp->r_cs = fix_segreg(rp->r_cs, IS_CS, datamodel);
 833         rp->r_ss = fix_segreg(rp->r_ss, IS_NOT_CS, datamodel);
 834         rp->r_ds = fix_segreg(rp->r_ds, IS_NOT_CS, datamodel);
 835         rp->r_es = fix_segreg(rp->r_es, IS_NOT_CS, datamodel);
 836         rp->r_fs = fix_segreg(rp->r_fs, IS_NOT_CS, datamodel);
 837         rp->r_gs = fix_segreg(rp->r_gs, IS_NOT_CS, datamodel);
 838 
 839 #endif  /* __i386 */
 840 }
 841 
 842 /*
 843  * Determine whether eip is likely to have an interrupt frame
 844  * on the stack.  We do this by comparing the address to the
 845  * range of addresses spanned by several well-known routines.
 846  */
 847 extern void _interrupt();
 848 extern void _allsyscalls();
 849 extern void _cmntrap();
 850 extern void fakesoftint();
 851 
 852 extern size_t _interrupt_size;
 853 extern size_t _allsyscalls_size;
 854 extern size_t _cmntrap_size;
 855 extern size_t _fakesoftint_size;
 856 
 857 /*
 858  * Get a pc-only stacktrace.  Used for kmem_alloc() buffer ownership tracking.
 859  * Returns MIN(current stack depth, pcstack_limit).
 860  */
 861 int
 862 getpcstack(pc_t *pcstack, int pcstack_limit)
 863 {
 864         struct frame *fp = (struct frame *)getfp();
 865         struct frame *nextfp, *minfp, *stacktop;
 866         int depth = 0;
 867         int on_intr;
 868         uintptr_t pc;
 869 
 870         if ((on_intr = CPU_ON_INTR(CPU)) != 0)
 871                 stacktop = (struct frame *)(CPU->cpu_intr_stack + SA(MINFRAME));
 872         else
 873                 stacktop = (struct frame *)curthread->t_stk;
 874         minfp = fp;
 875 
 876         pc = ((struct regs *)fp)->r_pc;
 877 
 878         while (depth < pcstack_limit) {
 879                 nextfp = (struct frame *)fp->fr_savfp;
 880                 pc = fp->fr_savpc;
 881                 if (nextfp <= minfp || nextfp >= stacktop) {
 882                         if (on_intr) {
 883                                 /*
 884                                  * Hop from interrupt stack to thread stack.
 885                                  */
 886                                 stacktop = (struct frame *)curthread->t_stk;
 887                                 minfp = (struct frame *)curthread->t_stkbase;
 888                                 on_intr = 0;
 889                                 continue;
 890                         }
 891                         break;
 892                 }
 893                 pcstack[depth++] = (pc_t)pc;
 894                 fp = nextfp;
 895                 minfp = fp;
 896         }
 897         return (depth);
 898 }
 899 
 900 /*
 901  * The following ELF header fields are defined as processor-specific
 902  * in the V8 ABI:
 903  *
 904  *      e_ident[EI_DATA]        encoding of the processor-specific
 905  *                              data in the object file
 906  *      e_machine               processor identification
 907  *      e_flags                 processor-specific flags associated
 908  *                              with the file
 909  */
 910 
 911 /*
 912  * The value of at_flags reflects a platform's cpu module support.
 913  * at_flags is used to check for allowing a binary to execute and
 914  * is passed as the value of the AT_FLAGS auxiliary vector.
 915  */
 916 int at_flags = 0;
 917 
 918 /*
 919  * Check the processor-specific fields of an ELF header.
 920  *
 921  * returns 1 if the fields are valid, 0 otherwise
 922  */
 923 /*ARGSUSED2*/
 924 int
 925 elfheadcheck(
 926         unsigned char e_data,
 927         Elf32_Half e_machine,
 928         Elf32_Word e_flags)
 929 {
 930         if (e_data != ELFDATA2LSB)
 931                 return (0);
 932 #if defined(__amd64)
 933         if (e_machine == EM_AMD64)
 934                 return (1);
 935 #endif
 936         return (e_machine == EM_386);
 937 }
 938 
 939 uint_t auxv_hwcap_include = 0;  /* patch to enable unrecognized features */
 940 uint_t auxv_hwcap_exclude = 0;  /* patch for broken cpus, debugging */
 941 #if defined(_SYSCALL32_IMPL)
 942 uint_t auxv_hwcap32_include = 0;        /* ditto for 32-bit apps */
 943 uint_t auxv_hwcap32_exclude = 0;        /* ditto for 32-bit apps */
 944 #endif
 945 
 946 /*
 947  * Gather information about the processor and place it into auxv_hwcap
 948  * so that it can be exported to the linker via the aux vector.
 949  *
 950  * We use this seemingly complicated mechanism so that we can ensure
 951  * that /etc/system can be used to override what the system can or
 952  * cannot discover for itself.
 953  */
 954 void
 955 bind_hwcap(void)
 956 {
 957         uint_t cpu_hwcap_flags = cpuid_pass4(NULL);
 958 
 959         auxv_hwcap = (auxv_hwcap_include | cpu_hwcap_flags) &
 960             ~auxv_hwcap_exclude;
 961 
 962 #if defined(__amd64)
 963         /*
 964          * On AMD processors, sysenter just doesn't work at all
 965          * when the kernel is in long mode.  On IA-32e processors
 966          * it does, but there's no real point in all the alternate
 967          * mechanism when syscall works on both.
 968          *
 969          * Besides, the kernel's sysenter handler is expecting a
 970          * 32-bit lwp ...
 971          */
 972         auxv_hwcap &= ~AV_386_SEP;
 973 #else
 974         /*
 975          * 32-bit processes can -always- use the lahf/sahf instructions
 976          */
 977         auxv_hwcap |= AV_386_AHF;
 978 #endif
 979 
 980         if (auxv_hwcap_include || auxv_hwcap_exclude)
 981                 cmn_err(CE_CONT, "?user ABI extensions: %b\n",
 982                     auxv_hwcap, FMT_AV_386);
 983 
 984 #if defined(_SYSCALL32_IMPL)
 985         auxv_hwcap32 = (auxv_hwcap32_include | cpu_hwcap_flags) &
 986             ~auxv_hwcap32_exclude;
 987 
 988 #if defined(__amd64)
 989         /*
 990          * If this is an amd64 architecture machine from Intel, then
 991          * syscall -doesn't- work in compatibility mode, only sysenter does.
 992          *
 993          * Sigh.
 994          */
 995         if (!cpuid_syscall32_insn(NULL))
 996                 auxv_hwcap32 &= ~AV_386_AMD_SYSC;
 997 
 998         /*
 999          * 32-bit processes can -always- use the lahf/sahf instructions
1000          */
1001         auxv_hwcap32 |= AV_386_AHF;
1002 #endif
1003 
1004         if (auxv_hwcap32_include || auxv_hwcap32_exclude)
1005                 cmn_err(CE_CONT, "?32-bit user ABI extensions: %b\n",
1006                     auxv_hwcap32, FMT_AV_386);
1007 #endif
1008 }
1009 
1010 /*
1011  *      sync_icache() - this is called
1012  *      in proc/fs/prusrio.c. x86 has an unified cache and therefore
1013  *      this is a nop.
1014  */
1015 /* ARGSUSED */
1016 void
1017 sync_icache(caddr_t addr, uint_t len)
1018 {
1019         /* Do nothing for now */
1020 }
1021 
1022 /*ARGSUSED*/
1023 void
1024 sync_data_memory(caddr_t va, size_t len)
1025 {
1026         /* Not implemented for this platform */
1027 }
1028 
1029 int
1030 __ipltospl(int ipl)
1031 {
1032         return (ipltospl(ipl));
1033 }
1034 
1035 /*
1036  * The panic code invokes panic_saveregs() to record the contents of a
1037  * regs structure into the specified panic_data structure for debuggers.
1038  */
1039 void
1040 panic_saveregs(panic_data_t *pdp, struct regs *rp)
1041 {
1042         panic_nv_t *pnv = PANICNVGET(pdp);
1043 
1044         struct cregs    creg;
1045 
1046         getcregs(&creg);
1047 
1048 #if defined(__amd64)
1049         PANICNVADD(pnv, "rdi", rp->r_rdi);
1050         PANICNVADD(pnv, "rsi", rp->r_rsi);
1051         PANICNVADD(pnv, "rdx", rp->r_rdx);
1052         PANICNVADD(pnv, "rcx", rp->r_rcx);
1053         PANICNVADD(pnv, "r8", rp->r_r8);
1054         PANICNVADD(pnv, "r9", rp->r_r9);
1055         PANICNVADD(pnv, "rax", rp->r_rax);
1056         PANICNVADD(pnv, "rbx", rp->r_rbx);
1057         PANICNVADD(pnv, "rbp", rp->r_rbp);
1058         PANICNVADD(pnv, "r10", rp->r_r10);
1059         PANICNVADD(pnv, "r10", rp->r_r10);
1060         PANICNVADD(pnv, "r11", rp->r_r11);
1061         PANICNVADD(pnv, "r12", rp->r_r12);
1062         PANICNVADD(pnv, "r13", rp->r_r13);
1063         PANICNVADD(pnv, "r14", rp->r_r14);
1064         PANICNVADD(pnv, "r15", rp->r_r15);
1065         PANICNVADD(pnv, "fsbase", rdmsr(MSR_AMD_FSBASE));
1066         PANICNVADD(pnv, "gsbase", rdmsr(MSR_AMD_GSBASE));
1067         PANICNVADD(pnv, "ds", rp->r_ds);
1068         PANICNVADD(pnv, "es", rp->r_es);
1069         PANICNVADD(pnv, "fs", rp->r_fs);
1070         PANICNVADD(pnv, "gs", rp->r_gs);
1071         PANICNVADD(pnv, "trapno", rp->r_trapno);
1072         PANICNVADD(pnv, "err", rp->r_err);
1073         PANICNVADD(pnv, "rip", rp->r_rip);
1074         PANICNVADD(pnv, "cs", rp->r_cs);
1075         PANICNVADD(pnv, "rflags", rp->r_rfl);
1076         PANICNVADD(pnv, "rsp", rp->r_rsp);
1077         PANICNVADD(pnv, "ss", rp->r_ss);
1078         PANICNVADD(pnv, "gdt_hi", (uint64_t)(creg.cr_gdt._l[3]));
1079         PANICNVADD(pnv, "gdt_lo", (uint64_t)(creg.cr_gdt._l[0]));
1080         PANICNVADD(pnv, "idt_hi", (uint64_t)(creg.cr_idt._l[3]));
1081         PANICNVADD(pnv, "idt_lo", (uint64_t)(creg.cr_idt._l[0]));
1082 #elif defined(__i386)
1083         PANICNVADD(pnv, "gs", (uint32_t)rp->r_gs);
1084         PANICNVADD(pnv, "fs", (uint32_t)rp->r_fs);
1085         PANICNVADD(pnv, "es", (uint32_t)rp->r_es);
1086         PANICNVADD(pnv, "ds", (uint32_t)rp->r_ds);
1087         PANICNVADD(pnv, "edi", (uint32_t)rp->r_edi);
1088         PANICNVADD(pnv, "esi", (uint32_t)rp->r_esi);
1089         PANICNVADD(pnv, "ebp", (uint32_t)rp->r_ebp);
1090         PANICNVADD(pnv, "esp", (uint32_t)rp->r_esp);
1091         PANICNVADD(pnv, "ebx", (uint32_t)rp->r_ebx);
1092         PANICNVADD(pnv, "edx", (uint32_t)rp->r_edx);
1093         PANICNVADD(pnv, "ecx", (uint32_t)rp->r_ecx);
1094         PANICNVADD(pnv, "eax", (uint32_t)rp->r_eax);
1095         PANICNVADD(pnv, "trapno", (uint32_t)rp->r_trapno);
1096         PANICNVADD(pnv, "err", (uint32_t)rp->r_err);
1097         PANICNVADD(pnv, "eip", (uint32_t)rp->r_eip);
1098         PANICNVADD(pnv, "cs", (uint32_t)rp->r_cs);
1099         PANICNVADD(pnv, "eflags", (uint32_t)rp->r_efl);
1100         PANICNVADD(pnv, "uesp", (uint32_t)rp->r_uesp);
1101         PANICNVADD(pnv, "ss", (uint32_t)rp->r_ss);
1102         PANICNVADD(pnv, "gdt", creg.cr_gdt);
1103         PANICNVADD(pnv, "idt", creg.cr_idt);
1104 #endif  /* __i386 */
1105 
1106         PANICNVADD(pnv, "ldt", creg.cr_ldt);
1107         PANICNVADD(pnv, "task", creg.cr_task);
1108         PANICNVADD(pnv, "cr0", creg.cr_cr0);
1109         PANICNVADD(pnv, "cr2", creg.cr_cr2);
1110         PANICNVADD(pnv, "cr3", creg.cr_cr3);
1111         if (creg.cr_cr4)
1112                 PANICNVADD(pnv, "cr4", creg.cr_cr4);
1113 
1114         PANICNVSET(pdp, pnv);
1115 }
1116 
1117 #define TR_ARG_MAX 6    /* Max args to print, same as SPARC */
1118 
1119 #if !defined(__amd64)
1120 
1121 /*
1122  * Given a return address (%eip), determine the likely number of arguments
1123  * that were pushed on the stack prior to its execution.  We do this by
1124  * expecting that a typical call sequence consists of pushing arguments on
1125  * the stack, executing a call instruction, and then performing an add
1126  * on %esp to restore it to the value prior to pushing the arguments for
1127  * the call.  We attempt to detect such an add, and divide the addend
1128  * by the size of a word to determine the number of pushed arguments.
1129  *
1130  * If we do not find such an add, we punt and return TR_ARG_MAX. It is not
1131  * possible to reliably determine if a function took no arguments (i.e. was
1132  * void) because assembler routines do not reliably perform an add on %esp
1133  * immediately upon returning (eg. _sys_call()), so returning TR_ARG_MAX is
1134  * safer than returning 0.
1135  */
1136 static ulong_t
1137 argcount(uintptr_t eip)
1138 {
1139         const uint8_t *ins = (const uint8_t *)eip;
1140         ulong_t n;
1141 
1142         enum {
1143                 M_MODRM_ESP = 0xc4,     /* Mod/RM byte indicates %esp */
1144                 M_ADD_IMM32 = 0x81,     /* ADD imm32 to r/m32 */
1145                 M_ADD_IMM8  = 0x83      /* ADD imm8 to r/m32 */
1146         };
1147 
1148         if (eip < KERNELBASE || ins[1] != M_MODRM_ESP)
1149                 return (TR_ARG_MAX);
1150 
1151         switch (ins[0]) {
1152         case M_ADD_IMM32:
1153                 n = ins[2] + (ins[3] << 8) + (ins[4] << 16) + (ins[5] << 24);
1154                 break;
1155 
1156         case M_ADD_IMM8:
1157                 n = ins[2];
1158                 break;
1159 
1160         default:
1161                 return (TR_ARG_MAX);
1162         }
1163 
1164         n /= sizeof (long);
1165         return (MIN(n, TR_ARG_MAX));
1166 }
1167 
1168 #endif  /* !__amd64 */
1169 
1170 /*
1171  * Print a stack backtrace using the specified frame pointer.  We delay two
1172  * seconds before continuing, unless this is the panic traceback.
1173  * If we are in the process of panicking, we also attempt to write the
1174  * stack backtrace to a staticly assigned buffer, to allow the panic
1175  * code to find it and write it in to uncompressed pages within the
1176  * system crash dump.
1177  * Note that the frame for the starting stack pointer value is omitted because
1178  * the corresponding %eip is not known.
1179  */
1180 
1181 extern char *dump_stack_scratch;
1182 
1183 #if defined(__amd64)
1184 
1185 void
1186 traceback(caddr_t fpreg)
1187 {
1188         struct frame    *fp = (struct frame *)fpreg;
1189         struct frame    *nextfp;
1190         uintptr_t       pc, nextpc;
1191         ulong_t         off;
1192         char            args[TR_ARG_MAX * 2 + 16], *sym;
1193         uint_t    offset = 0;
1194         uint_t    next_offset = 0;
1195         char        stack_buffer[1024];
1196 
1197         if (!panicstr)
1198                 printf("traceback: %%fp = %p\n", (void *)fp);
1199 
1200         if (panicstr && !dump_stack_scratch) {
1201                 printf("Warning - stack not written to the dump buffer\n");
1202         }
1203 
1204         fp = (struct frame *)plat_traceback(fpreg);
1205         if ((uintptr_t)fp < KERNELBASE)
1206                 goto out;
1207 
1208         pc = fp->fr_savpc;
1209         fp = (struct frame *)fp->fr_savfp;
1210 
1211         while ((uintptr_t)fp >= KERNELBASE) {
1212                 /*
1213                  * XX64 Until port is complete tolerate 8-byte aligned
1214                  * frame pointers but flag with a warning so they can
1215                  * be fixed.
1216                  */
1217                 if (((uintptr_t)fp & (STACK_ALIGN - 1)) != 0) {
1218                         if (((uintptr_t)fp & (8 - 1)) == 0) {
1219                                 printf("  >> warning! 8-byte"
1220                                     " aligned %%fp = %p\n", (void *)fp);
1221                         } else {
1222                                 printf(
1223                                     "  >> mis-aligned %%fp = %p\n", (void *)fp);
1224                                 break;
1225                         }
1226                 }
1227 
1228                 args[0] = '\0';
1229                 nextpc = (uintptr_t)fp->fr_savpc;
1230                 nextfp = (struct frame *)fp->fr_savfp;
1231                 if ((sym = kobj_getsymname(pc, &off)) != NULL) {
1232                         printf("%016lx %s:%s+%lx (%s)\n", (uintptr_t)fp,
1233                             mod_containing_pc((caddr_t)pc), sym, off, args);
1234                         (void) snprintf(stack_buffer, sizeof (stack_buffer),
1235                             "%s:%s+%lx (%s) | ",
1236                             mod_containing_pc((caddr_t)pc), sym, off, args);
1237                 } else {
1238                         printf("%016lx %lx (%s)\n",
1239                             (uintptr_t)fp, pc, args);
1240                         (void) snprintf(stack_buffer, sizeof (stack_buffer),
1241                             "%lx (%s) | ", pc, args);
1242                 }
1243 
1244                 if (panicstr && dump_stack_scratch) {
1245                         next_offset = offset + strlen(stack_buffer);
1246                         if (next_offset < STACK_BUF_SIZE) {
1247                                 bcopy(stack_buffer, dump_stack_scratch + offset,
1248                                     strlen(stack_buffer));
1249                                 offset = next_offset;
1250                         } else {
1251                                 /*
1252                                  * In attempting to save the panic stack
1253                                  * to the dumpbuf we have overflowed that area.
1254                                  * Print a warning and continue to printf the
1255                                  * stack to the msgbuf
1256                                  */
1257                                 printf("Warning: stack in the dump buffer"
1258                                     " may be incomplete\n");
1259                                 offset = next_offset;
1260                         }
1261                 }
1262 
1263                 pc = nextpc;
1264                 fp = nextfp;
1265         }
1266 out:
1267         if (!panicstr) {
1268                 printf("end of traceback\n");
1269                 DELAY(2 * MICROSEC);
1270         } else if (dump_stack_scratch) {
1271                 dump_stack_scratch[offset] = '\0';
1272         }
1273 }
1274 
1275 #elif defined(__i386)
1276 
1277 void
1278 traceback(caddr_t fpreg)
1279 {
1280         struct frame *fp = (struct frame *)fpreg;
1281         struct frame *nextfp, *minfp, *stacktop;
1282         uintptr_t pc, nextpc;
1283         uint_t    offset = 0;
1284         uint_t    next_offset = 0;
1285         char        stack_buffer[1024];
1286 
1287         cpu_t *cpu;
1288 
1289         /*
1290          * args[] holds TR_ARG_MAX hex long args, plus ", " or '\0'.
1291          */
1292         char args[TR_ARG_MAX * 2 + 8], *p;
1293 
1294         int on_intr;
1295         ulong_t off;
1296         char *sym;
1297 
1298         if (!panicstr)
1299                 printf("traceback: %%fp = %p\n", (void *)fp);
1300 
1301         if (panicstr && !dump_stack_scratch) {
1302                 printf("Warning - stack not written to the dumpbuf\n");
1303         }
1304 
1305         /*
1306          * If we are panicking, all high-level interrupt information in
1307          * CPU was overwritten.  panic_cpu has the correct values.
1308          */
1309         kpreempt_disable();                     /* prevent migration */
1310 
1311         cpu = (panicstr && CPU->cpu_id == panic_cpu.cpu_id)? &panic_cpu : CPU;
1312 
1313         if ((on_intr = CPU_ON_INTR(cpu)) != 0)
1314                 stacktop = (struct frame *)(cpu->cpu_intr_stack + SA(MINFRAME));
1315         else
1316                 stacktop = (struct frame *)curthread->t_stk;
1317 
1318         kpreempt_enable();
1319 
1320         fp = (struct frame *)plat_traceback(fpreg);
1321         if ((uintptr_t)fp < KERNELBASE)
1322                 goto out;
1323 
1324         minfp = fp;     /* Baseline minimum frame pointer */
1325         pc = fp->fr_savpc;
1326         fp = (struct frame *)fp->fr_savfp;
1327 
1328         while ((uintptr_t)fp >= KERNELBASE) {
1329                 ulong_t argc;
1330                 long *argv;
1331 
1332                 if (fp <= minfp || fp >= stacktop) {
1333                         if (on_intr) {
1334                                 /*
1335                                  * Hop from interrupt stack to thread stack.
1336                                  */
1337                                 stacktop = (struct frame *)curthread->t_stk;
1338                                 minfp = (struct frame *)curthread->t_stkbase;
1339                                 on_intr = 0;
1340                                 continue;
1341                         }
1342                         break; /* we're outside of the expected range */
1343                 }
1344 
1345                 if ((uintptr_t)fp & (STACK_ALIGN - 1)) {
1346                         printf("  >> mis-aligned %%fp = %p\n", (void *)fp);
1347                         break;
1348                 }
1349 
1350                 nextpc = fp->fr_savpc;
1351                 nextfp = (struct frame *)fp->fr_savfp;
1352                 argc = argcount(nextpc);
1353                 argv = (long *)((char *)fp + sizeof (struct frame));
1354 
1355                 args[0] = '\0';
1356                 p = args;
1357                 while (argc-- > 0 && argv < (long *)stacktop) {
1358                         p += snprintf(p, args + sizeof (args) - p,
1359                             "%s%lx", (p == args) ? "" : ", ", *argv++);
1360                 }
1361 
1362                 if ((sym = kobj_getsymname(pc, &off)) != NULL) {
1363                         printf("%08lx %s:%s+%lx (%s)\n", (uintptr_t)fp,
1364                             mod_containing_pc((caddr_t)pc), sym, off, args);
1365                         (void) snprintf(stack_buffer, sizeof (stack_buffer),
1366                             "%s:%s+%lx (%s) | ",
1367                             mod_containing_pc((caddr_t)pc), sym, off, args);
1368 
1369                 } else {
1370                         printf("%08lx %lx (%s)\n",
1371                             (uintptr_t)fp, pc, args);
1372                         (void) snprintf(stack_buffer, sizeof (stack_buffer),
1373                             "%lx (%s) | ", pc, args);
1374 
1375                 }
1376 
1377                 if (panicstr && dump_stack_scratch) {
1378                         next_offset = offset + strlen(stack_buffer);
1379                         if (next_offset < STACK_BUF_SIZE) {
1380                                 bcopy(stack_buffer, dump_stack_scratch + offset,
1381                                     strlen(stack_buffer));
1382                                 offset = next_offset;
1383                         } else {
1384                                 /*
1385                                  * In attempting to save the panic stack
1386                                  * to the dumpbuf we have overflowed that area.
1387                                  * Print a warning and continue to printf the
1388                                  * stack to the msgbuf
1389                                  */
1390                                 printf("Warning: stack in the dumpbuf"
1391                                     " may be incomplete\n");
1392                                 offset = next_offset;
1393                         }
1394                 }
1395 
1396                 minfp = fp;
1397                 pc = nextpc;
1398                 fp = nextfp;
1399         }
1400 out:
1401         if (!panicstr) {
1402                 printf("end of traceback\n");
1403                 DELAY(2 * MICROSEC);
1404         } else if (dump_stack_scratch) {
1405                 dump_stack_scratch[offset] = '\0';
1406         }
1407 
1408 }
1409 
1410 #endif  /* __i386 */
1411 
1412 /*
1413  * Generate a stack backtrace from a saved register set.
1414  */
1415 void
1416 traceregs(struct regs *rp)
1417 {
1418         traceback((caddr_t)rp->r_fp);
1419 }
1420 
1421 void
1422 exec_set_sp(size_t stksize)
1423 {
1424         klwp_t *lwp = ttolwp(curthread);
1425 
1426         lwptoregs(lwp)->r_sp = (uintptr_t)curproc->p_usrstack - stksize;
1427 }
1428 
1429 hrtime_t
1430 gethrtime_waitfree(void)
1431 {
1432         return (dtrace_gethrtime());
1433 }
1434 
1435 hrtime_t
1436 gethrtime(void)
1437 {
1438         return (gethrtimef());
1439 }
1440 
1441 hrtime_t
1442 gethrtime_unscaled(void)
1443 {
1444         return (gethrtimeunscaledf());
1445 }
1446 
1447 void
1448 scalehrtime(hrtime_t *hrt)
1449 {
1450         scalehrtimef(hrt);
1451 }
1452 
1453 uint64_t
1454 unscalehrtime(hrtime_t nsecs)
1455 {
1456         return (unscalehrtimef(nsecs));
1457 }
1458 
1459 void
1460 gethrestime(timespec_t *tp)
1461 {
1462         gethrestimef(tp);
1463 }
1464 
1465 #if defined(__amd64)
1466 /*
1467  * Part of the implementation of hres_tick(); this routine is
1468  * easier in C than assembler .. called with the hres_lock held.
1469  *
1470  * XX64 Many of these timekeeping variables need to be extern'ed in a header
1471  */
1472 
1473 #include <sys/time.h>
1474 #include <sys/machlock.h>
1475 
1476 extern int one_sec;
1477 extern int max_hres_adj;
1478 
1479 void
1480 __adj_hrestime(void)
1481 {
1482         long long adj;
1483 
1484         if (hrestime_adj == 0)
1485                 adj = 0;
1486         else if (hrestime_adj > 0) {
1487                 if (hrestime_adj < max_hres_adj)
1488                         adj = hrestime_adj;
1489                 else
1490                         adj = max_hres_adj;
1491         } else {
1492                 if (hrestime_adj < -max_hres_adj)
1493                         adj = -max_hres_adj;
1494                 else
1495                         adj = hrestime_adj;
1496         }
1497 
1498         timedelta -= adj;
1499         hrestime_adj = timedelta;
1500         hrestime.tv_nsec += adj;
1501 
1502         while (hrestime.tv_nsec >= NANOSEC) {
1503                 one_sec++;
1504                 hrestime.tv_sec++;
1505                 hrestime.tv_nsec -= NANOSEC;
1506         }
1507 }
1508 #endif
1509 
1510 /*
1511  * Wrapper functions to maintain backwards compability
1512  */
1513 int
1514 xcopyin(const void *uaddr, void *kaddr, size_t count)
1515 {
1516         return (xcopyin_nta(uaddr, kaddr, count, UIO_COPY_CACHED));
1517 }
1518 
1519 int
1520 xcopyout(const void *kaddr, void *uaddr, size_t count)
1521 {
1522         return (xcopyout_nta(kaddr, uaddr, count, UIO_COPY_CACHED));
1523 }