1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24 
  25 #include <sys/asm_linkage.h>
  26 #include <sys/asm_misc.h>
  27 #include <sys/regset.h>
  28 #include <sys/privregs.h>
  29 #include <sys/psw.h>
  30 #include <sys/machbrand.h>
  31 
  32 #if defined(__lint)
  33 
  34 #include <sys/types.h>
  35 #include <sys/thread.h>
  36 #include <sys/systm.h>
  37 
  38 #else   /* __lint */
  39 
  40 #include <sys/segments.h>
  41 #include <sys/pcb.h>
  42 #include <sys/trap.h>
  43 #include <sys/ftrace.h>
  44 #include <sys/traptrace.h>
  45 #include <sys/clock.h>
  46 #include <sys/model.h>
  47 #include <sys/panic.h>
  48 
  49 #if defined(__xpv)
  50 #include <sys/hypervisor.h>
  51 #endif
  52 
  53 #include "assym.h"
  54 
  55 #endif  /* __lint */
  56 
  57 /*
  58  * We implement five flavours of system call entry points
  59  *
  60  * -    syscall/sysretq         (amd64 generic)
  61  * -    syscall/sysretl         (i386 plus SYSC bit)
  62  * -    sysenter/sysexit        (i386 plus SEP bit)
  63  * -    int/iret                (i386 generic)
  64  * -    lcall/iret              (i386 generic)
  65  *
  66  * The current libc included in Solaris uses int/iret as the base unoptimized
  67  * kernel entry method. Older libc implementations and legacy binaries may use
  68  * the lcall call gate, so it must continue to be supported.
  69  *
  70  * System calls that use an lcall call gate are processed in trap() via a
  71  * segment-not-present trap, i.e. lcalls are extremely slow(!).
  72  *
  73  * The basic pattern used in the 32-bit SYSC handler at this point in time is
  74  * to have the bare minimum of assembler, and get to the C handlers as
  75  * quickly as possible.
  76  *
  77  * The 64-bit handler is much closer to the sparcv9 handler; that's
  78  * because of passing arguments in registers.  The 32-bit world still
  79  * passes arguments on the stack -- that makes that handler substantially
  80  * more complex.
  81  *
  82  * The two handlers share a few code fragments which are broken
  83  * out into preprocessor macros below.
  84  *
  85  * XX64 come back and speed all this up later.  The 32-bit stuff looks
  86  * especially easy to speed up the argument copying part ..
  87  *
  88  *
  89  * Notes about segment register usage (c.f. the 32-bit kernel)
  90  *
  91  * In the 32-bit kernel, segment registers are dutifully saved and
  92  * restored on all mode transitions because the kernel uses them directly.
  93  * When the processor is running in 64-bit mode, segment registers are
  94  * largely ignored.
  95  *
  96  * %cs and %ss
  97  *      controlled by the hardware mechanisms that make mode transitions
  98  *
  99  * The remaining segment registers have to either be pointing at a valid
 100  * descriptor i.e. with the 'present' bit set, or they can NULL descriptors
 101  *
 102  * %ds and %es
 103  *      always ignored
 104  *
 105  * %fs and %gs
 106  *      fsbase and gsbase are used to control the place they really point at.
 107  *      The kernel only depends on %gs, and controls its own gsbase via swapgs
 108  *
 109  * Note that loading segment registers is still costly because the GDT
 110  * lookup still happens (this is because the hardware can't know that we're
 111  * not setting up these segment registers for a 32-bit program).  Thus we
 112  * avoid doing this in the syscall path, and defer them to lwp context switch
 113  * handlers, so the register values remain virtualized to the lwp.
 114  */
 115 
 116 #if defined(SYSCALLTRACE)
 117 #define ORL_SYSCALLTRACE(r32)           \
 118         orl     syscalltrace(%rip), r32
 119 #else
 120 #define ORL_SYSCALLTRACE(r32)
 121 #endif
 122 
 123 /*
 124  * In the 32-bit kernel, we do absolutely nothing before getting into the
 125  * brand callback checks.  In 64-bit land, we do swapgs and then come here.
 126  * We assume that the %rsp- and %r15-stashing fields in the CPU structure
 127  * are still unused.
 128  *
 129  * Check if a brand_mach_ops callback is defined for the specified callback_id
 130  * type.  If so invoke it with the kernel's %gs value loaded and the following
 131  * data on the stack:
 132  *
 133  * stack:  --------------------------------------
 134  *      32 | callback pointer                   |
 135  *    | 24 | user (or interrupt) stack pointer  |
 136  *    | 16 | lwp pointer                        |
 137  *    v  8 | userland return address            |
 138  *       0 | callback wrapper return addr       |
 139  *         --------------------------------------
 140  *
 141  * Since we're pushing the userland return address onto the kernel stack
 142  * we need to get that address without accessing the user's stack (since we
 143  * can't trust that data).  There are different ways to get the userland
 144  * return address depending on how the syscall trap was made:
 145  *
 146  * a) For sys_syscall and sys_syscall32 the return address is in %rcx.
 147  * b) For sys_sysenter the return address is in %rdx.
 148  * c) For sys_int80 and sys_syscall_int (int91), upon entry into the macro,
 149  *    the stack pointer points at the state saved when we took the interrupt:
 150  *       ------------------------
 151  *    |  | user's %ss           |
 152  *    |  | user's %esp          |
 153  *    |  | EFLAGS register      |
 154  *    v  | user's %cs           |
 155  *       | user's %eip          |
 156  *       ------------------------
 157  *
 158  * The 2nd parameter to the BRAND_CALLBACK macro is either the
 159  * BRAND_URET_FROM_REG or BRAND_URET_FROM_INTR_STACK macro.  These macros are
 160  * used to generate the proper code to get the userland return address for
 161  * each syscall entry point.
 162  *
 163  * The interface to the brand callbacks on the 64-bit kernel assumes %r15
 164  * is available as a scratch register within the callback.  If the callback
 165  * returns within the kernel then this macro will restore %r15.  If the
 166  * callback is going to return directly to userland then it should restore
 167  * %r15 before returning to userland.
 168  */
 169 #define BRAND_URET_FROM_REG(rip_reg)                                    \
 170         pushq   rip_reg                 /* push the return address      */
 171 
 172 /*
 173  * The interrupt stack pointer we saved on entry to the BRAND_CALLBACK macro
 174  * is currently pointing at the user return address (%eip).
 175  */
 176 #define BRAND_URET_FROM_INTR_STACK()                                    \
 177         movq    %gs:CPU_RTMP_RSP, %r15  /* grab the intr. stack pointer */ ;\
 178         pushq   (%r15)                  /* push the return address      */
 179 
 180 #define BRAND_CALLBACK(callback_id, push_userland_ret)                      \
 181         movq    %rsp, %gs:CPU_RTMP_RSP  /* save the stack pointer       */ ;\
 182         movq    %r15, %gs:CPU_RTMP_R15  /* save %r15                    */ ;\
 183         movq    %gs:CPU_THREAD, %r15    /* load the thread pointer      */ ;\
 184         movq    T_STACK(%r15), %rsp     /* switch to the kernel stack   */ ;\
 185         subq    $16, %rsp               /* save space for 2 pointers    */ ;\
 186         pushq   %r14                    /* save %r14                    */ ;\
 187         movq    %gs:CPU_RTMP_RSP, %r14                                     ;\
 188         movq    %r14, 8(%rsp)           /* stash the user stack pointer */ ;\
 189         popq    %r14                    /* restore %r14                 */ ;\
 190         movq    T_LWP(%r15), %r15       /* load the lwp pointer         */ ;\
 191         pushq   %r15                    /* push the lwp pointer         */ ;\
 192         movq    LWP_PROCP(%r15), %r15   /* load the proc pointer        */ ;\
 193         movq    P_BRAND(%r15), %r15     /* load the brand pointer       */ ;\
 194         movq    B_MACHOPS(%r15), %r15   /* load the machops pointer     */ ;\
 195         movq    _CONST(_MUL(callback_id, CPTRSIZE))(%r15), %r15            ;\
 196         cmpq    $0, %r15                                                   ;\
 197         je      1f                                                         ;\
 198         movq    %r15, 16(%rsp)          /* save the callback pointer    */ ;\
 199         push_userland_ret               /* push the return address      */ ;\
 200         call    *24(%rsp)               /* call callback                */ ;\
 201 1:      movq    %gs:CPU_RTMP_R15, %r15  /* restore %r15                 */ ;\
 202         movq    %gs:CPU_RTMP_RSP, %rsp  /* restore the stack pointer    */
 203 
 204 #define MSTATE_TRANSITION(from, to)             \
 205         movl    $from, %edi;                    \
 206         movl    $to, %esi;                      \
 207         call    syscall_mstate
 208 
 209 /*
 210  * Check to see if a simple (direct) return is possible i.e.
 211  *
 212  *      if (t->t_post_sys_ast | syscalltrace |
 213  *          lwp->lwp_pcb.pcb_rupdate == 1)
 214  *              do full version ;
 215  *
 216  * Preconditions:
 217  * -    t is curthread
 218  * Postconditions:
 219  * -    condition code NE is set if post-sys is too complex
 220  * -    rtmp is zeroed if it isn't (we rely on this!)
 221  * -    ltmp is smashed
 222  */
 223 #define CHECK_POSTSYS_NE(t, ltmp, rtmp)                 \
 224         movq    T_LWP(t), ltmp;                         \
 225         movzbl  PCB_RUPDATE(ltmp), rtmp;                \
 226         ORL_SYSCALLTRACE(rtmp);                         \
 227         orl     T_POST_SYS_AST(t), rtmp;                \
 228         cmpl    $0, rtmp
 229         
 230 /*
 231  * Fix up the lwp, thread, and eflags for a successful return
 232  *
 233  * Preconditions:
 234  * -    zwreg contains zero
 235  */
 236 #define SIMPLE_SYSCALL_POSTSYS(t, lwp, zwreg)           \
 237         movb    $LWP_USER, LWP_STATE(lwp);              \
 238         movw    zwreg, T_SYSNUM(t);                     \
 239         andb    $_CONST(0xffff - PS_C), REGOFF_RFL(%rsp)
 240 
 241 /*
 242  * ASSERT(lwptoregs(lwp) == rp);
 243  *
 244  * This may seem obvious, but very odd things happen if this
 245  * assertion is false
 246  *
 247  * Preconditions:
 248  *      (%rsp is ready for normal call sequence)
 249  * Postconditions (if assertion is true):
 250  *      %r11 is smashed
 251  *
 252  * ASSERT(rp->r_cs == descnum)
 253  *
 254  * The code selector is written into the regs structure when the
 255  * lwp stack is created.  We use this ASSERT to validate that
 256  * the regs structure really matches how we came in.
 257  *
 258  * Preconditions:
 259  *      (%rsp is ready for normal call sequence)
 260  * Postconditions (if assertion is true):
 261  *      -none-
 262  *
 263  * ASSERT(lwp->lwp_pcb.pcb_rupdate == 0);
 264  *
 265  * If this is false, it meant that we returned to userland without
 266  * updating the segment registers as we were supposed to.
 267  *
 268  * Note that we must ensure no interrupts or other traps intervene
 269  * between entering privileged mode and performing the assertion,
 270  * otherwise we may perform a context switch on the thread, which
 271  * will end up setting pcb_rupdate to 1 again.
 272  */
 273 #if defined(DEBUG)
 274 
 275 #if !defined(__lint)
 276 
 277 __lwptoregs_msg:
 278         .string "syscall_asm_amd64.s:%d lwptoregs(%p) [%p] != rp [%p]"
 279 
 280 __codesel_msg:
 281         .string "syscall_asm_amd64.s:%d rp->r_cs [%ld] != %ld"
 282 
 283 __no_rupdate_msg:
 284         .string "syscall_asm_amd64.s:%d lwp %p, pcb_rupdate != 0"
 285 
 286 #endif  /* !__lint */
 287 
 288 #define ASSERT_LWPTOREGS(lwp, rp)                       \
 289         movq    LWP_REGS(lwp), %r11;                    \
 290         cmpq    rp, %r11;                               \
 291         je      7f;                                     \
 292         leaq    __lwptoregs_msg(%rip), %rdi;            \
 293         movl    $__LINE__, %esi;                        \
 294         movq    lwp, %rdx;                              \
 295         movq    %r11, %rcx;                             \
 296         movq    rp, %r8;                                \
 297         xorl    %eax, %eax;                             \
 298         call    panic;                                  \
 299 7:
 300 
 301 #define ASSERT_NO_RUPDATE_PENDING(lwp)                  \
 302         testb   $0x1, PCB_RUPDATE(lwp);                 \
 303         je      8f;                                     \
 304         movq    lwp, %rdx;                              \
 305         leaq    __no_rupdate_msg(%rip), %rdi;           \
 306         movl    $__LINE__, %esi;                        \
 307         xorl    %eax, %eax;                             \
 308         call    panic;                                  \
 309 8:
 310 
 311 #else
 312 #define ASSERT_LWPTOREGS(lwp, rp)
 313 #define ASSERT_NO_RUPDATE_PENDING(lwp)
 314 #endif
 315 
 316 /*
 317  * Do the traptrace thing and restore any registers we used
 318  * in situ.  Assumes that %rsp is pointing at the base of
 319  * the struct regs, obviously ..
 320  */     
 321 #ifdef TRAPTRACE        
 322 #define SYSCALL_TRAPTRACE(ttype)                                \
 323         TRACE_PTR(%rdi, %rbx, %ebx, %rcx, ttype);               \
 324         TRACE_REGS(%rdi, %rsp, %rbx, %rcx);                     \
 325         TRACE_STAMP(%rdi);      /* rdtsc clobbers %eax, %edx */ \
 326         movq    REGOFF_RAX(%rsp), %rax;                         \
 327         movq    REGOFF_RBX(%rsp), %rbx;                         \
 328         movq    REGOFF_RCX(%rsp), %rcx;                         \
 329         movq    REGOFF_RDX(%rsp), %rdx;                         \
 330         movl    %eax, TTR_SYSNUM(%rdi);                         \
 331         movq    REGOFF_RDI(%rsp), %rdi
 332 
 333 #define SYSCALL_TRAPTRACE32(ttype)                              \
 334         SYSCALL_TRAPTRACE(ttype);                               \
 335         /* paranoia: clean the top 32-bits of the registers */  \
 336         orl     %eax, %eax;                                     \
 337         orl     %ebx, %ebx;                                     \
 338         orl     %ecx, %ecx;                                     \
 339         orl     %edx, %edx;                                     \
 340         orl     %edi, %edi      
 341 #else   /* TRAPTRACE */
 342 #define SYSCALL_TRAPTRACE(ttype)
 343 #define SYSCALL_TRAPTRACE32(ttype)      
 344 #endif  /* TRAPTRACE */
 345 
 346 /*
 347  * The 64-bit libc syscall wrapper does this:
 348  *
 349  * fn(<args>)
 350  * {
 351  *      movq    %rcx, %r10      -- because syscall smashes %rcx
 352  *      movl    $CODE, %eax
 353  *      syscall
 354  *      <error processing>
 355  * }
 356  *
 357  * Thus when we come into the kernel:
 358  *
 359  *      %rdi, %rsi, %rdx, %r10, %r8, %r9 contain first six args
 360  *      %rax is the syscall number
 361  *      %r12-%r15 contain caller state
 362  *
 363  * The syscall instruction arranges that:
 364  *      
 365  *      %rcx contains the return %rip
 366  *      %r11d contains bottom 32-bits of %rflags
 367  *      %rflags is masked (as determined by the SFMASK msr)
 368  *      %cs is set to UCS_SEL (as determined by the STAR msr)
 369  *      %ss is set to UDS_SEL (as determined by the STAR msr)
 370  *      %rip is set to sys_syscall (as determined by the LSTAR msr)
 371  *
 372  * Or in other words, we have no registers available at all.
 373  * Only swapgs can save us!
 374  *
 375  * Under the hypervisor, the swapgs has happened already.  However, the
 376  * state of the world is very different from that we're familiar with.
 377  *
 378  * In particular, we have a stack structure like that for interrupt
 379  * gates, except that the %cs and %ss registers are modified for reasons
 380  * that are not entirely clear.  Critically, the %rcx/%r11 values do
 381  * *not* reflect the usage of those registers under a 'real' syscall[1];
 382  * the stack, therefore, looks like this:
 383  *
 384  *      0x0(rsp)        potentially junk %rcx
 385  *      0x8(rsp)        potentially junk %r11
 386  *      0x10(rsp)       user %rip
 387  *      0x18(rsp)       modified %cs
 388  *      0x20(rsp)       user %rflags
 389  *      0x28(rsp)       user %rsp
 390  *      0x30(rsp)       modified %ss
 391  *
 392  *
 393  * and before continuing on, we must load the %rip into %rcx and the
 394  * %rflags into %r11.
 395  *
 396  * [1] They used to, and we relied on it, but this was broken in 3.1.1.
 397  * Sigh.
 398  */
 399 #if defined(__xpv)
 400 #define XPV_SYSCALL_PROD                                                \
 401         movq    0x10(%rsp), %rcx;                                       \
 402         movq    0x20(%rsp), %r11;                                       \
 403         movq    0x28(%rsp), %rsp
 404 #else
 405 #define XPV_SYSCALL_PROD /* nothing */
 406 #endif
 407 
 408 #if defined(__lint)
 409 
 410 /*ARGSUSED*/
 411 void
 412 sys_syscall()
 413 {}
 414 
 415 void
 416 _allsyscalls()
 417 {}
 418 
 419 size_t _allsyscalls_size;
 420 
 421 #else   /* __lint */
 422 
 423         ENTRY_NP2(brand_sys_syscall,_allsyscalls)
 424         SWAPGS                          /* kernel gsbase */
 425         XPV_SYSCALL_PROD
 426         BRAND_CALLBACK(BRAND_CB_SYSCALL, BRAND_URET_FROM_REG(%rcx))
 427         jmp     noprod_sys_syscall
 428 
 429         ALTENTRY(sys_syscall)
 430         SWAPGS                          /* kernel gsbase */
 431         XPV_SYSCALL_PROD
 432 
 433 noprod_sys_syscall:
 434         movq    %r15, %gs:CPU_RTMP_R15
 435         movq    %rsp, %gs:CPU_RTMP_RSP
 436 
 437         movq    %gs:CPU_THREAD, %r15
 438         movq    T_STACK(%r15), %rsp     /* switch from user to kernel stack */
 439 
 440         ASSERT_UPCALL_MASK_IS_SET
 441 
 442         movl    $UCS_SEL, REGOFF_CS(%rsp)
 443         movq    %rcx, REGOFF_RIP(%rsp)          /* syscall: %rip -> %rcx */
 444         movq    %r11, REGOFF_RFL(%rsp)          /* syscall: %rfl -> %r11d */
 445         movl    $UDS_SEL, REGOFF_SS(%rsp)
 446 
 447         movl    %eax, %eax                      /* wrapper: sysc# -> %eax */
 448         movq    %rdi, REGOFF_RDI(%rsp)
 449         movq    %rsi, REGOFF_RSI(%rsp)
 450         movq    %rdx, REGOFF_RDX(%rsp)
 451         movq    %r10, REGOFF_RCX(%rsp)          /* wrapper: %rcx -> %r10 */
 452         movq    %r10, %rcx                      /* arg[3] for direct calls */
 453 
 454         movq    %r8, REGOFF_R8(%rsp)
 455         movq    %r9, REGOFF_R9(%rsp)
 456         movq    %rax, REGOFF_RAX(%rsp)
 457         movq    %rbx, REGOFF_RBX(%rsp)
 458 
 459         movq    %rbp, REGOFF_RBP(%rsp)
 460         movq    %r10, REGOFF_R10(%rsp)
 461         movq    %gs:CPU_RTMP_RSP, %r11
 462         movq    %r11, REGOFF_RSP(%rsp)
 463         movq    %r12, REGOFF_R12(%rsp)
 464 
 465         movq    %r13, REGOFF_R13(%rsp)
 466         movq    %r14, REGOFF_R14(%rsp)
 467         movq    %gs:CPU_RTMP_R15, %r10
 468         movq    %r10, REGOFF_R15(%rsp)
 469         movq    $0, REGOFF_SAVFP(%rsp)
 470         movq    $0, REGOFF_SAVPC(%rsp)
 471 
 472         /*
 473          * Copy these registers here in case we end up stopped with
 474          * someone (like, say, /proc) messing with our register state.
 475          * We don't -restore- them unless we have to in update_sregs.
 476          *
 477          * Since userland -can't- change fsbase or gsbase directly,
 478          * and capturing them involves two serializing instructions,
 479          * we don't bother to capture them here.
 480          */
 481         xorl    %ebx, %ebx
 482         movw    %ds, %bx
 483         movq    %rbx, REGOFF_DS(%rsp)
 484         movw    %es, %bx
 485         movq    %rbx, REGOFF_ES(%rsp)
 486         movw    %fs, %bx
 487         movq    %rbx, REGOFF_FS(%rsp)
 488         movw    %gs, %bx
 489         movq    %rbx, REGOFF_GS(%rsp)
 490 
 491         /*
 492          * Machine state saved in the regs structure on the stack
 493          * First six args in %rdi, %rsi, %rdx, %rcx, %r8, %r9
 494          * %eax is the syscall number
 495          * %rsp is the thread's stack, %r15 is curthread
 496          * REG_RSP(%rsp) is the user's stack
 497          */
 498 
 499         SYSCALL_TRAPTRACE($TT_SYSC64)
 500 
 501         movq    %rsp, %rbp
 502         
 503         movq    T_LWP(%r15), %r14
 504         ASSERT_NO_RUPDATE_PENDING(%r14)
 505         ENABLE_INTR_FLAGS
 506 
 507         MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
 508         movl    REGOFF_RAX(%rsp), %eax  /* (%rax damaged by mstate call) */
 509 
 510         ASSERT_LWPTOREGS(%r14, %rsp)
 511 
 512         movb    $LWP_SYS, LWP_STATE(%r14)
 513         incq    LWP_RU_SYSC(%r14)
 514         movb    $NORMALRETURN, LWP_EOSYS(%r14)
 515 
 516         incq    %gs:CPU_STATS_SYS_SYSCALL
 517 
 518         movw    %ax, T_SYSNUM(%r15)
 519         movzbl  T_PRE_SYS(%r15), %ebx
 520         ORL_SYSCALLTRACE(%ebx)
 521         testl   %ebx, %ebx
 522         jne     _syscall_pre
 523 
 524 _syscall_invoke:
 525         movq    REGOFF_RDI(%rbp), %rdi
 526         movq    REGOFF_RSI(%rbp), %rsi
 527         movq    REGOFF_RDX(%rbp), %rdx
 528         movq    REGOFF_RCX(%rbp), %rcx
 529         movq    REGOFF_R8(%rbp), %r8
 530         movq    REGOFF_R9(%rbp), %r9
 531 
 532         cmpl    $NSYSCALL, %eax
 533         jae     _syscall_ill    
 534         shll    $SYSENT_SIZE_SHIFT, %eax
 535         leaq    sysent(%rax), %rbx
 536 
 537         call    *SY_CALLC(%rbx)
 538 
 539         movq    %rax, %r12
 540         movq    %rdx, %r13
 541 
 542         /*
 543          * If the handler returns two ints, then we need to split the
 544          * 64-bit return value into two 32-bit values.
 545          */
 546         testw   $SE_32RVAL2, SY_FLAGS(%rbx)
 547         je      5f
 548         movq    %r12, %r13
 549         shrq    $32, %r13       /* upper 32-bits into %edx */
 550         movl    %r12d, %r12d    /* lower 32-bits into %eax */
 551 5:
 552         /*
 553          * Optimistically assume that there's no post-syscall
 554          * work to do.  (This is to avoid having to call syscall_mstate()
 555          * with interrupts disabled)
 556          */
 557         MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
 558 
 559         /*
 560          * We must protect ourselves from being descheduled here;
 561          * If we were, and we ended up on another cpu, or another
 562          * lwp got in ahead of us, it could change the segment
 563          * registers without us noticing before we return to userland.
 564          */
 565         CLI(%r14)
 566         CHECK_POSTSYS_NE(%r15, %r14, %ebx)
 567         jne     _syscall_post
 568         SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
 569 
 570         movq    %r12, REGOFF_RAX(%rsp)
 571         movq    %r13, REGOFF_RDX(%rsp)
 572 
 573         /*
 574          * To get back to userland, we need the return %rip in %rcx and
 575          * the return %rfl in %r11d.  The sysretq instruction also arranges
 576          * to fix up %cs and %ss; everything else is our responsibility.
 577          */
 578         movq    REGOFF_RDI(%rsp), %rdi
 579         movq    REGOFF_RSI(%rsp), %rsi
 580         movq    REGOFF_RDX(%rsp), %rdx
 581         /* %rcx used to restore %rip value */
 582 
 583         movq    REGOFF_R8(%rsp), %r8
 584         movq    REGOFF_R9(%rsp), %r9
 585         movq    REGOFF_RAX(%rsp), %rax
 586         movq    REGOFF_RBX(%rsp), %rbx
 587 
 588         movq    REGOFF_RBP(%rsp), %rbp  
 589         movq    REGOFF_R10(%rsp), %r10
 590         /* %r11 used to restore %rfl value */
 591         movq    REGOFF_R12(%rsp), %r12
 592 
 593         movq    REGOFF_R13(%rsp), %r13
 594         movq    REGOFF_R14(%rsp), %r14
 595         movq    REGOFF_R15(%rsp), %r15
 596 
 597         movq    REGOFF_RIP(%rsp), %rcx  
 598         movl    REGOFF_RFL(%rsp), %r11d
 599 
 600 #if defined(__xpv)
 601         addq    $REGOFF_RIP, %rsp
 602 #else
 603         movq    REGOFF_RSP(%rsp), %rsp
 604 #endif
 605 
 606         /*
 607          * There can be no instructions between the ALTENTRY below and
 608          * SYSRET or we could end up breaking brand support. See label usage
 609          * in sn1_brand_syscall_callback for an example.
 610          */
 611         ASSERT_UPCALL_MASK_IS_SET
 612 #if defined(__xpv)
 613         SYSRETQ
 614         ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
 615 
 616         /*
 617          * We can only get here after executing a brand syscall
 618          * interposition callback handler and simply need to
 619          * "sysretq" back to userland. On the hypervisor this
 620          * involves the iret hypercall which requires us to construct
 621          * just enough of the stack needed for the hypercall.
 622          * (rip, cs, rflags, rsp, ss).
 623          */
 624         movq    %rsp, %gs:CPU_RTMP_RSP          /* save user's rsp */
 625         movq    %gs:CPU_THREAD, %r11
 626         movq    T_STACK(%r11), %rsp
 627 
 628         movq    %rcx, REGOFF_RIP(%rsp)
 629         movl    $UCS_SEL, REGOFF_CS(%rsp)
 630         movq    %gs:CPU_RTMP_RSP, %r11
 631         movq    %r11, REGOFF_RSP(%rsp)
 632         pushfq
 633         popq    %r11                            /* hypercall enables ints */
 634         movq    %r11, REGOFF_RFL(%rsp)
 635         movl    $UDS_SEL, REGOFF_SS(%rsp)
 636         addq    $REGOFF_RIP, %rsp
 637         /*
 638          * XXPV: see comment in SYSRETQ definition for future optimization
 639          *       we could take.
 640          */
 641         ASSERT_UPCALL_MASK_IS_SET
 642         SYSRETQ
 643 #else
 644         ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
 645         SWAPGS                          /* user gsbase */
 646         SYSRETQ
 647 #endif
 648         /*NOTREACHED*/
 649         SET_SIZE(nopop_sys_syscall_swapgs_sysretq)
 650 
 651 _syscall_pre:
 652         call    pre_syscall
 653         movl    %eax, %r12d
 654         testl   %eax, %eax
 655         jne     _syscall_post_call
 656         /*
 657          * Didn't abort, so reload the syscall args and invoke the handler.
 658          */
 659         movzwl  T_SYSNUM(%r15), %eax    
 660         jmp     _syscall_invoke
 661 
 662 _syscall_ill:
 663         call    nosys
 664         movq    %rax, %r12
 665         movq    %rdx, %r13
 666         jmp     _syscall_post_call
 667 
 668 _syscall_post:
 669         STI
 670         /*
 671          * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
 672          * so that we can account for the extra work it takes us to finish.
 673          */
 674         MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
 675 _syscall_post_call:
 676         movq    %r12, %rdi
 677         movq    %r13, %rsi
 678         call    post_syscall
 679         MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
 680         jmp     _sys_rtt
 681         SET_SIZE(sys_syscall)
 682         SET_SIZE(brand_sys_syscall)
 683 
 684 #endif  /* __lint */
 685 
 686 #if defined(__lint)
 687 
 688 /*ARGSUSED*/
 689 void
 690 sys_syscall32()
 691 {}
 692 
 693 #else   /* __lint */
 694 
 695         ENTRY_NP(brand_sys_syscall32)
 696         SWAPGS                          /* kernel gsbase */
 697         XPV_TRAP_POP
 698         BRAND_CALLBACK(BRAND_CB_SYSCALL32, BRAND_URET_FROM_REG(%rcx))
 699         jmp     nopop_sys_syscall32
 700 
 701         ALTENTRY(sys_syscall32)
 702         SWAPGS                          /* kernel gsbase */
 703         XPV_TRAP_POP
 704 
 705 nopop_sys_syscall32:
 706         movl    %esp, %r10d
 707         movq    %gs:CPU_THREAD, %r15
 708         movq    T_STACK(%r15), %rsp
 709         movl    %eax, %eax
 710 
 711         movl    $U32CS_SEL, REGOFF_CS(%rsp)
 712         movl    %ecx, REGOFF_RIP(%rsp)          /* syscall: %rip -> %rcx */
 713         movq    %r11, REGOFF_RFL(%rsp)          /* syscall: %rfl -> %r11d */
 714         movq    %r10, REGOFF_RSP(%rsp)
 715         movl    $UDS_SEL, REGOFF_SS(%rsp)
 716 
 717 _syscall32_save:
 718         movl    %edi, REGOFF_RDI(%rsp)
 719         movl    %esi, REGOFF_RSI(%rsp)
 720         movl    %ebp, REGOFF_RBP(%rsp)
 721         movl    %ebx, REGOFF_RBX(%rsp)
 722         movl    %edx, REGOFF_RDX(%rsp)
 723         movl    %ecx, REGOFF_RCX(%rsp)
 724         movl    %eax, REGOFF_RAX(%rsp)          /* wrapper: sysc# -> %eax */
 725         movq    $0, REGOFF_SAVFP(%rsp)
 726         movq    $0, REGOFF_SAVPC(%rsp)
 727 
 728         /*
 729          * Copy these registers here in case we end up stopped with
 730          * someone (like, say, /proc) messing with our register state.
 731          * We don't -restore- them unless we have to in update_sregs.
 732          *
 733          * Since userland -can't- change fsbase or gsbase directly,
 734          * we don't bother to capture them here.
 735          */
 736         xorl    %ebx, %ebx
 737         movw    %ds, %bx
 738         movq    %rbx, REGOFF_DS(%rsp)
 739         movw    %es, %bx
 740         movq    %rbx, REGOFF_ES(%rsp)
 741         movw    %fs, %bx
 742         movq    %rbx, REGOFF_FS(%rsp)
 743         movw    %gs, %bx
 744         movq    %rbx, REGOFF_GS(%rsp)
 745 
 746         /*
 747          * Application state saved in the regs structure on the stack
 748          * %eax is the syscall number
 749          * %rsp is the thread's stack, %r15 is curthread
 750          * REG_RSP(%rsp) is the user's stack
 751          */
 752 
 753         SYSCALL_TRAPTRACE32($TT_SYSC)
 754 
 755         movq    %rsp, %rbp
 756 
 757         movq    T_LWP(%r15), %r14
 758         ASSERT_NO_RUPDATE_PENDING(%r14)
 759 
 760         ENABLE_INTR_FLAGS
 761 
 762         MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
 763         movl    REGOFF_RAX(%rsp), %eax  /* (%rax damaged by mstate call) */
 764 
 765         ASSERT_LWPTOREGS(%r14, %rsp)
 766 
 767         incq     %gs:CPU_STATS_SYS_SYSCALL
 768 
 769         /*
 770          * Make some space for MAXSYSARGS (currently 8) 32-bit args placed
 771          * into 64-bit (long) arg slots, maintaining 16 byte alignment.  Or
 772          * more succinctly:
 773          *
 774          *      SA(MAXSYSARGS * sizeof (long)) == 64
 775          */
 776 #define SYS_DROP        64                      /* drop for args */
 777         subq    $SYS_DROP, %rsp
 778         movb    $LWP_SYS, LWP_STATE(%r14)
 779         movq    %r15, %rdi
 780         movq    %rsp, %rsi
 781         call    syscall_entry
 782 
 783         /*
 784          * Fetch the arguments copied onto the kernel stack and put
 785          * them in the right registers to invoke a C-style syscall handler.
 786          * %rax contains the handler address.
 787          *
 788          * Ideas for making all this go faster of course include simply
 789          * forcibly fetching 6 arguments from the user stack under lofault
 790          * protection, reverting to copyin_args only when watchpoints
 791          * are in effect.
 792          *
 793          * (If we do this, make sure that exec and libthread leave
 794          * enough space at the top of the stack to ensure that we'll
 795          * never do a fetch from an invalid page.)
 796          *
 797          * Lots of ideas here, but they won't really help with bringup B-)
 798          * Correctness can't wait, performance can wait a little longer ..
 799          */
 800 
 801         movq    %rax, %rbx
 802         movl    0(%rsp), %edi
 803         movl    8(%rsp), %esi
 804         movl    0x10(%rsp), %edx
 805         movl    0x18(%rsp), %ecx
 806         movl    0x20(%rsp), %r8d
 807         movl    0x28(%rsp), %r9d
 808 
 809         call    *SY_CALLC(%rbx)
 810 
 811         movq    %rbp, %rsp      /* pop the args */
 812 
 813         /*
 814          * amd64 syscall handlers -always- return a 64-bit value in %rax.
 815          * On the 32-bit kernel, they always return that value in %eax:%edx
 816          * as required by the 32-bit ABI.
 817          *
 818          * Simulate the same behaviour by unconditionally splitting the
 819          * return value in the same way.
 820          */
 821         movq    %rax, %r13
 822         shrq    $32, %r13       /* upper 32-bits into %edx */
 823         movl    %eax, %r12d     /* lower 32-bits into %eax */
 824 
 825         /*
 826          * Optimistically assume that there's no post-syscall
 827          * work to do.  (This is to avoid having to call syscall_mstate()
 828          * with interrupts disabled)
 829          */
 830         MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
 831 
 832         /*
 833          * We must protect ourselves from being descheduled here;
 834          * If we were, and we ended up on another cpu, or another
 835          * lwp got in ahead of us, it could change the segment
 836          * registers without us noticing before we return to userland.
 837          */
 838         CLI(%r14)
 839         CHECK_POSTSYS_NE(%r15, %r14, %ebx)
 840         jne     _full_syscall_postsys32
 841         SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
 842 
 843         /*
 844          * To get back to userland, we need to put the return %rip in %rcx and
 845          * the return %rfl in %r11d.  The sysret instruction also arranges
 846          * to fix up %cs and %ss; everything else is our responsibility.
 847          */
 848 
 849         movl    %r12d, %eax                     /* %eax: rval1 */
 850         movl    REGOFF_RBX(%rsp), %ebx
 851         /* %ecx used for return pointer */
 852         movl    %r13d, %edx                     /* %edx: rval2 */
 853         movl    REGOFF_RBP(%rsp), %ebp
 854         movl    REGOFF_RSI(%rsp), %esi
 855         movl    REGOFF_RDI(%rsp), %edi
 856 
 857         movl    REGOFF_RFL(%rsp), %r11d         /* %r11 -> eflags */
 858         movl    REGOFF_RIP(%rsp), %ecx          /* %ecx -> %eip */
 859         movl    REGOFF_RSP(%rsp), %esp
 860 
 861         ASSERT_UPCALL_MASK_IS_SET
 862         ALTENTRY(nopop_sys_syscall32_swapgs_sysretl)
 863         SWAPGS                          /* user gsbase */
 864         SYSRETL
 865         SET_SIZE(nopop_sys_syscall32_swapgs_sysretl)
 866         /*NOTREACHED*/
 867 
 868 _full_syscall_postsys32:
 869         STI
 870         /*
 871          * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
 872          * so that we can account for the extra work it takes us to finish.
 873          */
 874         MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
 875         movq    %r15, %rdi
 876         movq    %r12, %rsi                      /* rval1 - %eax */
 877         movq    %r13, %rdx                      /* rval2 - %edx */
 878         call    syscall_exit
 879         MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
 880         jmp     _sys_rtt
 881         SET_SIZE(sys_syscall32)
 882         SET_SIZE(brand_sys_syscall32)
 883 
 884 #endif  /* __lint */
 885 
 886 /*
 887  * System call handler via the sysenter instruction
 888  * Used only for 32-bit system calls on the 64-bit kernel.
 889  *
 890  * The caller in userland has arranged that:
 891  *
 892  * -    %eax contains the syscall number
 893  * -    %ecx contains the user %esp
 894  * -    %edx contains the return %eip
 895  * -    the user stack contains the args to the syscall
 896  *
 897  * Hardware and (privileged) initialization code have arranged that by
 898  * the time the sysenter instructions completes:
 899  *
 900  * - %rip is pointing to sys_sysenter (below).
 901  * - %cs and %ss are set to kernel text and stack (data) selectors.
 902  * - %rsp is pointing at the lwp's stack
 903  * - interrupts have been disabled.
 904  *
 905  * Note that we are unable to return both "rvals" to userland with
 906  * this call, as %edx is used by the sysexit instruction.
 907  *
 908  * One final complication in this routine is its interaction with
 909  * single-stepping in a debugger.  For most of the system call mechanisms,
 910  * the CPU automatically clears the single-step flag before we enter the
 911  * kernel.  The sysenter mechanism does not clear the flag, so a user
 912  * single-stepping through a libc routine may suddenly find him/herself
 913  * single-stepping through the kernel.  To detect this, kmdb compares the
 914  * trap %pc to the [brand_]sys_enter addresses on each single-step trap.
 915  * If it finds that we have single-stepped to a sysenter entry point, it
 916  * explicitly clears the flag and executes the sys_sysenter routine.
 917  *
 918  * One final complication in this final complication is the fact that we
 919  * have two different entry points for sysenter: brand_sys_sysenter and
 920  * sys_sysenter.  If we enter at brand_sys_sysenter and start single-stepping
 921  * through the kernel with kmdb, we will eventually hit the instruction at
 922  * sys_sysenter.  kmdb cannot distinguish between that valid single-step
 923  * and the undesirable one mentioned above.  To avoid this situation, we
 924  * simply add a jump over the instruction at sys_sysenter to make it
 925  * impossible to single-step to it.
 926  */
 927 #if defined(__lint)
 928 
 929 void
 930 sys_sysenter()
 931 {}
 932 
 933 #else   /* __lint */
 934 
 935         ENTRY_NP(brand_sys_sysenter)
 936         SWAPGS                          /* kernel gsbase */
 937         ALTENTRY(_brand_sys_sysenter_post_swapgs)
 938         BRAND_CALLBACK(BRAND_CB_SYSENTER, BRAND_URET_FROM_REG(%rdx))
 939         /*
 940          * Jump over sys_sysenter to allow single-stepping as described
 941          * above.
 942          */
 943         jmp     _sys_sysenter_post_swapgs
 944 
 945         ALTENTRY(sys_sysenter)
 946         SWAPGS                          /* kernel gsbase */
 947 
 948         ALTENTRY(_sys_sysenter_post_swapgs)
 949         movq    %gs:CPU_THREAD, %r15
 950 
 951         movl    $U32CS_SEL, REGOFF_CS(%rsp)
 952         movl    %ecx, REGOFF_RSP(%rsp)          /* wrapper: %esp -> %ecx */
 953         movl    %edx, REGOFF_RIP(%rsp)          /* wrapper: %eip -> %edx */
 954         pushfq
 955         popq    %r10
 956         movl    $UDS_SEL, REGOFF_SS(%rsp)
 957 
 958         /*
 959          * Set the interrupt flag before storing the flags to the
 960          * flags image on the stack so we can return to user with
 961          * interrupts enabled if we return via sys_rtt_syscall32
 962          */
 963         orq     $PS_IE, %r10
 964         movq    %r10, REGOFF_RFL(%rsp)
 965 
 966         movl    %edi, REGOFF_RDI(%rsp)
 967         movl    %esi, REGOFF_RSI(%rsp)
 968         movl    %ebp, REGOFF_RBP(%rsp)
 969         movl    %ebx, REGOFF_RBX(%rsp)
 970         movl    %edx, REGOFF_RDX(%rsp)
 971         movl    %ecx, REGOFF_RCX(%rsp)
 972         movl    %eax, REGOFF_RAX(%rsp)          /* wrapper: sysc# -> %eax */
 973         movq    $0, REGOFF_SAVFP(%rsp)
 974         movq    $0, REGOFF_SAVPC(%rsp)
 975 
 976         /*
 977          * Copy these registers here in case we end up stopped with
 978          * someone (like, say, /proc) messing with our register state.
 979          * We don't -restore- them unless we have to in update_sregs.
 980          *
 981          * Since userland -can't- change fsbase or gsbase directly,
 982          * we don't bother to capture them here.
 983          */
 984         xorl    %ebx, %ebx
 985         movw    %ds, %bx
 986         movq    %rbx, REGOFF_DS(%rsp)
 987         movw    %es, %bx
 988         movq    %rbx, REGOFF_ES(%rsp)
 989         movw    %fs, %bx
 990         movq    %rbx, REGOFF_FS(%rsp)
 991         movw    %gs, %bx
 992         movq    %rbx, REGOFF_GS(%rsp)
 993 
 994         /*
 995          * Application state saved in the regs structure on the stack
 996          * %eax is the syscall number
 997          * %rsp is the thread's stack, %r15 is curthread
 998          * REG_RSP(%rsp) is the user's stack
 999          */
1000 
1001         SYSCALL_TRAPTRACE($TT_SYSENTER)
1002 
1003         movq    %rsp, %rbp
1004 
1005         movq    T_LWP(%r15), %r14
1006         ASSERT_NO_RUPDATE_PENDING(%r14)
1007 
1008         ENABLE_INTR_FLAGS
1009 
1010         /*
1011          * Catch 64-bit process trying to issue sysenter instruction
1012          * on Nocona based systems.
1013          */
1014         movq    LWP_PROCP(%r14), %rax
1015         cmpq    $DATAMODEL_ILP32, P_MODEL(%rax)
1016         je      7f
1017 
1018         /*
1019          * For a non-32-bit process, simulate a #ud, since that's what
1020          * native hardware does.  The traptrace entry (above) will
1021          * let you know what really happened.
1022          */
1023         movq    $T_ILLINST, REGOFF_TRAPNO(%rsp)
1024         movq    REGOFF_CS(%rsp), %rdi
1025         movq    %rdi, REGOFF_ERR(%rsp)
1026         movq    %rsp, %rdi
1027         movq    REGOFF_RIP(%rsp), %rsi
1028         movl    %gs:CPU_ID, %edx
1029         call    trap
1030         jmp     _sys_rtt
1031 7:
1032 
1033         MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
1034         movl    REGOFF_RAX(%rsp), %eax  /* (%rax damaged by mstate calls) */
1035 
1036         ASSERT_LWPTOREGS(%r14, %rsp)
1037 
1038         incq    %gs:CPU_STATS_SYS_SYSCALL
1039 
1040         /*
1041          * Make some space for MAXSYSARGS (currently 8) 32-bit args
1042          * placed into 64-bit (long) arg slots, plus one 64-bit
1043          * (long) arg count, maintaining 16 byte alignment.
1044          */
1045         subq    $SYS_DROP, %rsp
1046         movb    $LWP_SYS, LWP_STATE(%r14)
1047         movq    %r15, %rdi
1048         movq    %rsp, %rsi
1049         call    syscall_entry
1050 
1051         /*
1052          * Fetch the arguments copied onto the kernel stack and put
1053          * them in the right registers to invoke a C-style syscall handler.
1054          * %rax contains the handler address.
1055          */
1056         movq    %rax, %rbx
1057         movl    0(%rsp), %edi
1058         movl    8(%rsp), %esi
1059         movl    0x10(%rsp), %edx
1060         movl    0x18(%rsp), %ecx
1061         movl    0x20(%rsp), %r8d
1062         movl    0x28(%rsp), %r9d
1063 
1064         call    *SY_CALLC(%rbx)
1065 
1066         movq    %rbp, %rsp      /* pop the args */
1067 
1068         /*
1069          * amd64 syscall handlers -always- return a 64-bit value in %rax.
1070          * On the 32-bit kernel, the always return that value in %eax:%edx
1071          * as required by the 32-bit ABI.
1072          *
1073          * Simulate the same behaviour by unconditionally splitting the
1074          * return value in the same way.
1075          */
1076         movq    %rax, %r13
1077         shrq    $32, %r13       /* upper 32-bits into %edx */
1078         movl    %eax, %r12d     /* lower 32-bits into %eax */
1079 
1080         /*
1081          * Optimistically assume that there's no post-syscall
1082          * work to do.  (This is to avoid having to call syscall_mstate()
1083          * with interrupts disabled)
1084          */
1085         MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
1086 
1087         /*
1088          * We must protect ourselves from being descheduled here;
1089          * If we were, and we ended up on another cpu, or another
1090          * lwp got int ahead of us, it could change the segment
1091          * registers without us noticing before we return to userland.
1092          */
1093         cli
1094         CHECK_POSTSYS_NE(%r15, %r14, %ebx)
1095         jne     _full_syscall_postsys32
1096         SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
1097 
1098         /*
1099          * To get back to userland, load up the 32-bit registers and
1100          * sysexit back where we came from.
1101          */
1102 
1103         /*
1104          * Interrupts will be turned on by the 'sti' executed just before
1105          * sysexit.  The following ensures that restoring the user's rflags
1106          * doesn't enable interrupts too soon.
1107          */
1108         andq    $_BITNOT(PS_IE), REGOFF_RFL(%rsp)
1109 
1110         /*
1111          * (There's no point in loading up %edx because the sysexit
1112          * mechanism smashes it.)
1113          */
1114         movl    %r12d, %eax
1115         movl    REGOFF_RBX(%rsp), %ebx
1116         movl    REGOFF_RBP(%rsp), %ebp
1117         movl    REGOFF_RSI(%rsp), %esi
1118         movl    REGOFF_RDI(%rsp), %edi
1119 
1120         movl    REGOFF_RIP(%rsp), %edx  /* sysexit: %edx -> %eip */
1121         pushq   REGOFF_RFL(%rsp)
1122         popfq
1123         movl    REGOFF_RSP(%rsp), %ecx  /* sysexit: %ecx -> %esp */
1124         ALTENTRY(sys_sysenter_swapgs_sysexit)
1125         swapgs
1126         sti
1127         sysexit
1128         SET_SIZE(sys_sysenter_swapgs_sysexit)
1129         SET_SIZE(sys_sysenter)
1130         SET_SIZE(_sys_sysenter_post_swapgs)
1131         SET_SIZE(brand_sys_sysenter)
1132 
1133 #endif  /* __lint */
1134 
1135 #if defined(__lint)
1136 /*
1137  * System call via an int80.  This entry point is only used by the Linux
1138  * application environment.  Unlike the other entry points, there is no
1139  * default action to take if no callback is registered for this process.
1140  */
1141 void
1142 sys_int80()
1143 {}
1144 
1145 #else   /* __lint */
1146 
1147         ENTRY_NP(brand_sys_int80)
1148         SWAPGS                          /* kernel gsbase */
1149         XPV_TRAP_POP
1150         BRAND_CALLBACK(BRAND_CB_INT80, BRAND_URET_FROM_INTR_STACK())
1151         SWAPGS                          /* user gsbase */
1152         jmp     nopop_int80
1153 
1154         ENTRY_NP(sys_int80)
1155         /*
1156          * We hit an int80, but this process isn't of a brand with an int80
1157          * handler.  Bad process!  Make it look as if the INT failed.
1158          * Modify %rip to point before the INT, push the expected error
1159          * code and fake a GP fault. Note on 64-bit hypervisor we need
1160          * to undo the XPV_TRAP_POP and push rcx and r11 back on the stack
1161          * because gptrap will pop them again with its own XPV_TRAP_POP.
1162          */
1163         XPV_TRAP_POP
1164 nopop_int80:
1165         subq    $2, (%rsp)      /* int insn 2-bytes */
1166         pushq   $_CONST(_MUL(T_INT80, GATE_DESC_SIZE) + 2)
1167 #if defined(__xpv)
1168         push    %r11
1169         push    %rcx
1170 #endif
1171         jmp     gptrap                  / GP fault
1172         SET_SIZE(sys_int80)
1173         SET_SIZE(brand_sys_int80)
1174 #endif  /* __lint */
1175 
1176 
1177 /*
1178  * This is the destination of the "int $T_SYSCALLINT" interrupt gate, used by
1179  * the generic i386 libc to do system calls. We do a small amount of setup
1180  * before jumping into the existing sys_syscall32 path.
1181  */
1182 #if defined(__lint)
1183 
1184 /*ARGSUSED*/
1185 void
1186 sys_syscall_int()
1187 {}
1188 
1189 #else   /* __lint */
1190 
1191         ENTRY_NP(brand_sys_syscall_int)
1192         SWAPGS                          /* kernel gsbase */
1193         XPV_TRAP_POP
1194         BRAND_CALLBACK(BRAND_CB_INT91, BRAND_URET_FROM_INTR_STACK())
1195         jmp     nopop_syscall_int
1196 
1197         ALTENTRY(sys_syscall_int)
1198         SWAPGS                          /* kernel gsbase */
1199         XPV_TRAP_POP
1200 
1201 nopop_syscall_int:
1202         movq    %gs:CPU_THREAD, %r15
1203         movq    T_STACK(%r15), %rsp
1204         movl    %eax, %eax
1205         /*
1206          * Set t_post_sys on this thread to force ourselves out via the slow
1207          * path. It might be possible at some later date to optimize this out
1208          * and use a faster return mechanism.
1209          */
1210         movb    $1, T_POST_SYS(%r15)
1211         CLEAN_CS
1212         jmp     _syscall32_save
1213         /*
1214          * There should be no instructions between this label and SWAPGS/IRET
1215          * or we could end up breaking branded zone support. See the usage of
1216          * this label in lx_brand_int80_callback and sn1_brand_int91_callback
1217          * for examples.
1218          */
1219         ALTENTRY(sys_sysint_swapgs_iret)
1220         SWAPGS                          /* user gsbase */
1221         IRET
1222         /*NOTREACHED*/
1223         SET_SIZE(sys_sysint_swapgs_iret)
1224         SET_SIZE(sys_syscall_int)
1225         SET_SIZE(brand_sys_syscall_int)
1226 
1227 #endif  /* __lint */
1228         
1229 /*
1230  * Legacy 32-bit applications and old libc implementations do lcalls;
1231  * we should never get here because the LDT entry containing the syscall
1232  * segment descriptor has the "segment present" bit cleared, which means
1233  * we end up processing those system calls in trap() via a not-present trap.
1234  *
1235  * We do it this way because a call gate unhelpfully does -nothing- to the
1236  * interrupt flag bit, so an interrupt can run us just after the lcall
1237  * completes, but just before the swapgs takes effect.   Thus the INTR_PUSH and
1238  * INTR_POP paths would have to be slightly more complex to dance around
1239  * this problem, and end up depending explicitly on the first
1240  * instruction of this handler being either swapgs or cli.
1241  */
1242 
1243 #if defined(__lint)
1244 
1245 /*ARGSUSED*/
1246 void
1247 sys_lcall32()
1248 {}
1249 
1250 #else   /* __lint */
1251 
1252         ENTRY_NP(sys_lcall32)
1253         SWAPGS                          /* kernel gsbase */
1254         pushq   $0
1255         pushq   %rbp
1256         movq    %rsp, %rbp
1257         leaq    __lcall_panic_str(%rip), %rdi
1258         xorl    %eax, %eax
1259         call    panic
1260         SET_SIZE(sys_lcall32)
1261 
1262 __lcall_panic_str:      
1263         .string "sys_lcall32: shouldn't be here!"
1264 
1265 /*
1266  * Declare a uintptr_t which covers the entire pc range of syscall
1267  * handlers for the stack walkers that need this.
1268  */
1269         .align  CPTRSIZE
1270         .globl  _allsyscalls_size
1271         .type   _allsyscalls_size, @object
1272 _allsyscalls_size:
1273         .NWORD  . - _allsyscalls
1274         SET_SIZE(_allsyscalls_size)
1275 
1276 #endif  /* __lint */
1277 
1278 /*
1279  * These are the thread context handlers for lwps using sysenter/sysexit.
1280  */
1281 
1282 #if defined(__lint)
1283 
1284 /*ARGSUSED*/
1285 void
1286 sep_save(void *ksp)
1287 {}
1288 
1289 /*ARGSUSED*/
1290 void
1291 sep_restore(void *ksp)
1292 {}
1293 
1294 #else   /* __lint */
1295 
1296         /*
1297          * setting this value to zero as we switch away causes the
1298          * stack-pointer-on-sysenter to be NULL, ensuring that we
1299          * don't silently corrupt another (preempted) thread stack
1300          * when running an lwp that (somehow) didn't get sep_restore'd
1301          */
1302         ENTRY_NP(sep_save)
1303         xorl    %edx, %edx
1304         xorl    %eax, %eax
1305         movl    $MSR_INTC_SEP_ESP, %ecx
1306         wrmsr
1307         ret
1308         SET_SIZE(sep_save)
1309 
1310         /*
1311          * Update the kernel stack pointer as we resume onto this cpu.
1312          */
1313         ENTRY_NP(sep_restore)
1314         movq    %rdi, %rdx
1315         shrq    $32, %rdx
1316         movl    %edi, %eax
1317         movl    $MSR_INTC_SEP_ESP, %ecx
1318         wrmsr
1319         ret
1320         SET_SIZE(sep_restore)
1321 
1322 #endif  /* __lint */