1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 #include <sys/asm_linkage.h> 26 #include <sys/asm_misc.h> 27 #include <sys/regset.h> 28 #include <sys/privregs.h> 29 #include <sys/psw.h> 30 #include <sys/machbrand.h> 31 32 #if defined(__lint) 33 34 #include <sys/types.h> 35 #include <sys/thread.h> 36 #include <sys/systm.h> 37 38 #else /* __lint */ 39 40 #include <sys/segments.h> 41 #include <sys/pcb.h> 42 #include <sys/trap.h> 43 #include <sys/ftrace.h> 44 #include <sys/traptrace.h> 45 #include <sys/clock.h> 46 #include <sys/model.h> 47 #include <sys/panic.h> 48 49 #if defined(__xpv) 50 #include <sys/hypervisor.h> 51 #endif 52 53 #include "assym.h" 54 55 #endif /* __lint */ 56 57 /* 58 * We implement five flavours of system call entry points 59 * 60 * - syscall/sysretq (amd64 generic) 61 * - syscall/sysretl (i386 plus SYSC bit) 62 * - sysenter/sysexit (i386 plus SEP bit) 63 * - int/iret (i386 generic) 64 * - lcall/iret (i386 generic) 65 * 66 * The current libc included in Solaris uses int/iret as the base unoptimized 67 * kernel entry method. Older libc implementations and legacy binaries may use 68 * the lcall call gate, so it must continue to be supported. 69 * 70 * System calls that use an lcall call gate are processed in trap() via a 71 * segment-not-present trap, i.e. lcalls are extremely slow(!). 72 * 73 * The basic pattern used in the 32-bit SYSC handler at this point in time is 74 * to have the bare minimum of assembler, and get to the C handlers as 75 * quickly as possible. 76 * 77 * The 64-bit handler is much closer to the sparcv9 handler; that's 78 * because of passing arguments in registers. The 32-bit world still 79 * passes arguments on the stack -- that makes that handler substantially 80 * more complex. 81 * 82 * The two handlers share a few code fragments which are broken 83 * out into preprocessor macros below. 84 * 85 * XX64 come back and speed all this up later. The 32-bit stuff looks 86 * especially easy to speed up the argument copying part .. 87 * 88 * 89 * Notes about segment register usage (c.f. the 32-bit kernel) 90 * 91 * In the 32-bit kernel, segment registers are dutifully saved and 92 * restored on all mode transitions because the kernel uses them directly. 93 * When the processor is running in 64-bit mode, segment registers are 94 * largely ignored. 95 * 96 * %cs and %ss 97 * controlled by the hardware mechanisms that make mode transitions 98 * 99 * The remaining segment registers have to either be pointing at a valid 100 * descriptor i.e. with the 'present' bit set, or they can NULL descriptors 101 * 102 * %ds and %es 103 * always ignored 104 * 105 * %fs and %gs 106 * fsbase and gsbase are used to control the place they really point at. 107 * The kernel only depends on %gs, and controls its own gsbase via swapgs 108 * 109 * Note that loading segment registers is still costly because the GDT 110 * lookup still happens (this is because the hardware can't know that we're 111 * not setting up these segment registers for a 32-bit program). Thus we 112 * avoid doing this in the syscall path, and defer them to lwp context switch 113 * handlers, so the register values remain virtualized to the lwp. 114 */ 115 116 #if defined(SYSCALLTRACE) 117 #define ORL_SYSCALLTRACE(r32) \ 118 orl syscalltrace(%rip), r32 119 #else 120 #define ORL_SYSCALLTRACE(r32) 121 #endif 122 123 /* 124 * In the 32-bit kernel, we do absolutely nothing before getting into the 125 * brand callback checks. In 64-bit land, we do swapgs and then come here. 126 * We assume that the %rsp- and %r15-stashing fields in the CPU structure 127 * are still unused. 128 * 129 * Check if a brand_mach_ops callback is defined for the specified callback_id 130 * type. If so invoke it with the kernel's %gs value loaded and the following 131 * data on the stack: 132 * 133 * stack: -------------------------------------- 134 * 32 | callback pointer | 135 * | 24 | user (or interrupt) stack pointer | 136 * | 16 | lwp pointer | 137 * v 8 | userland return address | 138 * 0 | callback wrapper return addr | 139 * -------------------------------------- 140 * 141 * Since we're pushing the userland return address onto the kernel stack 142 * we need to get that address without accessing the user's stack (since we 143 * can't trust that data). There are different ways to get the userland 144 * return address depending on how the syscall trap was made: 145 * 146 * a) For sys_syscall and sys_syscall32 the return address is in %rcx. 147 * b) For sys_sysenter the return address is in %rdx. 148 * c) For sys_int80 and sys_syscall_int (int91), upon entry into the macro, 149 * the stack pointer points at the state saved when we took the interrupt: 150 * ------------------------ 151 * | | user's %ss | 152 * | | user's %esp | 153 * | | EFLAGS register | 154 * v | user's %cs | 155 * | user's %eip | 156 * ------------------------ 157 * 158 * The 2nd parameter to the BRAND_CALLBACK macro is either the 159 * BRAND_URET_FROM_REG or BRAND_URET_FROM_INTR_STACK macro. These macros are 160 * used to generate the proper code to get the userland return address for 161 * each syscall entry point. 162 * 163 * The interface to the brand callbacks on the 64-bit kernel assumes %r15 164 * is available as a scratch register within the callback. If the callback 165 * returns within the kernel then this macro will restore %r15. If the 166 * callback is going to return directly to userland then it should restore 167 * %r15 before returning to userland. 168 */ 169 #define BRAND_URET_FROM_REG(rip_reg) \ 170 pushq rip_reg /* push the return address */ 171 172 /* 173 * The interrupt stack pointer we saved on entry to the BRAND_CALLBACK macro 174 * is currently pointing at the user return address (%eip). 175 */ 176 #define BRAND_URET_FROM_INTR_STACK() \ 177 movq %gs:CPU_RTMP_RSP, %r15 /* grab the intr. stack pointer */ ;\ 178 pushq (%r15) /* push the return address */ 179 180 #define BRAND_CALLBACK(callback_id, push_userland_ret) \ 181 movq %rsp, %gs:CPU_RTMP_RSP /* save the stack pointer */ ;\ 182 movq %r15, %gs:CPU_RTMP_R15 /* save %r15 */ ;\ 183 movq %gs:CPU_THREAD, %r15 /* load the thread pointer */ ;\ 184 movq T_STACK(%r15), %rsp /* switch to the kernel stack */ ;\ 185 subq $16, %rsp /* save space for 2 pointers */ ;\ 186 pushq %r14 /* save %r14 */ ;\ 187 movq %gs:CPU_RTMP_RSP, %r14 ;\ 188 movq %r14, 8(%rsp) /* stash the user stack pointer */ ;\ 189 popq %r14 /* restore %r14 */ ;\ 190 movq T_LWP(%r15), %r15 /* load the lwp pointer */ ;\ 191 pushq %r15 /* push the lwp pointer */ ;\ 192 movq LWP_PROCP(%r15), %r15 /* load the proc pointer */ ;\ 193 movq P_BRAND(%r15), %r15 /* load the brand pointer */ ;\ 194 movq B_MACHOPS(%r15), %r15 /* load the machops pointer */ ;\ 195 movq _CONST(_MUL(callback_id, CPTRSIZE))(%r15), %r15 ;\ 196 cmpq $0, %r15 ;\ 197 je 1f ;\ 198 movq %r15, 16(%rsp) /* save the callback pointer */ ;\ 199 push_userland_ret /* push the return address */ ;\ 200 call *24(%rsp) /* call callback */ ;\ 201 1: movq %gs:CPU_RTMP_R15, %r15 /* restore %r15 */ ;\ 202 movq %gs:CPU_RTMP_RSP, %rsp /* restore the stack pointer */ 203 204 #define MSTATE_TRANSITION(from, to) \ 205 movl $from, %edi; \ 206 movl $to, %esi; \ 207 call syscall_mstate 208 209 /* 210 * Check to see if a simple (direct) return is possible i.e. 211 * 212 * if (t->t_post_sys_ast | syscalltrace | 213 * lwp->lwp_pcb.pcb_rupdate == 1) 214 * do full version ; 215 * 216 * Preconditions: 217 * - t is curthread 218 * Postconditions: 219 * - condition code NE is set if post-sys is too complex 220 * - rtmp is zeroed if it isn't (we rely on this!) 221 * - ltmp is smashed 222 */ 223 #define CHECK_POSTSYS_NE(t, ltmp, rtmp) \ 224 movq T_LWP(t), ltmp; \ 225 movzbl PCB_RUPDATE(ltmp), rtmp; \ 226 ORL_SYSCALLTRACE(rtmp); \ 227 orl T_POST_SYS_AST(t), rtmp; \ 228 cmpl $0, rtmp 229 230 /* 231 * Fix up the lwp, thread, and eflags for a successful return 232 * 233 * Preconditions: 234 * - zwreg contains zero 235 */ 236 #define SIMPLE_SYSCALL_POSTSYS(t, lwp, zwreg) \ 237 movb $LWP_USER, LWP_STATE(lwp); \ 238 movw zwreg, T_SYSNUM(t); \ 239 andb $_CONST(0xffff - PS_C), REGOFF_RFL(%rsp) 240 241 /* 242 * ASSERT(lwptoregs(lwp) == rp); 243 * 244 * This may seem obvious, but very odd things happen if this 245 * assertion is false 246 * 247 * Preconditions: 248 * (%rsp is ready for normal call sequence) 249 * Postconditions (if assertion is true): 250 * %r11 is smashed 251 * 252 * ASSERT(rp->r_cs == descnum) 253 * 254 * The code selector is written into the regs structure when the 255 * lwp stack is created. We use this ASSERT to validate that 256 * the regs structure really matches how we came in. 257 * 258 * Preconditions: 259 * (%rsp is ready for normal call sequence) 260 * Postconditions (if assertion is true): 261 * -none- 262 * 263 * ASSERT(lwp->lwp_pcb.pcb_rupdate == 0); 264 * 265 * If this is false, it meant that we returned to userland without 266 * updating the segment registers as we were supposed to. 267 * 268 * Note that we must ensure no interrupts or other traps intervene 269 * between entering privileged mode and performing the assertion, 270 * otherwise we may perform a context switch on the thread, which 271 * will end up setting pcb_rupdate to 1 again. 272 */ 273 #if defined(DEBUG) 274 275 #if !defined(__lint) 276 277 __lwptoregs_msg: 278 .string "syscall_asm_amd64.s:%d lwptoregs(%p) [%p] != rp [%p]" 279 280 __codesel_msg: 281 .string "syscall_asm_amd64.s:%d rp->r_cs [%ld] != %ld" 282 283 __no_rupdate_msg: 284 .string "syscall_asm_amd64.s:%d lwp %p, pcb_rupdate != 0" 285 286 #endif /* !__lint */ 287 288 #define ASSERT_LWPTOREGS(lwp, rp) \ 289 movq LWP_REGS(lwp), %r11; \ 290 cmpq rp, %r11; \ 291 je 7f; \ 292 leaq __lwptoregs_msg(%rip), %rdi; \ 293 movl $__LINE__, %esi; \ 294 movq lwp, %rdx; \ 295 movq %r11, %rcx; \ 296 movq rp, %r8; \ 297 xorl %eax, %eax; \ 298 call panic; \ 299 7: 300 301 #define ASSERT_NO_RUPDATE_PENDING(lwp) \ 302 testb $0x1, PCB_RUPDATE(lwp); \ 303 je 8f; \ 304 movq lwp, %rdx; \ 305 leaq __no_rupdate_msg(%rip), %rdi; \ 306 movl $__LINE__, %esi; \ 307 xorl %eax, %eax; \ 308 call panic; \ 309 8: 310 311 #else 312 #define ASSERT_LWPTOREGS(lwp, rp) 313 #define ASSERT_NO_RUPDATE_PENDING(lwp) 314 #endif 315 316 /* 317 * Do the traptrace thing and restore any registers we used 318 * in situ. Assumes that %rsp is pointing at the base of 319 * the struct regs, obviously .. 320 */ 321 #ifdef TRAPTRACE 322 #define SYSCALL_TRAPTRACE(ttype) \ 323 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, ttype); \ 324 TRACE_REGS(%rdi, %rsp, %rbx, %rcx); \ 325 TRACE_STAMP(%rdi); /* rdtsc clobbers %eax, %edx */ \ 326 movq REGOFF_RAX(%rsp), %rax; \ 327 movq REGOFF_RBX(%rsp), %rbx; \ 328 movq REGOFF_RCX(%rsp), %rcx; \ 329 movq REGOFF_RDX(%rsp), %rdx; \ 330 movl %eax, TTR_SYSNUM(%rdi); \ 331 movq REGOFF_RDI(%rsp), %rdi 332 333 #define SYSCALL_TRAPTRACE32(ttype) \ 334 SYSCALL_TRAPTRACE(ttype); \ 335 /* paranoia: clean the top 32-bits of the registers */ \ 336 orl %eax, %eax; \ 337 orl %ebx, %ebx; \ 338 orl %ecx, %ecx; \ 339 orl %edx, %edx; \ 340 orl %edi, %edi 341 #else /* TRAPTRACE */ 342 #define SYSCALL_TRAPTRACE(ttype) 343 #define SYSCALL_TRAPTRACE32(ttype) 344 #endif /* TRAPTRACE */ 345 346 /* 347 * The 64-bit libc syscall wrapper does this: 348 * 349 * fn(<args>) 350 * { 351 * movq %rcx, %r10 -- because syscall smashes %rcx 352 * movl $CODE, %eax 353 * syscall 354 * <error processing> 355 * } 356 * 357 * Thus when we come into the kernel: 358 * 359 * %rdi, %rsi, %rdx, %r10, %r8, %r9 contain first six args 360 * %rax is the syscall number 361 * %r12-%r15 contain caller state 362 * 363 * The syscall instruction arranges that: 364 * 365 * %rcx contains the return %rip 366 * %r11d contains bottom 32-bits of %rflags 367 * %rflags is masked (as determined by the SFMASK msr) 368 * %cs is set to UCS_SEL (as determined by the STAR msr) 369 * %ss is set to UDS_SEL (as determined by the STAR msr) 370 * %rip is set to sys_syscall (as determined by the LSTAR msr) 371 * 372 * Or in other words, we have no registers available at all. 373 * Only swapgs can save us! 374 * 375 * Under the hypervisor, the swapgs has happened already. However, the 376 * state of the world is very different from that we're familiar with. 377 * 378 * In particular, we have a stack structure like that for interrupt 379 * gates, except that the %cs and %ss registers are modified for reasons 380 * that are not entirely clear. Critically, the %rcx/%r11 values do 381 * *not* reflect the usage of those registers under a 'real' syscall[1]; 382 * the stack, therefore, looks like this: 383 * 384 * 0x0(rsp) potentially junk %rcx 385 * 0x8(rsp) potentially junk %r11 386 * 0x10(rsp) user %rip 387 * 0x18(rsp) modified %cs 388 * 0x20(rsp) user %rflags 389 * 0x28(rsp) user %rsp 390 * 0x30(rsp) modified %ss 391 * 392 * 393 * and before continuing on, we must load the %rip into %rcx and the 394 * %rflags into %r11. 395 * 396 * [1] They used to, and we relied on it, but this was broken in 3.1.1. 397 * Sigh. 398 */ 399 #if defined(__xpv) 400 #define XPV_SYSCALL_PROD \ 401 movq 0x10(%rsp), %rcx; \ 402 movq 0x20(%rsp), %r11; \ 403 movq 0x28(%rsp), %rsp 404 #else 405 #define XPV_SYSCALL_PROD /* nothing */ 406 #endif 407 408 #if defined(__lint) 409 410 /*ARGSUSED*/ 411 void 412 sys_syscall() 413 {} 414 415 void 416 _allsyscalls() 417 {} 418 419 size_t _allsyscalls_size; 420 421 #else /* __lint */ 422 423 ENTRY_NP2(brand_sys_syscall,_allsyscalls) 424 SWAPGS /* kernel gsbase */ 425 XPV_SYSCALL_PROD 426 BRAND_CALLBACK(BRAND_CB_SYSCALL, BRAND_URET_FROM_REG(%rcx)) 427 jmp noprod_sys_syscall 428 429 ALTENTRY(sys_syscall) 430 SWAPGS /* kernel gsbase */ 431 XPV_SYSCALL_PROD 432 433 noprod_sys_syscall: 434 movq %r15, %gs:CPU_RTMP_R15 435 movq %rsp, %gs:CPU_RTMP_RSP 436 437 movq %gs:CPU_THREAD, %r15 438 movq T_STACK(%r15), %rsp /* switch from user to kernel stack */ 439 440 ASSERT_UPCALL_MASK_IS_SET 441 442 movl $UCS_SEL, REGOFF_CS(%rsp) 443 movq %rcx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */ 444 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */ 445 movl $UDS_SEL, REGOFF_SS(%rsp) 446 447 movl %eax, %eax /* wrapper: sysc# -> %eax */ 448 movq %rdi, REGOFF_RDI(%rsp) 449 movq %rsi, REGOFF_RSI(%rsp) 450 movq %rdx, REGOFF_RDX(%rsp) 451 movq %r10, REGOFF_RCX(%rsp) /* wrapper: %rcx -> %r10 */ 452 movq %r10, %rcx /* arg[3] for direct calls */ 453 454 movq %r8, REGOFF_R8(%rsp) 455 movq %r9, REGOFF_R9(%rsp) 456 movq %rax, REGOFF_RAX(%rsp) 457 movq %rbx, REGOFF_RBX(%rsp) 458 459 movq %rbp, REGOFF_RBP(%rsp) 460 movq %r10, REGOFF_R10(%rsp) 461 movq %gs:CPU_RTMP_RSP, %r11 462 movq %r11, REGOFF_RSP(%rsp) 463 movq %r12, REGOFF_R12(%rsp) 464 465 movq %r13, REGOFF_R13(%rsp) 466 movq %r14, REGOFF_R14(%rsp) 467 movq %gs:CPU_RTMP_R15, %r10 468 movq %r10, REGOFF_R15(%rsp) 469 movq $0, REGOFF_SAVFP(%rsp) 470 movq $0, REGOFF_SAVPC(%rsp) 471 472 /* 473 * Copy these registers here in case we end up stopped with 474 * someone (like, say, /proc) messing with our register state. 475 * We don't -restore- them unless we have to in update_sregs. 476 * 477 * Since userland -can't- change fsbase or gsbase directly, 478 * and capturing them involves two serializing instructions, 479 * we don't bother to capture them here. 480 */ 481 xorl %ebx, %ebx 482 movw %ds, %bx 483 movq %rbx, REGOFF_DS(%rsp) 484 movw %es, %bx 485 movq %rbx, REGOFF_ES(%rsp) 486 movw %fs, %bx 487 movq %rbx, REGOFF_FS(%rsp) 488 movw %gs, %bx 489 movq %rbx, REGOFF_GS(%rsp) 490 491 /* 492 * Machine state saved in the regs structure on the stack 493 * First six args in %rdi, %rsi, %rdx, %rcx, %r8, %r9 494 * %eax is the syscall number 495 * %rsp is the thread's stack, %r15 is curthread 496 * REG_RSP(%rsp) is the user's stack 497 */ 498 499 SYSCALL_TRAPTRACE($TT_SYSC64) 500 501 movq %rsp, %rbp 502 503 movq T_LWP(%r15), %r14 504 ASSERT_NO_RUPDATE_PENDING(%r14) 505 ENABLE_INTR_FLAGS 506 507 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 508 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */ 509 510 ASSERT_LWPTOREGS(%r14, %rsp) 511 512 movb $LWP_SYS, LWP_STATE(%r14) 513 incq LWP_RU_SYSC(%r14) 514 movb $NORMALRETURN, LWP_EOSYS(%r14) 515 516 incq %gs:CPU_STATS_SYS_SYSCALL 517 518 movw %ax, T_SYSNUM(%r15) 519 movzbl T_PRE_SYS(%r15), %ebx 520 ORL_SYSCALLTRACE(%ebx) 521 testl %ebx, %ebx 522 jne _syscall_pre 523 524 _syscall_invoke: 525 movq REGOFF_RDI(%rbp), %rdi 526 movq REGOFF_RSI(%rbp), %rsi 527 movq REGOFF_RDX(%rbp), %rdx 528 movq REGOFF_RCX(%rbp), %rcx 529 movq REGOFF_R8(%rbp), %r8 530 movq REGOFF_R9(%rbp), %r9 531 532 cmpl $NSYSCALL, %eax 533 jae _syscall_ill 534 shll $SYSENT_SIZE_SHIFT, %eax 535 leaq sysent(%rax), %rbx 536 537 call *SY_CALLC(%rbx) 538 539 movq %rax, %r12 540 movq %rdx, %r13 541 542 /* 543 * If the handler returns two ints, then we need to split the 544 * 64-bit return value into two 32-bit values. 545 */ 546 testw $SE_32RVAL2, SY_FLAGS(%rbx) 547 je 5f 548 movq %r12, %r13 549 shrq $32, %r13 /* upper 32-bits into %edx */ 550 movl %r12d, %r12d /* lower 32-bits into %eax */ 551 5: 552 /* 553 * Optimistically assume that there's no post-syscall 554 * work to do. (This is to avoid having to call syscall_mstate() 555 * with interrupts disabled) 556 */ 557 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 558 559 /* 560 * We must protect ourselves from being descheduled here; 561 * If we were, and we ended up on another cpu, or another 562 * lwp got in ahead of us, it could change the segment 563 * registers without us noticing before we return to userland. 564 */ 565 CLI(%r14) 566 CHECK_POSTSYS_NE(%r15, %r14, %ebx) 567 jne _syscall_post 568 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx) 569 570 movq %r12, REGOFF_RAX(%rsp) 571 movq %r13, REGOFF_RDX(%rsp) 572 573 /* 574 * To get back to userland, we need the return %rip in %rcx and 575 * the return %rfl in %r11d. The sysretq instruction also arranges 576 * to fix up %cs and %ss; everything else is our responsibility. 577 */ 578 movq REGOFF_RDI(%rsp), %rdi 579 movq REGOFF_RSI(%rsp), %rsi 580 movq REGOFF_RDX(%rsp), %rdx 581 /* %rcx used to restore %rip value */ 582 583 movq REGOFF_R8(%rsp), %r8 584 movq REGOFF_R9(%rsp), %r9 585 movq REGOFF_RAX(%rsp), %rax 586 movq REGOFF_RBX(%rsp), %rbx 587 588 movq REGOFF_RBP(%rsp), %rbp 589 movq REGOFF_R10(%rsp), %r10 590 /* %r11 used to restore %rfl value */ 591 movq REGOFF_R12(%rsp), %r12 592 593 movq REGOFF_R13(%rsp), %r13 594 movq REGOFF_R14(%rsp), %r14 595 movq REGOFF_R15(%rsp), %r15 596 597 movq REGOFF_RIP(%rsp), %rcx 598 movl REGOFF_RFL(%rsp), %r11d 599 600 #if defined(__xpv) 601 addq $REGOFF_RIP, %rsp 602 #else 603 movq REGOFF_RSP(%rsp), %rsp 604 #endif 605 606 /* 607 * There can be no instructions between the ALTENTRY below and 608 * SYSRET or we could end up breaking brand support. See label usage 609 * in sn1_brand_syscall_callback for an example. 610 */ 611 ASSERT_UPCALL_MASK_IS_SET 612 #if defined(__xpv) 613 SYSRETQ 614 ALTENTRY(nopop_sys_syscall_swapgs_sysretq) 615 616 /* 617 * We can only get here after executing a brand syscall 618 * interposition callback handler and simply need to 619 * "sysretq" back to userland. On the hypervisor this 620 * involves the iret hypercall which requires us to construct 621 * just enough of the stack needed for the hypercall. 622 * (rip, cs, rflags, rsp, ss). 623 */ 624 movq %rsp, %gs:CPU_RTMP_RSP /* save user's rsp */ 625 movq %gs:CPU_THREAD, %r11 626 movq T_STACK(%r11), %rsp 627 628 movq %rcx, REGOFF_RIP(%rsp) 629 movl $UCS_SEL, REGOFF_CS(%rsp) 630 movq %gs:CPU_RTMP_RSP, %r11 631 movq %r11, REGOFF_RSP(%rsp) 632 pushfq 633 popq %r11 /* hypercall enables ints */ 634 movq %r11, REGOFF_RFL(%rsp) 635 movl $UDS_SEL, REGOFF_SS(%rsp) 636 addq $REGOFF_RIP, %rsp 637 /* 638 * XXPV: see comment in SYSRETQ definition for future optimization 639 * we could take. 640 */ 641 ASSERT_UPCALL_MASK_IS_SET 642 SYSRETQ 643 #else 644 ALTENTRY(nopop_sys_syscall_swapgs_sysretq) 645 SWAPGS /* user gsbase */ 646 SYSRETQ 647 #endif 648 /*NOTREACHED*/ 649 SET_SIZE(nopop_sys_syscall_swapgs_sysretq) 650 651 _syscall_pre: 652 call pre_syscall 653 movl %eax, %r12d 654 testl %eax, %eax 655 jne _syscall_post_call 656 /* 657 * Didn't abort, so reload the syscall args and invoke the handler. 658 */ 659 movzwl T_SYSNUM(%r15), %eax 660 jmp _syscall_invoke 661 662 _syscall_ill: 663 call nosys 664 movq %rax, %r12 665 movq %rdx, %r13 666 jmp _syscall_post_call 667 668 _syscall_post: 669 STI 670 /* 671 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM 672 * so that we can account for the extra work it takes us to finish. 673 */ 674 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 675 _syscall_post_call: 676 movq %r12, %rdi 677 movq %r13, %rsi 678 call post_syscall 679 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 680 jmp _sys_rtt 681 SET_SIZE(sys_syscall) 682 SET_SIZE(brand_sys_syscall) 683 684 #endif /* __lint */ 685 686 #if defined(__lint) 687 688 /*ARGSUSED*/ 689 void 690 sys_syscall32() 691 {} 692 693 #else /* __lint */ 694 695 ENTRY_NP(brand_sys_syscall32) 696 SWAPGS /* kernel gsbase */ 697 XPV_TRAP_POP 698 BRAND_CALLBACK(BRAND_CB_SYSCALL32, BRAND_URET_FROM_REG(%rcx)) 699 jmp nopop_sys_syscall32 700 701 ALTENTRY(sys_syscall32) 702 SWAPGS /* kernel gsbase */ 703 XPV_TRAP_POP 704 705 nopop_sys_syscall32: 706 movl %esp, %r10d 707 movq %gs:CPU_THREAD, %r15 708 movq T_STACK(%r15), %rsp 709 movl %eax, %eax 710 711 movl $U32CS_SEL, REGOFF_CS(%rsp) 712 movl %ecx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */ 713 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */ 714 movq %r10, REGOFF_RSP(%rsp) 715 movl $UDS_SEL, REGOFF_SS(%rsp) 716 717 _syscall32_save: 718 movl %edi, REGOFF_RDI(%rsp) 719 movl %esi, REGOFF_RSI(%rsp) 720 movl %ebp, REGOFF_RBP(%rsp) 721 movl %ebx, REGOFF_RBX(%rsp) 722 movl %edx, REGOFF_RDX(%rsp) 723 movl %ecx, REGOFF_RCX(%rsp) 724 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */ 725 movq $0, REGOFF_SAVFP(%rsp) 726 movq $0, REGOFF_SAVPC(%rsp) 727 728 /* 729 * Copy these registers here in case we end up stopped with 730 * someone (like, say, /proc) messing with our register state. 731 * We don't -restore- them unless we have to in update_sregs. 732 * 733 * Since userland -can't- change fsbase or gsbase directly, 734 * we don't bother to capture them here. 735 */ 736 xorl %ebx, %ebx 737 movw %ds, %bx 738 movq %rbx, REGOFF_DS(%rsp) 739 movw %es, %bx 740 movq %rbx, REGOFF_ES(%rsp) 741 movw %fs, %bx 742 movq %rbx, REGOFF_FS(%rsp) 743 movw %gs, %bx 744 movq %rbx, REGOFF_GS(%rsp) 745 746 /* 747 * Application state saved in the regs structure on the stack 748 * %eax is the syscall number 749 * %rsp is the thread's stack, %r15 is curthread 750 * REG_RSP(%rsp) is the user's stack 751 */ 752 753 SYSCALL_TRAPTRACE32($TT_SYSC) 754 755 movq %rsp, %rbp 756 757 movq T_LWP(%r15), %r14 758 ASSERT_NO_RUPDATE_PENDING(%r14) 759 760 ENABLE_INTR_FLAGS 761 762 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 763 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */ 764 765 ASSERT_LWPTOREGS(%r14, %rsp) 766 767 incq %gs:CPU_STATS_SYS_SYSCALL 768 769 /* 770 * Make some space for MAXSYSARGS (currently 8) 32-bit args placed 771 * into 64-bit (long) arg slots, maintaining 16 byte alignment. Or 772 * more succinctly: 773 * 774 * SA(MAXSYSARGS * sizeof (long)) == 64 775 */ 776 #define SYS_DROP 64 /* drop for args */ 777 subq $SYS_DROP, %rsp 778 movb $LWP_SYS, LWP_STATE(%r14) 779 movq %r15, %rdi 780 movq %rsp, %rsi 781 call syscall_entry 782 783 /* 784 * Fetch the arguments copied onto the kernel stack and put 785 * them in the right registers to invoke a C-style syscall handler. 786 * %rax contains the handler address. 787 * 788 * Ideas for making all this go faster of course include simply 789 * forcibly fetching 6 arguments from the user stack under lofault 790 * protection, reverting to copyin_args only when watchpoints 791 * are in effect. 792 * 793 * (If we do this, make sure that exec and libthread leave 794 * enough space at the top of the stack to ensure that we'll 795 * never do a fetch from an invalid page.) 796 * 797 * Lots of ideas here, but they won't really help with bringup B-) 798 * Correctness can't wait, performance can wait a little longer .. 799 */ 800 801 movq %rax, %rbx 802 movl 0(%rsp), %edi 803 movl 8(%rsp), %esi 804 movl 0x10(%rsp), %edx 805 movl 0x18(%rsp), %ecx 806 movl 0x20(%rsp), %r8d 807 movl 0x28(%rsp), %r9d 808 809 call *SY_CALLC(%rbx) 810 811 movq %rbp, %rsp /* pop the args */ 812 813 /* 814 * amd64 syscall handlers -always- return a 64-bit value in %rax. 815 * On the 32-bit kernel, they always return that value in %eax:%edx 816 * as required by the 32-bit ABI. 817 * 818 * Simulate the same behaviour by unconditionally splitting the 819 * return value in the same way. 820 */ 821 movq %rax, %r13 822 shrq $32, %r13 /* upper 32-bits into %edx */ 823 movl %eax, %r12d /* lower 32-bits into %eax */ 824 825 /* 826 * Optimistically assume that there's no post-syscall 827 * work to do. (This is to avoid having to call syscall_mstate() 828 * with interrupts disabled) 829 */ 830 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 831 832 /* 833 * We must protect ourselves from being descheduled here; 834 * If we were, and we ended up on another cpu, or another 835 * lwp got in ahead of us, it could change the segment 836 * registers without us noticing before we return to userland. 837 */ 838 CLI(%r14) 839 CHECK_POSTSYS_NE(%r15, %r14, %ebx) 840 jne _full_syscall_postsys32 841 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx) 842 843 /* 844 * To get back to userland, we need to put the return %rip in %rcx and 845 * the return %rfl in %r11d. The sysret instruction also arranges 846 * to fix up %cs and %ss; everything else is our responsibility. 847 */ 848 849 movl %r12d, %eax /* %eax: rval1 */ 850 movl REGOFF_RBX(%rsp), %ebx 851 /* %ecx used for return pointer */ 852 movl %r13d, %edx /* %edx: rval2 */ 853 movl REGOFF_RBP(%rsp), %ebp 854 movl REGOFF_RSI(%rsp), %esi 855 movl REGOFF_RDI(%rsp), %edi 856 857 movl REGOFF_RFL(%rsp), %r11d /* %r11 -> eflags */ 858 movl REGOFF_RIP(%rsp), %ecx /* %ecx -> %eip */ 859 movl REGOFF_RSP(%rsp), %esp 860 861 ASSERT_UPCALL_MASK_IS_SET 862 ALTENTRY(nopop_sys_syscall32_swapgs_sysretl) 863 SWAPGS /* user gsbase */ 864 SYSRETL 865 SET_SIZE(nopop_sys_syscall32_swapgs_sysretl) 866 /*NOTREACHED*/ 867 868 _full_syscall_postsys32: 869 STI 870 /* 871 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM 872 * so that we can account for the extra work it takes us to finish. 873 */ 874 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 875 movq %r15, %rdi 876 movq %r12, %rsi /* rval1 - %eax */ 877 movq %r13, %rdx /* rval2 - %edx */ 878 call syscall_exit 879 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 880 jmp _sys_rtt 881 SET_SIZE(sys_syscall32) 882 SET_SIZE(brand_sys_syscall32) 883 884 #endif /* __lint */ 885 886 /* 887 * System call handler via the sysenter instruction 888 * Used only for 32-bit system calls on the 64-bit kernel. 889 * 890 * The caller in userland has arranged that: 891 * 892 * - %eax contains the syscall number 893 * - %ecx contains the user %esp 894 * - %edx contains the return %eip 895 * - the user stack contains the args to the syscall 896 * 897 * Hardware and (privileged) initialization code have arranged that by 898 * the time the sysenter instructions completes: 899 * 900 * - %rip is pointing to sys_sysenter (below). 901 * - %cs and %ss are set to kernel text and stack (data) selectors. 902 * - %rsp is pointing at the lwp's stack 903 * - interrupts have been disabled. 904 * 905 * Note that we are unable to return both "rvals" to userland with 906 * this call, as %edx is used by the sysexit instruction. 907 * 908 * One final complication in this routine is its interaction with 909 * single-stepping in a debugger. For most of the system call mechanisms, 910 * the CPU automatically clears the single-step flag before we enter the 911 * kernel. The sysenter mechanism does not clear the flag, so a user 912 * single-stepping through a libc routine may suddenly find him/herself 913 * single-stepping through the kernel. To detect this, kmdb compares the 914 * trap %pc to the [brand_]sys_enter addresses on each single-step trap. 915 * If it finds that we have single-stepped to a sysenter entry point, it 916 * explicitly clears the flag and executes the sys_sysenter routine. 917 * 918 * One final complication in this final complication is the fact that we 919 * have two different entry points for sysenter: brand_sys_sysenter and 920 * sys_sysenter. If we enter at brand_sys_sysenter and start single-stepping 921 * through the kernel with kmdb, we will eventually hit the instruction at 922 * sys_sysenter. kmdb cannot distinguish between that valid single-step 923 * and the undesirable one mentioned above. To avoid this situation, we 924 * simply add a jump over the instruction at sys_sysenter to make it 925 * impossible to single-step to it. 926 */ 927 #if defined(__lint) 928 929 void 930 sys_sysenter() 931 {} 932 933 #else /* __lint */ 934 935 ENTRY_NP(brand_sys_sysenter) 936 SWAPGS /* kernel gsbase */ 937 ALTENTRY(_brand_sys_sysenter_post_swapgs) 938 BRAND_CALLBACK(BRAND_CB_SYSENTER, BRAND_URET_FROM_REG(%rdx)) 939 /* 940 * Jump over sys_sysenter to allow single-stepping as described 941 * above. 942 */ 943 jmp _sys_sysenter_post_swapgs 944 945 ALTENTRY(sys_sysenter) 946 SWAPGS /* kernel gsbase */ 947 948 ALTENTRY(_sys_sysenter_post_swapgs) 949 movq %gs:CPU_THREAD, %r15 950 951 movl $U32CS_SEL, REGOFF_CS(%rsp) 952 movl %ecx, REGOFF_RSP(%rsp) /* wrapper: %esp -> %ecx */ 953 movl %edx, REGOFF_RIP(%rsp) /* wrapper: %eip -> %edx */ 954 pushfq 955 popq %r10 956 movl $UDS_SEL, REGOFF_SS(%rsp) 957 958 /* 959 * Set the interrupt flag before storing the flags to the 960 * flags image on the stack so we can return to user with 961 * interrupts enabled if we return via sys_rtt_syscall32 962 */ 963 orq $PS_IE, %r10 964 movq %r10, REGOFF_RFL(%rsp) 965 966 movl %edi, REGOFF_RDI(%rsp) 967 movl %esi, REGOFF_RSI(%rsp) 968 movl %ebp, REGOFF_RBP(%rsp) 969 movl %ebx, REGOFF_RBX(%rsp) 970 movl %edx, REGOFF_RDX(%rsp) 971 movl %ecx, REGOFF_RCX(%rsp) 972 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */ 973 movq $0, REGOFF_SAVFP(%rsp) 974 movq $0, REGOFF_SAVPC(%rsp) 975 976 /* 977 * Copy these registers here in case we end up stopped with 978 * someone (like, say, /proc) messing with our register state. 979 * We don't -restore- them unless we have to in update_sregs. 980 * 981 * Since userland -can't- change fsbase or gsbase directly, 982 * we don't bother to capture them here. 983 */ 984 xorl %ebx, %ebx 985 movw %ds, %bx 986 movq %rbx, REGOFF_DS(%rsp) 987 movw %es, %bx 988 movq %rbx, REGOFF_ES(%rsp) 989 movw %fs, %bx 990 movq %rbx, REGOFF_FS(%rsp) 991 movw %gs, %bx 992 movq %rbx, REGOFF_GS(%rsp) 993 994 /* 995 * Application state saved in the regs structure on the stack 996 * %eax is the syscall number 997 * %rsp is the thread's stack, %r15 is curthread 998 * REG_RSP(%rsp) is the user's stack 999 */ 1000 1001 SYSCALL_TRAPTRACE($TT_SYSENTER) 1002 1003 movq %rsp, %rbp 1004 1005 movq T_LWP(%r15), %r14 1006 ASSERT_NO_RUPDATE_PENDING(%r14) 1007 1008 ENABLE_INTR_FLAGS 1009 1010 /* 1011 * Catch 64-bit process trying to issue sysenter instruction 1012 * on Nocona based systems. 1013 */ 1014 movq LWP_PROCP(%r14), %rax 1015 cmpq $DATAMODEL_ILP32, P_MODEL(%rax) 1016 je 7f 1017 1018 /* 1019 * For a non-32-bit process, simulate a #ud, since that's what 1020 * native hardware does. The traptrace entry (above) will 1021 * let you know what really happened. 1022 */ 1023 movq $T_ILLINST, REGOFF_TRAPNO(%rsp) 1024 movq REGOFF_CS(%rsp), %rdi 1025 movq %rdi, REGOFF_ERR(%rsp) 1026 movq %rsp, %rdi 1027 movq REGOFF_RIP(%rsp), %rsi 1028 movl %gs:CPU_ID, %edx 1029 call trap 1030 jmp _sys_rtt 1031 7: 1032 1033 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 1034 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate calls) */ 1035 1036 ASSERT_LWPTOREGS(%r14, %rsp) 1037 1038 incq %gs:CPU_STATS_SYS_SYSCALL 1039 1040 /* 1041 * Make some space for MAXSYSARGS (currently 8) 32-bit args 1042 * placed into 64-bit (long) arg slots, plus one 64-bit 1043 * (long) arg count, maintaining 16 byte alignment. 1044 */ 1045 subq $SYS_DROP, %rsp 1046 movb $LWP_SYS, LWP_STATE(%r14) 1047 movq %r15, %rdi 1048 movq %rsp, %rsi 1049 call syscall_entry 1050 1051 /* 1052 * Fetch the arguments copied onto the kernel stack and put 1053 * them in the right registers to invoke a C-style syscall handler. 1054 * %rax contains the handler address. 1055 */ 1056 movq %rax, %rbx 1057 movl 0(%rsp), %edi 1058 movl 8(%rsp), %esi 1059 movl 0x10(%rsp), %edx 1060 movl 0x18(%rsp), %ecx 1061 movl 0x20(%rsp), %r8d 1062 movl 0x28(%rsp), %r9d 1063 1064 call *SY_CALLC(%rbx) 1065 1066 movq %rbp, %rsp /* pop the args */ 1067 1068 /* 1069 * amd64 syscall handlers -always- return a 64-bit value in %rax. 1070 * On the 32-bit kernel, the always return that value in %eax:%edx 1071 * as required by the 32-bit ABI. 1072 * 1073 * Simulate the same behaviour by unconditionally splitting the 1074 * return value in the same way. 1075 */ 1076 movq %rax, %r13 1077 shrq $32, %r13 /* upper 32-bits into %edx */ 1078 movl %eax, %r12d /* lower 32-bits into %eax */ 1079 1080 /* 1081 * Optimistically assume that there's no post-syscall 1082 * work to do. (This is to avoid having to call syscall_mstate() 1083 * with interrupts disabled) 1084 */ 1085 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 1086 1087 /* 1088 * We must protect ourselves from being descheduled here; 1089 * If we were, and we ended up on another cpu, or another 1090 * lwp got int ahead of us, it could change the segment 1091 * registers without us noticing before we return to userland. 1092 */ 1093 cli 1094 CHECK_POSTSYS_NE(%r15, %r14, %ebx) 1095 jne _full_syscall_postsys32 1096 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx) 1097 1098 /* 1099 * To get back to userland, load up the 32-bit registers and 1100 * sysexit back where we came from. 1101 */ 1102 1103 /* 1104 * Interrupts will be turned on by the 'sti' executed just before 1105 * sysexit. The following ensures that restoring the user's rflags 1106 * doesn't enable interrupts too soon. 1107 */ 1108 andq $_BITNOT(PS_IE), REGOFF_RFL(%rsp) 1109 1110 /* 1111 * (There's no point in loading up %edx because the sysexit 1112 * mechanism smashes it.) 1113 */ 1114 movl %r12d, %eax 1115 movl REGOFF_RBX(%rsp), %ebx 1116 movl REGOFF_RBP(%rsp), %ebp 1117 movl REGOFF_RSI(%rsp), %esi 1118 movl REGOFF_RDI(%rsp), %edi 1119 1120 movl REGOFF_RIP(%rsp), %edx /* sysexit: %edx -> %eip */ 1121 pushq REGOFF_RFL(%rsp) 1122 popfq 1123 movl REGOFF_RSP(%rsp), %ecx /* sysexit: %ecx -> %esp */ 1124 ALTENTRY(sys_sysenter_swapgs_sysexit) 1125 swapgs 1126 sti 1127 sysexit 1128 SET_SIZE(sys_sysenter_swapgs_sysexit) 1129 SET_SIZE(sys_sysenter) 1130 SET_SIZE(_sys_sysenter_post_swapgs) 1131 SET_SIZE(brand_sys_sysenter) 1132 1133 #endif /* __lint */ 1134 1135 #if defined(__lint) 1136 /* 1137 * System call via an int80. This entry point is only used by the Linux 1138 * application environment. Unlike the other entry points, there is no 1139 * default action to take if no callback is registered for this process. 1140 */ 1141 void 1142 sys_int80() 1143 {} 1144 1145 #else /* __lint */ 1146 1147 ENTRY_NP(brand_sys_int80) 1148 SWAPGS /* kernel gsbase */ 1149 XPV_TRAP_POP 1150 BRAND_CALLBACK(BRAND_CB_INT80, BRAND_URET_FROM_INTR_STACK()) 1151 SWAPGS /* user gsbase */ 1152 jmp nopop_int80 1153 1154 ENTRY_NP(sys_int80) 1155 /* 1156 * We hit an int80, but this process isn't of a brand with an int80 1157 * handler. Bad process! Make it look as if the INT failed. 1158 * Modify %rip to point before the INT, push the expected error 1159 * code and fake a GP fault. Note on 64-bit hypervisor we need 1160 * to undo the XPV_TRAP_POP and push rcx and r11 back on the stack 1161 * because gptrap will pop them again with its own XPV_TRAP_POP. 1162 */ 1163 XPV_TRAP_POP 1164 nopop_int80: 1165 subq $2, (%rsp) /* int insn 2-bytes */ 1166 pushq $_CONST(_MUL(T_INT80, GATE_DESC_SIZE) + 2) 1167 #if defined(__xpv) 1168 push %r11 1169 push %rcx 1170 #endif 1171 jmp gptrap / GP fault 1172 SET_SIZE(sys_int80) 1173 SET_SIZE(brand_sys_int80) 1174 #endif /* __lint */ 1175 1176 1177 /* 1178 * This is the destination of the "int $T_SYSCALLINT" interrupt gate, used by 1179 * the generic i386 libc to do system calls. We do a small amount of setup 1180 * before jumping into the existing sys_syscall32 path. 1181 */ 1182 #if defined(__lint) 1183 1184 /*ARGSUSED*/ 1185 void 1186 sys_syscall_int() 1187 {} 1188 1189 #else /* __lint */ 1190 1191 ENTRY_NP(brand_sys_syscall_int) 1192 SWAPGS /* kernel gsbase */ 1193 XPV_TRAP_POP 1194 BRAND_CALLBACK(BRAND_CB_INT91, BRAND_URET_FROM_INTR_STACK()) 1195 jmp nopop_syscall_int 1196 1197 ALTENTRY(sys_syscall_int) 1198 SWAPGS /* kernel gsbase */ 1199 XPV_TRAP_POP 1200 1201 nopop_syscall_int: 1202 movq %gs:CPU_THREAD, %r15 1203 movq T_STACK(%r15), %rsp 1204 movl %eax, %eax 1205 /* 1206 * Set t_post_sys on this thread to force ourselves out via the slow 1207 * path. It might be possible at some later date to optimize this out 1208 * and use a faster return mechanism. 1209 */ 1210 movb $1, T_POST_SYS(%r15) 1211 CLEAN_CS 1212 jmp _syscall32_save 1213 /* 1214 * There should be no instructions between this label and SWAPGS/IRET 1215 * or we could end up breaking branded zone support. See the usage of 1216 * this label in lx_brand_int80_callback and sn1_brand_int91_callback 1217 * for examples. 1218 */ 1219 ALTENTRY(sys_sysint_swapgs_iret) 1220 SWAPGS /* user gsbase */ 1221 IRET 1222 /*NOTREACHED*/ 1223 SET_SIZE(sys_sysint_swapgs_iret) 1224 SET_SIZE(sys_syscall_int) 1225 SET_SIZE(brand_sys_syscall_int) 1226 1227 #endif /* __lint */ 1228 1229 /* 1230 * Legacy 32-bit applications and old libc implementations do lcalls; 1231 * we should never get here because the LDT entry containing the syscall 1232 * segment descriptor has the "segment present" bit cleared, which means 1233 * we end up processing those system calls in trap() via a not-present trap. 1234 * 1235 * We do it this way because a call gate unhelpfully does -nothing- to the 1236 * interrupt flag bit, so an interrupt can run us just after the lcall 1237 * completes, but just before the swapgs takes effect. Thus the INTR_PUSH and 1238 * INTR_POP paths would have to be slightly more complex to dance around 1239 * this problem, and end up depending explicitly on the first 1240 * instruction of this handler being either swapgs or cli. 1241 */ 1242 1243 #if defined(__lint) 1244 1245 /*ARGSUSED*/ 1246 void 1247 sys_lcall32() 1248 {} 1249 1250 #else /* __lint */ 1251 1252 ENTRY_NP(sys_lcall32) 1253 SWAPGS /* kernel gsbase */ 1254 pushq $0 1255 pushq %rbp 1256 movq %rsp, %rbp 1257 leaq __lcall_panic_str(%rip), %rdi 1258 xorl %eax, %eax 1259 call panic 1260 SET_SIZE(sys_lcall32) 1261 1262 __lcall_panic_str: 1263 .string "sys_lcall32: shouldn't be here!" 1264 1265 /* 1266 * Declare a uintptr_t which covers the entire pc range of syscall 1267 * handlers for the stack walkers that need this. 1268 */ 1269 .align CPTRSIZE 1270 .globl _allsyscalls_size 1271 .type _allsyscalls_size, @object 1272 _allsyscalls_size: 1273 .NWORD . - _allsyscalls 1274 SET_SIZE(_allsyscalls_size) 1275 1276 #endif /* __lint */ 1277 1278 /* 1279 * These are the thread context handlers for lwps using sysenter/sysexit. 1280 */ 1281 1282 #if defined(__lint) 1283 1284 /*ARGSUSED*/ 1285 void 1286 sep_save(void *ksp) 1287 {} 1288 1289 /*ARGSUSED*/ 1290 void 1291 sep_restore(void *ksp) 1292 {} 1293 1294 #else /* __lint */ 1295 1296 /* 1297 * setting this value to zero as we switch away causes the 1298 * stack-pointer-on-sysenter to be NULL, ensuring that we 1299 * don't silently corrupt another (preempted) thread stack 1300 * when running an lwp that (somehow) didn't get sep_restore'd 1301 */ 1302 ENTRY_NP(sep_save) 1303 xorl %edx, %edx 1304 xorl %eax, %eax 1305 movl $MSR_INTC_SEP_ESP, %ecx 1306 wrmsr 1307 ret 1308 SET_SIZE(sep_save) 1309 1310 /* 1311 * Update the kernel stack pointer as we resume onto this cpu. 1312 */ 1313 ENTRY_NP(sep_restore) 1314 movq %rdi, %rdx 1315 shrq $32, %rdx 1316 movl %edi, %eax 1317 movl $MSR_INTC_SEP_ESP, %ecx 1318 wrmsr 1319 ret 1320 SET_SIZE(sep_restore) 1321 1322 #endif /* __lint */