Merge branch 'upstream' of master.kernel.org:/pub/scm/linux/kernel/git/jgarzik/netdev-2.6

[powerpc.git] / arch / x86_64 / kernel / entry.S

diff --git a/arch/x86_64/kernel/entry.S b/arch/x86_64/kernel/entry.S
index 053d6e8..be51dbe 100644 (file)
--- a/arch/x86_64/kernel/entry.S
+++ b/arch/x86_64/kernel/entry.S
@@ -44,10 +44,7 @@
 
        .code64
 
-#ifdef CONFIG_PREEMPT
-#define preempt_stop cli
-#else
-#define preempt_stop
+#ifndef CONFIG_PREEMPT
 #define retint_kernel retint_restore_args
 #endif 
        
@@ -79,7 +76,7 @@
 
        .macro FAKE_STACK_FRAME child_rip
        /* push in order ss, rsp, eflags, cs, rip */
-       xorq %rax, %rax
+       xorl %eax, %eax
        pushq %rax /* ss */
        CFI_ADJUST_CFA_OFFSET   8
        pushq %rax /* rsp */
@@ -197,7 +194,7 @@ ENTRY(system_call)
  */            
        .globl ret_from_sys_call
 ret_from_sys_call:
-       movl $_TIF_WORK_MASK,%edi
+       movl $_TIF_ALLWORK_MASK,%edi
        /* edi: flagmask */
 sysret_check:          
        GET_THREAD_INFO(%rcx)
@@ -289,6 +286,7 @@ int_careful:
        pushq %rdi
        call schedule
        popq %rdi
+       cli
        jmp int_with_check
 
        /* handle signals and tracing -- both require a full stack frame */
@@ -303,6 +301,7 @@ int_very_careful:
        call syscall_trace_leave
        popq %rdi
        andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
+       cli
        jmp int_restore_rest
        
 int_signal:
@@ -314,6 +313,7 @@ int_signal:
 1:     movl $_TIF_NEED_RESCHED,%edi    
 int_restore_rest:
        RESTORE_REST
+       cli
        jmp int_with_check
        CFI_ENDPROC
                
@@ -423,7 +423,7 @@ ENTRY(stub_rt_sigreturn)
        testl $3,CS(%rdi)
        je 1f
        swapgs  
-1:     addl $1,%gs:pda_irqcount        # RED-PEN should check preempt count
+1:     incl    %gs:pda_irqcount        # RED-PEN should check preempt count
        movq %gs:pda_irqstackptr,%rax
        cmoveq %rax,%rsp                                                        
        pushq %rdi                      # save old stack        
@@ -436,7 +436,7 @@ ENTRY(common_interrupt)
 ret_from_intr:         
        popq  %rdi
        cli     
-       subl $1,%gs:pda_irqcount
+       decl %gs:pda_irqcount
 #ifdef CONFIG_DEBUG_INFO
        movq RBP(%rdi),%rbp
 #endif
@@ -458,7 +458,6 @@ retint_check:
        andl %edi,%edx
        jnz  retint_careful
 retint_swapgs:         
-       cli
        swapgs 
 retint_restore_args:                           
        cli
@@ -495,13 +494,13 @@ retint_signal:
        sti
        SAVE_REST
        movq $-1,ORIG_RAX(%rsp)                         
-       xorq %rsi,%rsi          # oldset
+       xorl %esi,%esi          # oldset
        movq %rsp,%rdi          # &pt_regs
        call do_notify_resume
        RESTORE_REST
        cli
        movl $_TIF_NEED_RESCHED,%edi
-       GET_THREAD_INFO(%rcx)   
+       GET_THREAD_INFO(%rcx)
        jmp retint_check
 
 #ifdef CONFIG_PREEMPT
@@ -587,6 +586,7 @@ ENTRY(spurious_interrupt)
        movq ORIG_RAX(%rsp),%rsi
        movq $-1,ORIG_RAX(%rsp)
        call \sym
+       cli
        .endm
        
 /*
@@ -752,7 +752,7 @@ child_rip:
        movq %rsi, %rdi
        call *%rax
        # exit
-       xorq %rdi, %rdi
+       xorl %edi, %edi
        call do_exit
 
 /*
@@ -784,8 +784,9 @@ ENTRY(execve)
        ret
        CFI_ENDPROC
 
-ENTRY(page_fault)
+KPROBE_ENTRY(page_fault)
        errorentry do_page_fault
+       .previous .text
 
 ENTRY(coprocessor_error)
        zeroentry do_coprocessor_error
@@ -797,17 +798,14 @@ ENTRY(device_not_available)
        zeroentry math_state_restore
 
        /* runs on exception stack */
-ENTRY(debug)
+KPROBE_ENTRY(debug)
        CFI_STARTPROC
        pushq $0
        CFI_ADJUST_CFA_OFFSET 8         
        paranoidentry do_debug
-       /* switch back to process stack to restore the state ptrace touched */
-       movq %rax,%rsp  
-       testl $3,CS(%rsp)
-       jnz   paranoid_userspace        
        jmp paranoid_exit
        CFI_ENDPROC
+       .previous .text
 
        /* runs on exception stack */   
 ENTRY(nmi)
@@ -815,39 +813,52 @@ ENTRY(nmi)
        pushq $-1
        CFI_ADJUST_CFA_OFFSET 8         
        paranoidentry do_nmi
+       /*
+        * "Paranoid" exit path from exception stack.
+        * Paranoid because this is used by NMIs and cannot take
+        * any kernel state for granted.
+        * We don't do kernel preemption checks here, because only
+        * NMI should be common and it does not enable IRQs and
+        * cannot get reschedule ticks.
+        */
        /* ebx: no swapgs flag */
 paranoid_exit:
        testl %ebx,%ebx                         /* swapgs needed? */
        jnz paranoid_restore
+       testl $3,CS(%rsp)
+       jnz   paranoid_userspace
 paranoid_swapgs:       
-       cli
        swapgs
 paranoid_restore:      
        RESTORE_ALL 8
        iretq
 paranoid_userspace:    
-       cli
        GET_THREAD_INFO(%rcx)
-       movl threadinfo_flags(%rcx),%edx
-       testl $_TIF_NEED_RESCHED,%edx
-       jnz paranoid_resched
-       testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
-       jnz paranoid_signal
-       jmp paranoid_swapgs
-paranoid_resched:              
-       sti
-       call schedule
-       jmp paranoid_exit
-paranoid_signal:               
+       movl threadinfo_flags(%rcx),%ebx
+       andl $_TIF_WORK_MASK,%ebx
+       jz paranoid_swapgs
+       movq %rsp,%rdi                  /* &pt_regs */
+       call sync_regs
+       movq %rax,%rsp                  /* switch stack for scheduling */
+       testl $_TIF_NEED_RESCHED,%ebx
+       jnz paranoid_schedule
+       movl %ebx,%edx                  /* arg3: thread flags */
        sti
-       xorl %esi,%esi /* oldset */
-       movq %rsp,%rdi /* &pt_regs */
+       xorl %esi,%esi                  /* arg2: oldset */
+       movq %rsp,%rdi                  /* arg1: &pt_regs */
        call do_notify_resume
-       jmp paranoid_exit
+       cli
+       jmp paranoid_userspace
+paranoid_schedule:
+       sti
+       call schedule
+       cli
+       jmp paranoid_userspace
        CFI_ENDPROC
-       
-ENTRY(int3)
+
+KPROBE_ENTRY(int3)
        zeroentry do_int3       
+       .previous .text
 
 ENTRY(overflow)
        zeroentry do_overflow
@@ -868,9 +879,6 @@ ENTRY(reserved)
 ENTRY(double_fault)
        CFI_STARTPROC
        paranoidentry do_double_fault
-       movq %rax,%rsp
-       testl $3,CS(%rsp)
-       jnz paranoid_userspace          
        jmp paranoid_exit
        CFI_ENDPROC
 
@@ -884,14 +892,12 @@ ENTRY(segment_not_present)
 ENTRY(stack_segment)
        CFI_STARTPROC
        paranoidentry do_stack_segment
-       movq %rax,%rsp
-       testl $3,CS(%rsp)
-       jnz paranoid_userspace
        jmp paranoid_exit
        CFI_ENDPROC
 
-ENTRY(general_protection)
+KPROBE_ENTRY(general_protection)
        errorentry do_general_protection
+       .previous .text
 
 ENTRY(alignment_check)
        errorentry do_alignment_check
@@ -916,3 +922,15 @@ ENTRY(machine_check)
 ENTRY(call_debug)
        zeroentry do_call_debug
 
+ENTRY(call_softirq)
+       movq %gs:pda_irqstackptr,%rax
+       pushq %r15
+       movq %rsp,%r15
+       incl %gs:pda_irqcount
+       cmove %rax,%rsp
+       call __do_softirq
+       movq %r15,%rsp
+       decl %gs:pda_irqcount
+       popq %r15
+       ret
+