/linux-master/arch/x86/include/asm/ |
H A D | current.h | 15 struct pcpu_hot { struct 36 static_assert(sizeof(struct pcpu_hot) == 64); 38 DECLARE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot); 40 /* const-qualified alias to pcpu_hot, aliased by linker. */ 41 DECLARE_PER_CPU_ALIGNED(const struct pcpu_hot __percpu_seg_override, 49 return this_cpu_read_stable(pcpu_hot.current_task);
|
H A D | preempt.h | 26 return raw_cpu_read_4(pcpu_hot.preempt_count) & ~PREEMPT_NEED_RESCHED; 33 old = raw_cpu_read_4(pcpu_hot.preempt_count); 37 } while (!raw_cpu_try_cmpxchg_4(pcpu_hot.preempt_count, &old, new)); 46 per_cpu(pcpu_hot.preempt_count, (cpu)) = PREEMPT_DISABLED; \ 60 raw_cpu_and_4(pcpu_hot.preempt_count, ~PREEMPT_NEED_RESCHED); 65 raw_cpu_or_4(pcpu_hot.preempt_count, PREEMPT_NEED_RESCHED); 70 return !(raw_cpu_read_4(pcpu_hot.preempt_count) & PREEMPT_NEED_RESCHED); 79 raw_cpu_add_4(pcpu_hot.preempt_count, val); 84 raw_cpu_add_4(pcpu_hot.preempt_count, -val); 94 return GEN_UNARY_RMWcc("decl", __my_cpu_var(pcpu_hot [all...] |
H A D | irq_stack.h | 119 call_on_stack(__this_cpu_read(pcpu_hot.hardirq_stack_ptr), \ 138 if (user_mode(regs) || __this_cpu_read(pcpu_hot.hardirq_stack_inuse)) { \ 149 __this_cpu_write(pcpu_hot.hardirq_stack_inuse, true); \ 151 __this_cpu_write(pcpu_hot.hardirq_stack_inuse, false); \ 215 __this_cpu_write(pcpu_hot.hardirq_stack_inuse, true); \ 217 __this_cpu_write(pcpu_hot.hardirq_stack_inuse, false); \
|
H A D | smp.h | 135 #define raw_smp_processor_id() this_cpu_read(pcpu_hot.cpu_number) 136 #define __smp_processor_id() __this_cpu_read(pcpu_hot.cpu_number)
|
H A D | hardirq.h | 63 #define local_softirq_pending_ref pcpu_hot.softirq_pending
|
H A D | nospec-branch.h | 81 movq $-1, PER_CPU_VAR(pcpu_hot + X86_call_depth); 86 movq %rax, PER_CPU_VAR(pcpu_hot + X86_call_depth); 91 movq %rax, PER_CPU_VAR(pcpu_hot + X86_call_depth); \ 95 sarq $5, PER_CPU_VAR(pcpu_hot + X86_call_depth); \
|
H A D | processor.h | 532 return this_cpu_read_stable(pcpu_hot.top_of_stack);
|
/linux-master/arch/x86/kernel/ |
H A D | irq_64.c | 53 per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8; 66 per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = va + IRQ_STACK_SIZE - 8; 73 if (per_cpu(pcpu_hot.hardirq_stack_ptr, cpu))
|
H A D | irq_32.c | 77 irqstk = __this_cpu_read(pcpu_hot.hardirq_stack_ptr); 115 if (per_cpu(pcpu_hot.hardirq_stack_ptr, cpu)) 127 per_cpu(pcpu_hot.hardirq_stack_ptr, cpu) = page_address(ph); 128 per_cpu(pcpu_hot.softirq_stack_ptr, cpu) = page_address(ps); 138 irqstk = __this_cpu_read(pcpu_hot.softirq_stack_ptr);
|
H A D | asm-offsets.c | 110 OFFSET(X86_top_of_stack, pcpu_hot, top_of_stack); 111 OFFSET(X86_current_task, pcpu_hot, current_task); 113 OFFSET(X86_call_depth, pcpu_hot, call_depth);
|
H A D | dumpstack_32.c | 40 unsigned long *begin = (unsigned long *)this_cpu_read(pcpu_hot.hardirq_stack_ptr); 65 unsigned long *begin = (unsigned long *)this_cpu_read(pcpu_hot.softirq_stack_ptr);
|
H A D | process_32.c | 193 * Reload esp0 and pcpu_hot.top_of_stack. This changes 199 this_cpu_write(pcpu_hot.top_of_stack, 209 raw_cpu_write(pcpu_hot.current_task, next_p);
|
H A D | dumpstack_64.c | 137 unsigned long *end = (unsigned long *)this_cpu_read(pcpu_hot.hardirq_stack_ptr);
|
H A D | setup_percpu.c | 172 per_cpu(pcpu_hot.cpu_number, cpu) = cpu;
|
H A D | process_64.c | 617 this_cpu_read(pcpu_hot.hardirq_stack_inuse)); 671 raw_cpu_write(pcpu_hot.current_task, next_p); 672 raw_cpu_write(pcpu_hot.top_of_stack, task_top_of_stack(next_p));
|
H A D | head_64.S | 322 movq pcpu_hot + X86_current_task(%rdx), %rax 438 movq PER_CPU_VAR(pcpu_hot + X86_current_task), %rcx
|
H A D | vmlinux.lds.S | 49 const_pcpu_hot = pcpu_hot;
|
H A D | smpboot.c | 856 per_cpu(pcpu_hot.current_task, cpu) = idle; 866 per_cpu(pcpu_hot.top_of_stack, cpu) = task_top_of_stack(idle);
|
/linux-master/arch/x86/entry/ |
H A D | entry_64_compat.S | 61 movq PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp 191 movq PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
|
H A D | entry_64.S | 95 movq PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp 1170 movq PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp 1488 movq PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rax
|
H A D | entry_32.S | 1156 movl PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %esp 1218 movl PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %esi
|
/linux-master/arch/x86/lib/ |
H A D | retpoline.S | 345 shlq $5, PER_CPU_VAR(pcpu_hot + X86_call_depth)
|
/linux-master/arch/x86/kernel/cpu/ |
H A D | common.c | 2002 DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = { 2007 EXPORT_PER_CPU_SYMBOL(pcpu_hot); variable
|