diff options
Diffstat (limited to 'include/asm-x86_64')
43 files changed, 397 insertions, 418 deletions
diff --git a/include/asm-x86_64/acpi.h b/include/asm-x86_64/acpi.h index 2c95a31..ed59aa4 100644 --- a/include/asm-x86_64/acpi.h +++ b/include/asm-x86_64/acpi.h @@ -155,8 +155,6 @@ extern void acpi_reserve_bootmem(void); #endif /*CONFIG_ACPI_SLEEP*/ -#define boot_cpu_physical_apicid boot_cpu_id - extern int acpi_disabled; extern int acpi_pci_disabled; diff --git a/include/asm-x86_64/alternative-asm.i b/include/asm-x86_64/alternative-asm.i new file mode 100644 index 0000000..e4041f4 --- /dev/null +++ b/include/asm-x86_64/alternative-asm.i @@ -0,0 +1,14 @@ +#include <linux/config.h> + +#ifdef CONFIG_SMP + .macro LOCK_PREFIX +1: lock + .section .smp_locks,"a" + .align 8 + .quad 1b + .previous + .endm +#else + .macro LOCK_PREFIX + .endm +#endif diff --git a/include/asm-x86_64/apic.h b/include/asm-x86_64/apic.h index 9c96a0a..9e66d32 100644 --- a/include/asm-x86_64/apic.h +++ b/include/asm-x86_64/apic.h @@ -17,6 +17,8 @@ extern int apic_verbosity; extern int apic_runs_main_timer; +extern int ioapic_force; +extern int apic_mapped; /* * Define the default level of output to be very little @@ -29,8 +31,6 @@ extern int apic_runs_main_timer; printk(s, ##a); \ } while (0) -#ifdef CONFIG_X86_LOCAL_APIC - struct pt_regs; /* @@ -95,17 +95,12 @@ extern void setup_APIC_extened_lvt(unsigned char lvt_off, unsigned char vector, #define K8_APIC_EXT_INT_MSG_EXT 0x7 #define K8_APIC_EXT_LVT_ENTRY_THRESHOLD 0 -extern int disable_timer_pin_1; - - void smp_send_timer_broadcast_ipi(void); void switch_APIC_timer_to_ipi(void *cpumask); void switch_ipi_to_APIC_timer(void *cpumask); #define ARCH_APICTIMER_STOPS_ON_C3 1 -#endif /* CONFIG_X86_LOCAL_APIC */ - extern unsigned boot_cpu_id; #endif /* __ASM_APIC_H */ diff --git a/include/asm-x86_64/bitops.h b/include/asm-x86_64/bitops.h index f7ba57b..5b535ea 100644 --- a/include/asm-x86_64/bitops.h +++ b/include/asm-x86_64/bitops.h @@ -399,6 +399,8 @@ static __inline__ int fls(int x) return r+1; } +#define ARCH_HAS_FAST_MULTIPLIER 1 + #include <asm-generic/bitops/hweight.h> #endif /* __KERNEL__ */ diff --git a/include/asm-x86_64/calgary.h b/include/asm-x86_64/calgary.h index 4e39195..6b93f5a 100644 --- a/include/asm-x86_64/calgary.h +++ b/include/asm-x86_64/calgary.h @@ -24,7 +24,6 @@ #ifndef _ASM_X86_64_CALGARY_H #define _ASM_X86_64_CALGARY_H -#include <linux/config.h> #include <linux/spinlock.h> #include <linux/device.h> #include <linux/dma-mapping.h> @@ -34,12 +33,12 @@ struct iommu_table { unsigned long it_base; /* mapped address of tce table */ unsigned long it_hint; /* Hint for next alloc */ unsigned long *it_map; /* A simple allocation bitmap for now */ + void __iomem *bbar; /* Bridge BAR */ + u64 tar_val; /* Table Address Register */ + struct timer_list watchdog_timer; spinlock_t it_lock; /* Protects it_map */ unsigned int it_size; /* Size of iommu table in entries */ unsigned char it_busno; /* Bus number this table belongs to */ - void __iomem *bbar; - u64 tar_val; - struct timer_list watchdog_timer; }; #define TCE_TABLE_SIZE_UNSPECIFIED ~0 diff --git a/include/asm-x86_64/dwarf2.h b/include/asm-x86_64/dwarf2.h index 0744db7..eedc085 100644 --- a/include/asm-x86_64/dwarf2.h +++ b/include/asm-x86_64/dwarf2.h @@ -13,7 +13,7 @@ away for older version. */ -#ifdef CONFIG_UNWIND_INFO +#ifdef CONFIG_AS_CFI #define CFI_STARTPROC .cfi_startproc #define CFI_ENDPROC .cfi_endproc @@ -28,6 +28,11 @@ #define CFI_REMEMBER_STATE .cfi_remember_state #define CFI_RESTORE_STATE .cfi_restore_state #define CFI_UNDEFINED .cfi_undefined +#ifdef CONFIG_AS_CFI_SIGNAL_FRAME +#define CFI_SIGNAL_FRAME .cfi_signal_frame +#else +#define CFI_SIGNAL_FRAME +#endif #else @@ -45,6 +50,7 @@ #define CFI_REMEMBER_STATE # #define CFI_RESTORE_STATE # #define CFI_UNDEFINED # +#define CFI_SIGNAL_FRAME # #endif diff --git a/include/asm-x86_64/e820.h b/include/asm-x86_64/e820.h index f656748..e15d3c8 100644 --- a/include/asm-x86_64/e820.h +++ b/include/asm-x86_64/e820.h @@ -19,13 +19,9 @@ #define E820_RAM 1 #define E820_RESERVED 2 -#define E820_ACPI 3 /* usable as RAM once ACPI tables have been read */ +#define E820_ACPI 3 #define E820_NVS 4 -#define HIGH_MEMORY (1024*1024) - -#define LOWMEMSIZE() (0x9f000) - #ifndef __ASSEMBLY__ struct e820entry { u64 addr; /* start of memory segment */ @@ -56,8 +52,7 @@ extern void e820_setup_gap(void); extern unsigned long e820_hole_size(unsigned long start_pfn, unsigned long end_pfn); -extern void __init parse_memopt(char *p, char **end); -extern void __init parse_memmapopt(char *p, char **end); +extern void finish_e820_parsing(void); extern struct e820map e820; diff --git a/include/asm-x86_64/fixmap.h b/include/asm-x86_64/fixmap.h index 0b4ffbd..1b620db 100644 --- a/include/asm-x86_64/fixmap.h +++ b/include/asm-x86_64/fixmap.h @@ -37,13 +37,9 @@ enum fixed_addresses { VSYSCALL_FIRST_PAGE = VSYSCALL_LAST_PAGE + ((VSYSCALL_END-VSYSCALL_START) >> PAGE_SHIFT) - 1, VSYSCALL_HPET, FIX_HPET_BASE, -#ifdef CONFIG_X86_LOCAL_APIC FIX_APIC_BASE, /* local (CPU) APIC) -- required for SMP or not */ -#endif -#ifdef CONFIG_X86_IO_APIC FIX_IO_APIC_BASE_0, FIX_IO_APIC_BASE_END = FIX_IO_APIC_BASE_0 + MAX_IO_APICS-1, -#endif __end_of_fixed_addresses }; diff --git a/include/asm-x86_64/genapic.h b/include/asm-x86_64/genapic.h index 50b38e7..81e7146 100644 --- a/include/asm-x86_64/genapic.h +++ b/include/asm-x86_64/genapic.h @@ -16,7 +16,6 @@ struct genapic { char *name; u32 int_delivery_mode; u32 int_dest_mode; - u32 int_delivery_dest; /* for quick IPIs */ int (*apic_id_registered)(void); cpumask_t (*target_cpus)(void); void (*init_apic_ldr)(void); diff --git a/include/asm-x86_64/i387.h b/include/asm-x86_64/i387.h index cba8a3b..0217b74 100644 --- a/include/asm-x86_64/i387.h +++ b/include/asm-x86_64/i387.h @@ -24,6 +24,7 @@ extern unsigned int mxcsr_feature_mask; extern void mxcsr_feature_mask_init(void); extern void init_fpu(struct task_struct *child); extern int save_i387(struct _fpstate __user *buf); +extern asmlinkage void math_state_restore(void); /* * FPU lazy state save handling... @@ -31,7 +32,9 @@ extern int save_i387(struct _fpstate __user *buf); #define unlazy_fpu(tsk) do { \ if (task_thread_info(tsk)->status & TS_USEDFPU) \ - save_init_fpu(tsk); \ + save_init_fpu(tsk); \ + else \ + tsk->fpu_counter = 0; \ } while (0) /* Ignore delayed exceptions from user space */ @@ -134,8 +137,8 @@ static inline int save_i387_checking(struct i387_fxsave_struct __user *fx) #else : [fx] "cdaSDb" (fx), "0" (0)); #endif - if (unlikely(err)) - __clear_user(fx, sizeof(struct i387_fxsave_struct)); + if (unlikely(err) && __clear_user(fx, sizeof(struct i387_fxsave_struct))) + err = -EFAULT; /* No need to clear here because the caller clears USED_MATH */ return err; } diff --git a/include/asm-x86_64/intel_arch_perfmon.h b/include/asm-x86_64/intel_arch_perfmon.h index 59c3964..8633331 100644 --- a/include/asm-x86_64/intel_arch_perfmon.h +++ b/include/asm-x86_64/intel_arch_perfmon.h @@ -14,6 +14,18 @@ #define ARCH_PERFMON_UNHALTED_CORE_CYCLES_SEL (0x3c) #define ARCH_PERFMON_UNHALTED_CORE_CYCLES_UMASK (0x00 << 8) -#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT (1 << 0) +#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX (0) +#define ARCH_PERFMON_UNHALTED_CORE_CYCLES_PRESENT \ + (1 << (ARCH_PERFMON_UNHALTED_CORE_CYCLES_INDEX)) + +union cpuid10_eax { + struct { + unsigned int version_id:8; + unsigned int num_counters:8; + unsigned int bit_width:8; + unsigned int mask_length:8; + } split; + unsigned int full; +}; #endif /* X86_64_INTEL_ARCH_PERFMON_H */ diff --git a/include/asm-x86_64/io_apic.h b/include/asm-x86_64/io_apic.h index fb7a090..5d1b5c6 100644 --- a/include/asm-x86_64/io_apic.h +++ b/include/asm-x86_64/io_apic.h @@ -10,8 +10,6 @@ * Copyright (C) 1997, 1998, 1999, 2000 Ingo Molnar */ -#ifdef CONFIG_X86_IO_APIC - #ifdef CONFIG_PCI_MSI static inline int use_pci_vector(void) {return 1;} static inline void disable_edge_ioapic_vector(unsigned int vector) { } @@ -209,10 +207,6 @@ extern int timer_uses_ioapic_pin_0; extern int sis_apic_bug; /* dummy */ -#else /* !CONFIG_X86_IO_APIC */ -#define io_apic_assign_pci_irqs 0 -#endif - extern int assign_irq_vector(int irq); void enable_NMI_through_LVT0 (void * dummy); diff --git a/include/asm-x86_64/irq.h b/include/asm-x86_64/irq.h index 9db5a1b..43469d8a 100644 --- a/include/asm-x86_64/irq.h +++ b/include/asm-x86_64/irq.h @@ -44,9 +44,7 @@ static __inline__ int irq_canonicalize(int irq) return ((irq == 2) ? 9 : irq); } -#ifdef CONFIG_X86_LOCAL_APIC #define ARCH_HAS_NMI_WATCHDOG /* See include/linux/nmi.h */ -#endif #ifdef CONFIG_HOTPLUG_CPU #include <linux/cpumask.h> diff --git a/include/asm-x86_64/kexec.h b/include/asm-x86_64/kexec.h index c564bae..5fab957 100644 --- a/include/asm-x86_64/kexec.h +++ b/include/asm-x86_64/kexec.h @@ -1,6 +1,27 @@ #ifndef _X86_64_KEXEC_H #define _X86_64_KEXEC_H +#define PA_CONTROL_PAGE 0 +#define VA_CONTROL_PAGE 1 +#define PA_PGD 2 +#define VA_PGD 3 +#define PA_PUD_0 4 +#define VA_PUD_0 5 +#define PA_PMD_0 6 +#define VA_PMD_0 7 +#define PA_PTE_0 8 +#define VA_PTE_0 9 +#define PA_PUD_1 10 +#define VA_PUD_1 11 +#define PA_PMD_1 12 +#define VA_PMD_1 13 +#define PA_PTE_1 14 +#define VA_PTE_1 15 +#define PA_TABLE_PAGE 16 +#define PAGES_NR 17 + +#ifndef __ASSEMBLY__ + #include <linux/string.h> #include <asm/page.h> @@ -64,4 +85,12 @@ static inline void crash_setup_regs(struct pt_regs *newregs, newregs->rip = (unsigned long)current_text_addr(); } } + +NORET_TYPE void +relocate_kernel(unsigned long indirection_page, + unsigned long page_list, + unsigned long start_address) ATTRIB_NORET; + +#endif /* __ASSEMBLY__ */ + #endif /* _X86_64_KEXEC_H */ diff --git a/include/asm-x86_64/linkage.h b/include/asm-x86_64/linkage.h index 291c2d0..b5f39d0 100644 --- a/include/asm-x86_64/linkage.h +++ b/include/asm-x86_64/linkage.h @@ -1,6 +1,6 @@ #ifndef __ASM_LINKAGE_H #define __ASM_LINKAGE_H -/* Nothing to see here... */ +#define __ALIGN .p2align 4,,15 #endif diff --git a/include/asm-x86_64/mach_apic.h b/include/asm-x86_64/mach_apic.h index 0acea44..d334224 100644 --- a/include/asm-x86_64/mach_apic.h +++ b/include/asm-x86_64/mach_apic.h @@ -16,7 +16,6 @@ #define INT_DELIVERY_MODE (genapic->int_delivery_mode) #define INT_DEST_MODE (genapic->int_dest_mode) -#define INT_DELIVERY_DEST (genapic->int_delivery_dest) #define TARGET_CPUS (genapic->target_cpus()) #define apic_id_registered (genapic->apic_id_registered) #define init_apic_ldr (genapic->init_apic_ldr) diff --git a/include/asm-x86_64/mce.h b/include/asm-x86_64/mce.h index d13687d..5a11146 100644 --- a/include/asm-x86_64/mce.h +++ b/include/asm-x86_64/mce.h @@ -99,6 +99,8 @@ static inline void mce_amd_feature_init(struct cpuinfo_x86 *c) } #endif +void mce_log_therm_throt_event(unsigned int cpu, __u64 status); + extern atomic_t mce_entry; #endif diff --git a/include/asm-x86_64/mmx.h b/include/asm-x86_64/mmx.h deleted file mode 100644 index 46b71da..0000000 --- a/include/asm-x86_64/mmx.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef _ASM_MMX_H -#define _ASM_MMX_H - -/* - * MMX 3Dnow! helper operations - */ - -#include <linux/types.h> - -extern void *_mmx_memcpy(void *to, const void *from, size_t size); -extern void mmx_clear_page(void *page); -extern void mmx_copy_page(void *to, void *from); - -#endif diff --git a/include/asm-x86_64/mpspec.h b/include/asm-x86_64/mpspec.h index 14fc3dd..017fddb 100644 --- a/include/asm-x86_64/mpspec.h +++ b/include/asm-x86_64/mpspec.h @@ -159,13 +159,7 @@ struct mpc_config_lintsrc #define MAX_MP_BUSSES 256 /* Each PCI slot may be a combo card with its own bus. 4 IRQ pins per slot. */ #define MAX_IRQ_SOURCES (MAX_MP_BUSSES * 4) -enum mp_bustype { - MP_BUS_ISA = 1, - MP_BUS_EISA, - MP_BUS_PCI, - MP_BUS_MCA -}; -extern unsigned char mp_bus_id_to_type [MAX_MP_BUSSES]; +extern DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES); extern int mp_bus_id_to_pci_bus [MAX_MP_BUSSES]; extern unsigned int boot_cpu_physical_apicid; @@ -178,18 +172,15 @@ extern int mp_irq_entries; extern struct mpc_config_intsrc mp_irqs [MAX_IRQ_SOURCES]; extern int mpc_default_type; extern unsigned long mp_lapic_addr; -extern int pic_mode; #ifdef CONFIG_ACPI extern void mp_register_lapic (u8 id, u8 enabled); extern void mp_register_lapic_address (u64 address); -#ifdef CONFIG_X86_IO_APIC extern void mp_register_ioapic (u8 id, u32 address, u32 gsi_base); extern void mp_override_legacy_irq (u8 bus_irq, u8 polarity, u8 trigger, u32 gsi); extern void mp_config_acpi_legacy_irqs (void); extern int mp_register_gsi (u32 gsi, int triggering, int polarity); -#endif /*CONFIG_X86_IO_APIC*/ #endif extern int using_apic_timer; diff --git a/include/asm-x86_64/msr.h b/include/asm-x86_64/msr.h index 10f8b51..37e1941 100644 --- a/include/asm-x86_64/msr.h +++ b/include/asm-x86_64/msr.h @@ -66,14 +66,25 @@ #define rdtscl(low) \ __asm__ __volatile__ ("rdtsc" : "=a" (low) : : "edx") +#define rdtscp(low,high,aux) \ + asm volatile (".byte 0x0f,0x01,0xf9" : "=a" (low), "=d" (high), "=c" (aux)) + #define rdtscll(val) do { \ unsigned int __a,__d; \ asm volatile("rdtsc" : "=a" (__a), "=d" (__d)); \ (val) = ((unsigned long)__a) | (((unsigned long)__d)<<32); \ } while(0) +#define rdtscpll(val, aux) do { \ + unsigned long __a, __d; \ + asm volatile (".byte 0x0f,0x01,0xf9" : "=a" (__a), "=d" (__d), "=c" (aux)); \ + (val) = (__d << 32) | __a; \ +} while (0) + #define write_tsc(val1,val2) wrmsr(0x10, val1, val2) +#define write_rdtscp_aux(val) wrmsr(0xc0000103, val, 0) + #define rdpmc(counter,low,high) \ __asm__ __volatile__("rdpmc" \ : "=a" (low), "=d" (high) \ diff --git a/include/asm-x86_64/mutex.h b/include/asm-x86_64/mutex.h index 06fab6d..16396b1 100644 --- a/include/asm-x86_64/mutex.h +++ b/include/asm-x86_64/mutex.h @@ -25,13 +25,9 @@ do { \ \ __asm__ __volatile__( \ LOCK_PREFIX " decl (%%rdi) \n" \ - " js 2f \n" \ - "1: \n" \ - \ - LOCK_SECTION_START("") \ - "2: call "#fail_fn" \n" \ - " jmp 1b \n" \ - LOCK_SECTION_END \ + " jns 1f \n" \ + " call "#fail_fn" \n" \ + "1:" \ \ :"=D" (dummy) \ : "D" (v) \ @@ -75,13 +71,9 @@ do { \ \ __asm__ __volatile__( \ LOCK_PREFIX " incl (%%rdi) \n" \ - " jle 2f \n" \ - "1: \n" \ - \ - LOCK_SECTION_START("") \ - "2: call "#fail_fn" \n" \ - " jmp 1b \n" \ - LOCK_SECTION_END \ + " jg 1f \n" \ + " call "#fail_fn" \n" \ + "1: " \ \ :"=D" (dummy) \ : "D" (v) \ diff --git a/include/asm-x86_64/nmi.h b/include/asm-x86_64/nmi.h index efb45c8..cbf2669 100644 --- a/include/asm-x86_64/nmi.h +++ b/include/asm-x86_64/nmi.h @@ -7,24 +7,13 @@ #include <linux/pm.h> #include <asm/io.h> -struct pt_regs; - -typedef int (*nmi_callback_t)(struct pt_regs * regs, int cpu); - -/** - * set_nmi_callback - * - * Set a handler for an NMI. Only one handler may be - * set. Return 1 if the NMI was handled. - */ -void set_nmi_callback(nmi_callback_t callback); - /** - * unset_nmi_callback + * do_nmi_callback * - * Remove the handler previously set. + * Check to see if a callback exists and execute it. Return 1 + * if the handler exists and was handled successfully. */ -void unset_nmi_callback(void); +int do_nmi_callback(struct pt_regs *regs, int cpu); #ifdef CONFIG_PM @@ -48,25 +37,32 @@ static inline void unset_nmi_pm_callback(struct pm_dev * dev) #endif /* CONFIG_PM */ extern void default_do_nmi(struct pt_regs *); -extern void die_nmi(char *str, struct pt_regs *regs); +extern void die_nmi(char *str, struct pt_regs *regs, int do_panic); #define get_nmi_reason() inb(0x61) extern int panic_on_timeout; extern int unknown_nmi_panic; +extern int nmi_watchdog_enabled; extern int check_nmi_watchdog(void); - -extern void setup_apic_nmi_watchdog (void); -extern int reserve_lapic_nmi(void); -extern void release_lapic_nmi(void); +extern int avail_to_resrv_perfctr_nmi_bit(unsigned int); +extern int avail_to_resrv_perfctr_nmi(unsigned int); +extern int reserve_perfctr_nmi(unsigned int); +extern void release_perfctr_nmi(unsigned int); +extern int reserve_evntsel_nmi(unsigned int); +extern void release_evntsel_nmi(unsigned int); + +extern void setup_apic_nmi_watchdog (void *); +extern void stop_apic_nmi_watchdog (void *); extern void disable_timer_nmi_watchdog(void); extern void enable_timer_nmi_watchdog(void); -extern void nmi_watchdog_tick (struct pt_regs * regs, unsigned reason); +extern int nmi_watchdog_tick (struct pt_regs * regs, unsigned reason); extern void nmi_watchdog_default(void); extern int setup_nmi_watchdog(char *); +extern atomic_t nmi_active; extern unsigned int nmi_watchdog; #define NMI_DEFAULT -1 #define NMI_NONE 0 diff --git a/include/asm-x86_64/pci-direct.h b/include/asm-x86_64/pci-direct.h index 036b6ca..eba9cb4 100644 --- a/include/asm-x86_64/pci-direct.h +++ b/include/asm-x86_64/pci-direct.h @@ -2,47 +2,15 @@ #define ASM_PCI_DIRECT_H 1 #include <linux/types.h> -#include <asm/io.h> /* Direct PCI access. This is used for PCI accesses in early boot before the PCI subsystem works. */ -#define PDprintk(x...) +extern u32 read_pci_config(u8 bus, u8 slot, u8 func, u8 offset); +extern u8 read_pci_config_byte(u8 bus, u8 slot, u8 func, u8 offset); +extern u16 read_pci_config_16(u8 bus, u8 slot, u8 func, u8 offset); +extern void write_pci_config(u8 bus, u8 slot, u8 func, u8 offset, u32 val); -static inline u32 read_pci_config(u8 bus, u8 slot, u8 func, u8 offset) -{ - u32 v; - outl(0x80000000 | (bus<<16) | (slot<<11) | (func<<8) | offset, 0xcf8); - v = inl(0xcfc); - if (v != 0xffffffff) - PDprintk("%x reading 4 from %x: %x\n", slot, offset, v); - return v; -} - -static inline u8 read_pci_config_byte(u8 bus, u8 slot, u8 func, u8 offset) -{ - u8 v; - outl(0x80000000 | (bus<<16) | (slot<<11) | (func<<8) | offset, 0xcf8); - v = inb(0xcfc + (offset&3)); - PDprintk("%x reading 1 from %x: %x\n", slot, offset, v); - return v; -} - -static inline u16 read_pci_config_16(u8 bus, u8 slot, u8 func, u8 offset) -{ - u16 v; - outl(0x80000000 | (bus<<16) | (slot<<11) | (func<<8) | offset, 0xcf8); - v = inw(0xcfc + (offset&2)); - PDprintk("%x reading 2 from %x: %x\n", slot, offset, v); - return v; -} - -static inline void write_pci_config(u8 bus, u8 slot, u8 func, u8 offset, - u32 val) -{ - PDprintk("%x writing to %x: %x\n", slot, offset, val); - outl(0x80000000 | (bus<<16) | (slot<<11) | (func<<8) | offset, 0xcf8); - outl(val, 0xcfc); -} +extern int early_pci_allowed(void); #endif diff --git a/include/asm-x86_64/pda.h b/include/asm-x86_64/pda.h index b47c3df..14996d9 100644 --- a/include/asm-x86_64/pda.h +++ b/include/asm-x86_64/pda.h @@ -9,20 +9,24 @@ /* Per processor datastructure. %gs points to it while the kernel runs */ struct x8664_pda { - struct task_struct *pcurrent; /* Current process */ - unsigned long data_offset; /* Per cpu data offset from linker address */ - unsigned long kernelstack; /* top of kernel stack for current */ - unsigned long oldrsp; /* user rsp for system call */ -#if DEBUG_STKSZ > EXCEPTION_STKSZ - unsigned long debugstack; /* #DB/#BP stack. */ + struct task_struct *pcurrent; /* 0 Current process */ + unsigned long data_offset; /* 8 Per cpu data offset from linker + address */ + unsigned long kernelstack; /* 16 top of kernel stack for current */ + unsigned long oldrsp; /* 24 user rsp for system call */ + int irqcount; /* 32 Irq nesting counter. Starts with -1 */ + int cpunumber; /* 36 Logical CPU number */ +#ifdef CONFIG_CC_STACKPROTECTOR + unsigned long stack_canary; /* 40 stack canary value */ + /* gcc-ABI: this canary MUST be at + offset 40!!! */ #endif - int irqcount; /* Irq nesting counter. Starts with -1 */ - int cpunumber; /* Logical CPU number */ - char *irqstackptr; /* top of irqstack */ + char *irqstackptr; int nodenumber; /* number of current node */ unsigned int __softirq_pending; unsigned int __nmi_count; /* number of NMI on this CPUs */ - int mmu_state; + short mmu_state; + short isidle; struct mm_struct *active_mm; unsigned apic_timer_irqs; } ____cacheline_aligned_in_smp; @@ -36,44 +40,69 @@ extern struct x8664_pda boot_cpu_pda[]; * There is no fast way to get the base address of the PDA, all the accesses * have to mention %fs/%gs. So it needs to be done this Torvaldian way. */ -#define sizeof_field(type,field) (sizeof(((type *)0)->field)) -#define typeof_field(type,field) typeof(((type *)0)->field) +extern void __bad_pda_field(void) __attribute__((noreturn)); -extern void __bad_pda_field(void); +/* + * proxy_pda doesn't actually exist, but tell gcc it is accessed for + * all PDA accesses so it gets read/write dependencies right. + */ +extern struct x8664_pda _proxy_pda; #define pda_offset(field) offsetof(struct x8664_pda, field) -#define pda_to_op(op,field,val) do { \ - typedef typeof_field(struct x8664_pda, field) T__; \ - switch (sizeof_field(struct x8664_pda, field)) { \ -case 2: \ -asm volatile(op "w %0,%%gs:%P1"::"ri" ((T__)val),"i"(pda_offset(field)):"memory"); break; \ -case 4: \ -asm volatile(op "l %0,%%gs:%P1"::"ri" ((T__)val),"i"(pda_offset(field)):"memory"); break; \ -case 8: \ -asm volatile(op "q %0,%%gs:%P1"::"ri" ((T__)val),"i"(pda_offset(field)):"memory"); break; \ - default: __bad_pda_field(); \ - } \ +#define pda_to_op(op,field,val) do { \ + typedef typeof(_proxy_pda.field) T__; \ + if (0) { T__ tmp__; tmp__ = (val); } /* type checking */ \ + switch (sizeof(_proxy_pda.field)) { \ + case 2: \ + asm(op "w %1,%%gs:%c2" : \ + "+m" (_proxy_pda.field) : \ + "ri" ((T__)val), \ + "i"(pda_offset(field))); \ + break; \ + case 4: \ + asm(op "l %1,%%gs:%c2" : \ + "+m" (_proxy_pda.field) : \ + "ri" ((T__)val), \ + "i" (pda_offset(field))); \ + break; \ + case 8: \ + asm(op "q %1,%%gs:%c2": \ + "+m" (_proxy_pda.field) : \ + "ri" ((T__)val), \ + "i"(pda_offset(field))); \ + break; \ + default: \ + __bad_pda_field(); \ + } \ } while (0) -/* - * AK: PDA read accesses should be neither volatile nor have an memory clobber. - * Unfortunately removing them causes all hell to break lose currently. - */ -#define pda_from_op(op,field) ({ \ - typeof_field(struct x8664_pda, field) ret__; \ - switch (sizeof_field(struct x8664_pda, field)) { \ -case 2: \ -asm volatile(op "w %%gs:%P1,%0":"=r" (ret__):"i"(pda_offset(field)):"memory"); break;\ -case 4: \ -asm volatile(op "l %%gs:%P1,%0":"=r" (ret__):"i"(pda_offset(field)):"memory"); break;\ -case 8: \ -asm volatile(op "q %%gs:%P1,%0":"=r" (ret__):"i"(pda_offset(field)):"memory"); break;\ - default: __bad_pda_field(); \ - } \ +#define pda_from_op(op,field) ({ \ + typeof(_proxy_pda.field) ret__; \ + switch (sizeof(_proxy_pda.field)) { \ + case 2: \ + asm(op "w %%gs:%c1,%0" : \ + "=r" (ret__) : \ + "i" (pda_offset(field)), \ + "m" (_proxy_pda.field)); \ + break; \ + case 4: \ + asm(op "l %%gs:%c1,%0": \ + "=r" (ret__): \ + "i" (pda_offset(field)), \ + "m" (_proxy_pda.field)); \ + break; \ + case 8: \ + asm(op "q %%gs:%c1,%0": \ + "=r" (ret__) : \ + "i" (pda_offset(field)), \ + "m" (_proxy_pda.field)); \ + break; \ + default: \ + __bad_pda_field(); \ + } \ ret__; }) - #define read_pda(field) pda_from_op("mov",field) #define write_pda(field,val) pda_to_op("mov",field,val) #define add_pda(field,val) pda_to_op("add",field,val) diff --git a/include/asm-x86_64/percpu.h b/include/asm-x86_64/percpu.h index bffb2f8..2857560 100644 --- a/include/asm-x86_64/percpu.h +++ b/include/asm-x86_64/percpu.h @@ -11,6 +11,16 @@ #include <asm/pda.h> +#ifdef CONFIG_MODULES +# define PERCPU_MODULE_RESERVE 8192 +#else +# define PERCPU_MODULE_RESERVE 0 +#endif + +#define PERCPU_ENOUGH_ROOM \ + (ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES) + \ + PERCPU_MODULE_RESERVE) + #define __per_cpu_offset(cpu) (cpu_pda(cpu)->data_offset) #define __my_cpu_offset() read_pda(data_offset) diff --git a/include/asm-x86_64/pgtable.h b/include/asm-x86_64/pgtable.h index 51eba23..6899e77 100644 --- a/include/asm-x86_64/pgtable.h +++ b/include/asm-x86_64/pgtable.h @@ -21,12 +21,9 @@ extern unsigned long __supported_pte_mask; #define swapper_pg_dir init_level4_pgt -extern int nonx_setup(char *str); extern void paging_init(void); extern void clear_kernel_mapping(unsigned long addr, unsigned long size); -extern unsigned long pgkern_mask; - /* * ZERO_PAGE is a global shared page that is always zero: used * for zero-mapped memory areas etc.. @@ -265,7 +262,7 @@ static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot) #define __LARGE_PTE (_PAGE_PSE|_PAGE_PRESENT) static inline int pte_user(pte_t pte) { return pte_val(pte) & _PAGE_USER; } static inline int pte_read(pte_t pte) { return pte_val(pte) & _PAGE_USER; } -static inline int pte_exec(pte_t pte) { return pte_val(pte) & _PAGE_USER; } +static inline int pte_exec(pte_t pte) { return !(pte_val(pte) & _PAGE_NX); } static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_DIRTY; } static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; } static inline int pte_write(pte_t pte) { return pte_val(pte) & _PAGE_RW; } @@ -278,11 +275,12 @@ static inline pte_t pte_mkclean(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) & static inline pte_t pte_mkold(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) & ~_PAGE_ACCESSED)); return pte; } static inline pte_t pte_wrprotect(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) & ~_PAGE_RW)); return pte; } static inline pte_t pte_mkread(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) | _PAGE_USER)); return pte; } -static inline pte_t pte_mkexec(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) | _PAGE_USER)); return pte; } +static inline pte_t pte_mkexec(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) & ~_PAGE_NX)); return pte; } static inline pte_t pte_mkdirty(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) | _PAGE_DIRTY)); return pte; } static inline pte_t pte_mkyoung(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) | _PAGE_ACCESSED)); return pte; } static inline pte_t pte_mkwrite(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) | _PAGE_RW)); return pte; } static inline pte_t pte_mkhuge(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) | _PAGE_PSE)); return pte; } +static inline pte_t pte_clrhuge(pte_t pte) { set_pte(&pte, __pte(pte_val(pte) & ~_PAGE_PSE)); return pte; } struct vm_area_struct; diff --git a/include/asm-x86_64/proto.h b/include/asm-x86_64/proto.h index 038fe1f4..b73d0c7 100644 --- a/include/asm-x86_64/proto.h +++ b/include/asm-x86_64/proto.h @@ -51,10 +51,8 @@ extern unsigned long long monotonic_base; extern int sysctl_vsyscall; extern int nohpet; extern unsigned long vxtime_hz; +extern void time_init_gtod(void); -extern int numa_setup(char *opt); - -extern int setup_early_printk(char *); extern void early_printk(const char *fmt, ...) __attribute__((format(printf,1,2))); extern void early_identify_cpu(struct cpuinfo_x86 *c); @@ -91,7 +89,7 @@ extern void syscall32_cpu_init(void); extern void setup_node_bootmem(int nodeid, unsigned long start, unsigned long end); -extern void check_ioapic(void); +extern void early_quirks(void); extern void check_efer(void); extern int unhandled_signal(struct task_struct *tsk, int sig); @@ -103,13 +101,7 @@ extern void select_idle_routine(const struct cpuinfo_x86 *c); extern unsigned long table_start, table_end; extern int exception_trace; -extern int using_apic_timer; -extern int disable_apic; extern unsigned cpu_khz; -extern int ioapic_force; -extern int skip_ioapic_setup; -extern int acpi_ht; -extern int acpi_disabled; extern void no_iommu_init(void); extern int force_iommu, no_iommu; @@ -131,7 +123,8 @@ extern int fix_aperture; extern int reboot_force; extern int notsc_setup(char *); -extern int setup_additional_cpus(char *); + +extern int gsi_irq_sharing(int gsi); extern void smp_local_timer_interrupt(struct pt_regs * regs); diff --git a/include/asm-x86_64/rwlock.h b/include/asm-x86_64/rwlock.h index dea0e94..72aeebe 100644 --- a/include/asm-x86_64/rwlock.h +++ b/include/asm-x86_64/rwlock.h @@ -18,69 +18,9 @@ #ifndef _ASM_X86_64_RWLOCK_H #define _ASM_X86_64_RWLOCK_H -#include <linux/stringify.h> - #define RW_LOCK_BIAS 0x01000000 -#define RW_LOCK_BIAS_STR "0x01000000" - -#define __build_read_lock_ptr(rw, helper) \ - asm volatile(LOCK_PREFIX "subl $1,(%0)\n\t" \ - "js 2f\n" \ - "1:\n" \ - LOCK_SECTION_START("") \ - "2:\tcall " helper "\n\t" \ - "jmp 1b\n" \ - LOCK_SECTION_END \ - ::"a" (rw) : "memory") - -#define __build_read_lock_const(rw, helper) \ - asm volatile(LOCK_PREFIX "subl $1,%0\n\t" \ - "js 2f\n" \ - "1:\n" \ - LOCK_SECTION_START("") \ - "2:\tpushq %%rax\n\t" \ - "leaq %0,%%rax\n\t" \ - "call " helper "\n\t" \ - "popq %%rax\n\t" \ - "jmp 1b\n" \ - LOCK_SECTION_END \ - :"=m" (*((volatile int *)rw))::"memory") - -#define __build_read_lock(rw, helper) do { \ - if (__builtin_constant_p(rw)) \ - __build_read_lock_const(rw, helper); \ - else \ - __build_read_lock_ptr(rw, helper); \ - } while (0) - -#define __build_write_lock_ptr(rw, helper) \ - asm volatile(LOCK_PREFIX "subl $" RW_LOCK_BIAS_STR ",(%0)\n\t" \ - "jnz 2f\n" \ - "1:\n" \ - LOCK_SECTION_START("") \ - "2:\tcall " helper "\n\t" \ - "jmp 1b\n" \ - LOCK_SECTION_END \ - ::"a" (rw) : "memory") - -#define __build_write_lock_const(rw, helper) \ - asm volatile(LOCK_PREFIX "subl $" RW_LOCK_BIAS_STR ",%0\n\t" \ - "jnz 2f\n" \ - "1:\n" \ - LOCK_SECTION_START("") \ - "2:\tpushq %%rax\n\t" \ - "leaq %0,%%rax\n\t" \ - "call " helper "\n\t" \ - "popq %%rax\n\t" \ - "jmp 1b\n" \ - LOCK_SECTION_END \ - :"=m" (*((volatile long *)rw))::"memory") +#define RW_LOCK_BIAS_STR "0x01000000" -#define __build_write_lock(rw, helper) do { \ - if (__builtin_constant_p(rw)) \ - __build_write_lock_const(rw, helper); \ - else \ - __build_write_lock_ptr(rw, helper); \ - } while (0) +/* Actual code is in asm/spinlock.h or in arch/x86_64/lib/rwlock.S */ #endif diff --git a/include/asm-x86_64/segment.h b/include/asm-x86_64/segment.h index d4bed33..334ddcd 100644 --- a/include/asm-x86_64/segment.h +++ b/include/asm-x86_64/segment.h @@ -20,15 +20,16 @@ #define __USER_CS 0x33 /* 6*8+3 */ #define __USER32_DS __USER_DS -#define GDT_ENTRY_TLS 1 #define GDT_ENTRY_TSS 8 /* needs two entries */ #define GDT_ENTRY_LDT 10 /* needs two entries */ #define GDT_ENTRY_TLS_MIN 12 #define GDT_ENTRY_TLS_MAX 14 -/* 15 free */ #define GDT_ENTRY_TLS_ENTRIES 3 +#define GDT_ENTRY_PER_CPU 15 /* Abused to load per CPU data from limit */ +#define __PER_CPU_SEG (GDT_ENTRY_PER_CPU * 8 + 3) + /* TLS indexes for 64bit - hardcoded in arch_prctl */ #define FS_TLS 0 #define GS_TLS 1 diff --git a/include/asm-x86_64/semaphore.h b/include/asm-x86_64/semaphore.h index 064df08..107bd90 100644 --- a/include/asm-x86_64/semaphore.h +++ b/include/asm-x86_64/semaphore.h @@ -107,12 +107,9 @@ static inline void down(struct semaphore * sem) __asm__ __volatile__( "# atomic down operation\n\t" LOCK_PREFIX "decl %0\n\t" /* --sem->count */ - "js 2f\n" - "1:\n" - LOCK_SECTION_START("") - "2:\tcall __down_failed\n\t" - "jmp 1b\n" - LOCK_SECTION_END + "jns 1f\n\t" + "call __down_failed\n" + "1:" :"=m" (sem->count) :"D" (sem) :"memory"); @@ -130,14 +127,11 @@ static inline int down_interruptible(struct semaphore * sem) __asm__ __volatile__( "# atomic interruptible down operation\n\t" + "xorl %0,%0\n\t" LOCK_PREFIX "decl %1\n\t" /* --sem->count */ - "js 2f\n\t" - "xorl %0,%0\n" - "1:\n" - LOCK_SECTION_START("") - "2:\tcall __down_failed_interruptible\n\t" - "jmp 1b\n" - LOCK_SECTION_END + "jns 2f\n\t" + "call __down_failed_interruptible\n" + "2:\n" :"=a" (result), "=m" (sem->count) :"D" (sem) :"memory"); @@ -154,14 +148,11 @@ static inline int down_trylock(struct semaphore * sem) __asm__ __volatile__( "# atomic interruptible down operation\n\t" + "xorl %0,%0\n\t" LOCK_PREFIX "decl %1\n\t" /* --sem->count */ - "js 2f\n\t" - "xorl %0,%0\n" - "1:\n" - LOCK_SECTION_START("") - "2:\tcall __down_failed_trylock\n\t" - "jmp 1b\n" - LOCK_SECTION_END + "jns 2f\n\t" + "call __down_failed_trylock\n\t" + "2:\n" :"=a" (result), "=m" (sem->count) :"D" (sem) :"memory","cc"); @@ -179,12 +170,9 @@ static inline void up(struct semaphore * sem) __asm__ __volatile__( "# atomic up operation\n\t" LOCK_PREFIX "incl %0\n\t" /* ++sem->count */ - "jle 2f\n" - "1:\n" - LOCK_SECTION_START("") - "2:\tcall __up_wakeup\n\t" - "jmp 1b\n" - LOCK_SECTION_END + "jg 1f\n\t" + "call __up_wakeup\n" + "1:" :"=m" (sem->count) :"D" (sem) :"memory"); diff --git a/include/asm-x86_64/signal.h b/include/asm-x86_64/signal.h index 3ede2a6..4581f97 100644 --- a/include/asm-x86_64/signal.h +++ b/include/asm-x86_64/signal.h @@ -24,10 +24,6 @@ typedef struct { } sigset_t; -struct pt_regs; -asmlinkage int do_signal(struct pt_regs *regs, sigset_t *oldset); - - #else /* Here we must cater to libcs that poke about in kernel headers. */ diff --git a/include/asm-x86_64/smp.h b/include/asm-x86_64/smp.h index ce97f65..d6b7c05 100644 --- a/include/asm-x86_64/smp.h +++ b/include/asm-x86_64/smp.h @@ -4,27 +4,18 @@ /* * We need the APIC definitions automatically as part of 'smp.h' */ -#ifndef __ASSEMBLY__ #include <linux/threads.h> #include <linux/cpumask.h> #include <linux/bitops.h> extern int disable_apic; -#endif -#ifdef CONFIG_X86_LOCAL_APIC -#ifndef __ASSEMBLY__ #include <asm/fixmap.h> #include <asm/mpspec.h> -#ifdef CONFIG_X86_IO_APIC #include <asm/io_apic.h> -#endif #include <asm/apic.h> #include <asm/thread_info.h> -#endif -#endif #ifdef CONFIG_SMP -#ifndef ASSEMBLY #include <asm/pda.h> @@ -42,7 +33,6 @@ extern cpumask_t cpu_initialized; extern void smp_alloc_memory(void); extern volatile unsigned long smp_invalidate_needed; -extern int pic_mode; extern void lock_ipi_call_lock(void); extern void unlock_ipi_call_lock(void); extern int smp_num_siblings; @@ -74,20 +64,16 @@ static inline int hard_smp_processor_id(void) return GET_APIC_ID(*(unsigned int *)(APIC_BASE+APIC_ID)); } -extern int safe_smp_processor_id(void); extern int __cpu_disable(void); extern void __cpu_die(unsigned int cpu); extern void prefill_possible_map(void); extern unsigned num_processors; extern unsigned disabled_cpus; -#endif /* !ASSEMBLY */ - #define NO_PROC_ID 0xFF /* No processor magic marker */ #endif -#ifndef ASSEMBLY /* * Some lowlevel functions might want to know about * the real APIC ID <-> CPU # mapping. @@ -109,11 +95,8 @@ static inline int cpu_present_to_apicid(int mps_cpu) return BAD_APICID; } -#endif /* !ASSEMBLY */ - #ifndef CONFIG_SMP #define stack_smp_processor_id() 0 -#define safe_smp_processor_id() 0 #define cpu_logical_map(x) (x) #else #include <asm/thread_info.h> @@ -125,19 +108,23 @@ static inline int cpu_present_to_apicid(int mps_cpu) }) #endif -#ifndef __ASSEMBLY__ static __inline int logical_smp_processor_id(void) { /* we don't want to mark this access volatile - bad code generation */ return GET_APIC_LOGICAL_ID(*(unsigned long *)(APIC_BASE+APIC_LDR)); } -#endif #ifdef CONFIG_SMP #define cpu_physical_id(cpu) x86_cpu_to_apicid[cpu] #else #define cpu_physical_id(cpu) boot_cpu_id -#endif - +static inline int smp_call_function_single(int cpuid, void (*func) (void *info), + void *info, int retry, int wait) +{ + /* Disable interrupts here? */ + func(info); + return 0; +} +#endif /* !CONFIG_SMP */ #endif diff --git a/include/asm-x86_64/spinlock.h b/include/asm-x86_64/spinlock.h index 248a79f..be7a9e6 100644 --- a/include/asm-x86_64/spinlock.h +++ b/include/asm-x86_64/spinlock.h @@ -16,31 +16,23 @@ * (the type definitions are in asm/spinlock_types.h) */ -#define __raw_spin_is_locked(x) \ - (*(volatile signed int *)(&(x)->slock) <= 0) - -#define __raw_spin_lock_string \ - "\n1:\t" \ - LOCK_PREFIX " ; decl %0\n\t" \ - "js 2f\n" \ - LOCK_SECTION_START("") \ - "2:\t" \ - "rep;nop\n\t" \ - "cmpl $0,%0\n\t" \ - "jle 2b\n\t" \ - "jmp 1b\n" \ - LOCK_SECTION_END - -#define __raw_spin_lock_string_up \ - "\n\tdecl %0" - -#define __raw_spin_unlock_string \ - "movl $1,%0" \ - :"=m" (lock->slock) : : "memory" +static inline int __raw_spin_is_locked(raw_spinlock_t *lock) +{ + return *(volatile signed int *)(&(lock)->slock) <= 0; +} static inline void __raw_spin_lock(raw_spinlock_t *lock) { - asm volatile(__raw_spin_lock_string : "=m" (lock->slock) : : "memory"); + asm volatile( + "\n1:\t" + LOCK_PREFIX " ; decl %0\n\t" + "jns 2f\n" + "3:\n" + "rep;nop\n\t" + "cmpl $0,%0\n\t" + "jle 3b\n\t" + "jmp 1b\n" + "2:\t" : "=m" (lock->slock) : : "memory"); } #define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock) @@ -49,7 +41,7 @@ static inline int __raw_spin_trylock(raw_spinlock_t *lock) { int oldval; - __asm__ __volatile__( + asm volatile( "xchgl %0,%1" :"=q" (oldval), "=m" (lock->slock) :"0" (0) : "memory"); @@ -59,13 +51,14 @@ static inline int __raw_spin_trylock(raw_spinlock_t *lock) static inline void __raw_spin_unlock(raw_spinlock_t *lock) { - __asm__ __volatile__( - __raw_spin_unlock_string - ); + asm volatile("movl $1,%0" :"=m" (lock->slock) :: "memory"); } -#define __raw_spin_unlock_wait(lock) \ - do { while (__raw_spin_is_locked(lock)) cpu_relax(); } while (0) +static inline void __raw_spin_unlock_wait(raw_spinlock_t *lock) +{ + while (__raw_spin_is_locked(lock)) + cpu_relax(); +} /* * Read-write spinlocks, allowing multiple readers @@ -79,26 +72,34 @@ static inline void __raw_spin_unlock(raw_spinlock_t *lock) * * On x86, we implement read-write locks as a 32-bit counter * with the high bit (sign) being the "contended" bit. - * - * The inline assembly is non-obvious. Think about it. - * - * Changed to use the same technique as rw semaphores. See - * semaphore.h for details. -ben - * - * the helpers are in arch/i386/kernel/semaphore.c */ -#define __raw_read_can_lock(x) ((int)(x)->lock > 0) -#define __raw_write_can_lock(x) ((x)->lock == RW_LOCK_BIAS) +static inline int __raw_read_can_lock(raw_rwlock_t *lock) +{ + return (int)(lock)->lock > 0; +} + +static inline int __raw_write_can_lock(raw_rwlock_t *lock) +{ + return (lock)->lock == RW_LOCK_BIAS; +} static inline void __raw_read_lock(raw_rwlock_t *rw) { - __build_read_lock(rw, "__read_lock_failed"); + asm volatile(LOCK_PREFIX "subl $1,(%0)\n\t" + "jns 1f\n" + "call __read_lock_failed\n" + "1:\n" + ::"D" (rw), "i" (RW_LOCK_BIAS) : "memory"); } static inline void __raw_write_lock(raw_rwlock_t *rw) { - __build_write_lock(rw, "__write_lock_failed"); + asm volatile(LOCK_PREFIX "subl %1,(%0)\n\t" + "jz 1f\n" + "\tcall __write_lock_failed\n\t" + "1:\n" + ::"D" (rw), "i" (RW_LOCK_BIAS) : "memory"); } static inline int __raw_read_trylock(raw_rwlock_t *lock) diff --git a/include/asm-x86_64/stacktrace.h b/include/asm-x86_64/stacktrace.h new file mode 100644 index 0000000..5eb9799 --- /dev/null +++ b/include/asm-x86_64/stacktrace.h @@ -0,0 +1,18 @@ +#ifndef _ASM_STACKTRACE_H +#define _ASM_STACKTRACE_H 1 + +/* Generic stack tracer with callbacks */ + +struct stacktrace_ops { + void (*warning)(void *data, char *msg); + /* msg must contain %s for the symbol */ + void (*warning_symbol)(void *data, char *msg, unsigned long symbol); + void (*address)(void *data, unsigned long address); + /* On negative return stop dumping */ + int (*stack)(void *data, char *name); +}; + +void dump_trace(struct task_struct *tsk, struct pt_regs *regs, unsigned long *stack, + struct stacktrace_ops *ops, void *data); + +#endif diff --git a/include/asm-x86_64/system.h b/include/asm-x86_64/system.h index 6bf170b..bd376bc 100644 --- a/include/asm-x86_64/system.h +++ b/include/asm-x86_64/system.h @@ -14,12 +14,13 @@ #define __RESTORE(reg,offset) "movq (14-" #offset ")*8(%%rsp),%%" #reg "\n\t" /* frame pointer must be last for get_wchan */ -#define SAVE_CONTEXT "pushq %%rbp ; movq %%rsi,%%rbp\n\t" -#define RESTORE_CONTEXT "movq %%rbp,%%rsi ; popq %%rbp\n\t" +#define SAVE_CONTEXT "pushf ; pushq %%rbp ; movq %%rsi,%%rbp\n\t" +#define RESTORE_CONTEXT "movq %%rbp,%%rsi ; popq %%rbp ; popf\t" #define __EXTRA_CLOBBER \ ,"rcx","rbx","rdx","r8","r9","r10","r11","r12","r13","r14","r15" +/* Save restore flags to clear handle leaking NT */ #define switch_to(prev,next,last) \ asm volatile(SAVE_CONTEXT \ "movq %%rsp,%P[threadrsp](%[prev])\n\t" /* save RSP */ \ diff --git a/include/asm-x86_64/tce.h b/include/asm-x86_64/tce.h index 53e9a68..dbb047f 100644 --- a/include/asm-x86_64/tce.h +++ b/include/asm-x86_64/tce.h @@ -24,7 +24,6 @@ #ifndef _ASM_X86_64_TCE_H #define _ASM_X86_64_TCE_H -extern void* tce_table_kva[]; extern unsigned int specified_table_size; struct iommu_table; diff --git a/include/asm-x86_64/therm_throt.h b/include/asm-x86_64/therm_throt.h new file mode 100644 index 0000000..5aac059 --- /dev/null +++ b/include/asm-x86_64/therm_throt.h @@ -0,0 +1 @@ +#include <asm-i386/therm_throt.h> diff --git a/include/asm-x86_64/thread_info.h b/include/asm-x86_64/thread_info.h index 2029b00..787a081 100644 --- a/include/asm-x86_64/thread_info.h +++ b/include/asm-x86_64/thread_info.h @@ -114,11 +114,14 @@ static inline struct thread_info *stack_thread_info(void) #define TIF_IRET 5 /* force IRET */ #define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */ #define TIF_SECCOMP 8 /* secure computing */ +#define TIF_RESTORE_SIGMASK 9 /* restore signal mask in do_signal */ /* 16 free */ #define TIF_IA32 17 /* 32bit process */ #define TIF_FORK 18 /* ret_from_fork */ #define TIF_ABI_PENDING 19 #define TIF_MEMDIE 20 +#define TIF_DEBUG 21 /* uses debug registers */ +#define TIF_IO_BITMAP 22 /* uses I/O bitmap */ #define _TIF_SYSCALL_TRACE (1<<TIF_SYSCALL_TRACE) #define _TIF_NOTIFY_RESUME (1<<TIF_NOTIFY_RESUME) @@ -128,9 +131,12 @@ static inline struct thread_info *stack_thread_info(void) #define _TIF_IRET (1<<TIF_IRET) #define _TIF_SYSCALL_AUDIT (1<<TIF_SYSCALL_AUDIT) #define _TIF_SECCOMP (1<<TIF_SECCOMP) +#define _TIF_RESTORE_SIGMASK (1<<TIF_RESTORE_SIGMASK) #define _TIF_IA32 (1<<TIF_IA32) #define _TIF_FORK (1<<TIF_FORK) #define _TIF_ABI_PENDING (1<<TIF_ABI_PENDING) +#define _TIF_DEBUG (1<<TIF_DEBUG) +#define _TIF_IO_BITMAP (1<<TIF_IO_BITMAP) /* work to do on interrupt/exception return */ #define _TIF_WORK_MASK \ @@ -138,6 +144,9 @@ static inline struct thread_info *stack_thread_info(void) /* work to do on any return to user space */ #define _TIF_ALLWORK_MASK (0x0000FFFF & ~_TIF_SECCOMP) +/* flags to check in __switch_to() */ +#define _TIF_WORK_CTXSW (_TIF_DEBUG|_TIF_IO_BITMAP) + #define PREEMPT_ACTIVE 0x10000000 /* diff --git a/include/asm-x86_64/tlbflush.h b/include/asm-x86_64/tlbflush.h index d16d5b6..983bd29 100644 --- a/include/asm-x86_64/tlbflush.h +++ b/include/asm-x86_64/tlbflush.h @@ -4,44 +4,44 @@ #include <linux/mm.h> #include <asm/processor.h> -#define __flush_tlb() \ - do { \ - unsigned long tmpreg; \ - \ - __asm__ __volatile__( \ - "movq %%cr3, %0; # flush TLB \n" \ - "movq %0, %%cr3; \n" \ - : "=r" (tmpreg) \ - :: "memory"); \ - } while (0) +static inline unsigned long get_cr3(void) +{ + unsigned long cr3; + asm volatile("mov %%cr3,%0" : "=r" (cr3)); + return cr3; +} -/* - * Global pages have to be flushed a bit differently. Not a real - * performance problem because this does not happen often. - */ -#define __flush_tlb_global() \ - do { \ - unsigned long tmpreg, cr4, cr4_orig; \ - \ - __asm__ __volatile__( \ - "movq %%cr4, %2; # turn off PGE \n" \ - "movq %2, %1; \n" \ - "andq %3, %1; \n" \ - "movq %1, %%cr4; \n" \ - "movq %%cr3, %0; # flush TLB \n" \ - "movq %0, %%cr3; \n" \ - "movq %2, %%cr4; # turn PGE back on \n" \ - : "=&r" (tmpreg), "=&r" (cr4), "=&r" (cr4_orig) \ - : "i" (~X86_CR4_PGE) \ - : "memory"); \ - } while (0) - -extern unsigned long pgkern_mask; - -#define __flush_tlb_all() __flush_tlb_global() +static inline void set_cr3(unsigned long cr3) +{ + asm volatile("mov %0,%%cr3" :: "r" (cr3) : "memory"); +} + +static inline void __flush_tlb(void) +{ + set_cr3(get_cr3()); +} + +static inline unsigned long get_cr4(void) +{ + unsigned long cr4; + asm volatile("mov %%cr4,%0" : "=r" (cr4)); + return cr4; +} + +static inline void set_cr4(unsigned long cr4) +{ + asm volatile("mov %0,%%cr4" :: "r" (cr4) : "memory"); +} + +static inline void __flush_tlb_all(void) +{ + unsigned long cr4 = get_cr4(); + set_cr4(cr4 & ~X86_CR4_PGE); /* clear PGE */ + set_cr4(cr4); /* write old PGE again and flush TLBs */ +} #define __flush_tlb_one(addr) \ - __asm__ __volatile__("invlpg %0": :"m" (*(char *) addr)) + __asm__ __volatile__("invlpg (%0)" :: "r" (addr) : "memory") /* diff --git a/include/asm-x86_64/uaccess.h b/include/asm-x86_64/uaccess.h index 1e1fa00..e856570 100644 --- a/include/asm-x86_64/uaccess.h +++ b/include/asm-x86_64/uaccess.h @@ -84,7 +84,7 @@ struct exception_table_entry */ #define __get_user_x(size,ret,x,ptr) \ - __asm__ __volatile__("call __get_user_" #size \ + asm volatile("call __get_user_" #size \ :"=a" (ret),"=d" (x) \ :"c" (ptr) \ :"r8") @@ -101,7 +101,7 @@ struct exception_table_entry case 8: __get_user_x(8,__ret_gu,__val_gu,ptr); break; \ default: __get_user_bad(); break; \ } \ - (x) = (__typeof__(*(ptr)))__val_gu; \ + (x) = (typeof(*(ptr)))__val_gu; \ __ret_gu; \ }) @@ -112,7 +112,7 @@ extern void __put_user_8(void); extern void __put_user_bad(void); #define __put_user_x(size,ret,x,ptr) \ - __asm__ __volatile__("call __put_user_" #size \ + asm volatile("call __put_user_" #size \ :"=a" (ret) \ :"c" (ptr),"d" (x) \ :"r8") @@ -139,7 +139,7 @@ extern void __put_user_bad(void); #define __put_user_check(x,ptr,size) \ ({ \ int __pu_err; \ - __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ + typeof(*(ptr)) __user *__pu_addr = (ptr); \ switch (size) { \ case 1: __put_user_x(1,__pu_err,x,__pu_addr); break; \ case 2: __put_user_x(2,__pu_err,x,__pu_addr); break; \ @@ -173,7 +173,7 @@ struct __large_struct { unsigned long buf[100]; }; * aliasing issues. */ #define __put_user_asm(x, addr, err, itype, rtype, ltype, errno) \ - __asm__ __volatile__( \ + asm volatile( \ "1: mov"itype" %"rtype"1,%2\n" \ "2:\n" \ ".section .fixup,\"ax\"\n" \ @@ -193,7 +193,7 @@ struct __large_struct { unsigned long buf[100]; }; int __gu_err; \ unsigned long __gu_val; \ __get_user_size(__gu_val,(ptr),(size),__gu_err); \ - (x) = (__typeof__(*(ptr)))__gu_val; \ + (x) = (typeof(*(ptr)))__gu_val; \ __gu_err; \ }) @@ -217,7 +217,7 @@ do { \ } while (0) #define __get_user_asm(x, addr, err, itype, rtype, ltype, errno) \ - __asm__ __volatile__( \ + asm volatile( \ "1: mov"itype" %2,%"rtype"1\n" \ "2:\n" \ ".section .fixup,\"ax\"\n" \ @@ -237,15 +237,20 @@ do { \ */ /* Handles exceptions in both to and from, but doesn't do access_ok */ -extern unsigned long copy_user_generic(void *to, const void *from, unsigned len); - -extern unsigned long copy_to_user(void __user *to, const void *from, unsigned len); -extern unsigned long copy_from_user(void *to, const void __user *from, unsigned len); -extern unsigned long copy_in_user(void __user *to, const void __user *from, unsigned len); - -static __always_inline int __copy_from_user(void *dst, const void __user *src, unsigned size) +__must_check unsigned long +copy_user_generic(void *to, const void *from, unsigned len); + +__must_check unsigned long +copy_to_user(void __user *to, const void *from, unsigned len); +__must_check unsigned long +copy_from_user(void *to, const void __user *from, unsigned len); +__must_check unsigned long +copy_in_user(void __user *to, const void __user *from, unsigned len); + +static __always_inline __must_check +int __copy_from_user(void *dst, const void __user *src, unsigned size) { - int ret = 0; + int ret = 0; if (!__builtin_constant_p(size)) return copy_user_generic(dst,(__force void *)src,size); switch (size) { @@ -272,9 +277,10 @@ static __always_inline int __copy_from_user(void *dst, const void __user *src, u } } -static __always_inline int __copy_to_user(void __user *dst, const void *src, unsigned size) +static __always_inline __must_check +int __copy_to_user(void __user *dst, const void *src, unsigned size) { - int ret = 0; + int ret = 0; if (!__builtin_constant_p(size)) return copy_user_generic((__force void *)dst,src,size); switch (size) { @@ -303,10 +309,10 @@ static __always_inline int __copy_to_user(void __user *dst, const void *src, uns } } - -static __always_inline int __copy_in_user(void __user *dst, const void __user *src, unsigned size) +static __always_inline __must_check +int __copy_in_user(void __user *dst, const void __user *src, unsigned size) { - int ret = 0; + int ret = 0; if (!__builtin_constant_p(size)) return copy_user_generic((__force void *)dst,(__force void *)src,size); switch (size) { @@ -344,15 +350,17 @@ static __always_inline int __copy_in_user(void __user *dst, const void __user *s } } -long strncpy_from_user(char *dst, const char __user *src, long count); -long __strncpy_from_user(char *dst, const char __user *src, long count); -long strnlen_user(const char __user *str, long n); -long __strnlen_user(const char __user *str, long n); -long strlen_user(const char __user *str); -unsigned long clear_user(void __user *mem, unsigned long len); -unsigned long __clear_user(void __user *mem, unsigned long len); - -#define __copy_to_user_inatomic __copy_to_user -#define __copy_from_user_inatomic __copy_from_user +__must_check long +strncpy_from_user(char *dst, const char __user *src, long count); +__must_check long +__strncpy_from_user(char *dst, const char __user *src, long count); +__must_check long strnlen_user(const char __user *str, long n); +__must_check long __strnlen_user(const char __user *str, long n); +__must_check long strlen_user(const char __user *str); +__must_check unsigned long clear_user(void __user *mem, unsigned long len); +__must_check unsigned long __clear_user(void __user *mem, unsigned long len); + +__must_check long __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size); +#define __copy_to_user_inatomic copy_user_generic #endif /* __X86_64_UACCESS_H */ diff --git a/include/asm-x86_64/unistd.h b/include/asm-x86_64/unistd.h index 80fd48e..eeb98c1 100644 --- a/include/asm-x86_64/unistd.h +++ b/include/asm-x86_64/unistd.h @@ -600,9 +600,9 @@ __SYSCALL(__NR_fchmodat, sys_fchmodat) #define __NR_faccessat 269 __SYSCALL(__NR_faccessat, sys_faccessat) #define __NR_pselect6 270 -__SYSCALL(__NR_pselect6, sys_ni_syscall) /* for now */ +__SYSCALL(__NR_pselect6, sys_pselect6) #define __NR_ppoll 271 -__SYSCALL(__NR_ppoll, sys_ni_syscall) /* for now */ +__SYSCALL(__NR_ppoll, sys_ppoll) #define __NR_unshare 272 __SYSCALL(__NR_unshare, sys_unshare) #define __NR_set_robust_list 273 @@ -658,6 +658,7 @@ do { \ #define __ARCH_WANT_SYS_SIGPENDING #define __ARCH_WANT_SYS_SIGPROCMASK #define __ARCH_WANT_SYS_RT_SIGACTION +#define __ARCH_WANT_SYS_RT_SIGSUSPEND #define __ARCH_WANT_SYS_TIME #define __ARCH_WANT_COMPAT_SYS_TIME diff --git a/include/asm-x86_64/unwind.h b/include/asm-x86_64/unwind.h index 1f6e9bf..2e7ff10 100644 --- a/include/asm-x86_64/unwind.h +++ b/include/asm-x86_64/unwind.h @@ -18,6 +18,7 @@ struct unwind_frame_info { struct pt_regs regs; struct task_struct *task; + unsigned call_frame:1; }; #define UNW_PC(frame) (frame)->regs.rip @@ -57,6 +58,10 @@ struct unwind_frame_info PTREGS_INFO(r15), \ PTREGS_INFO(rip) +#define UNW_DEFAULT_RA(raItem, dataAlign) \ + ((raItem).where == Memory && \ + !((raItem).value * (dataAlign) + 8)) + static inline void arch_unw_init_frame_info(struct unwind_frame_info *info, /*const*/ struct pt_regs *regs) { @@ -94,8 +99,8 @@ static inline int arch_unw_user_mode(const struct unwind_frame_info *info) #else -#define UNW_PC(frame) ((void)(frame), 0) -#define UNW_SP(frame) ((void)(frame), 0) +#define UNW_PC(frame) ((void)(frame), 0UL) +#define UNW_SP(frame) ((void)(frame), 0UL) static inline int arch_unw_user_mode(const void *info) { diff --git a/include/asm-x86_64/vsyscall.h b/include/asm-x86_64/vsyscall.h index 146b244..2281e93 100644 --- a/include/asm-x86_64/vsyscall.h +++ b/include/asm-x86_64/vsyscall.h @@ -4,6 +4,7 @@ enum vsyscall_num { __NR_vgettimeofday, __NR_vtime, + __NR_vgetcpu, }; #define VSYSCALL_START (-10UL << 20) @@ -15,6 +16,7 @@ enum vsyscall_num { #include <linux/seqlock.h> #define __section_vxtime __attribute__ ((unused, __section__ (".vxtime"), aligned(16))) +#define __section_vgetcpu_mode __attribute__ ((unused, __section__ (".vgetcpu_mode"), aligned(16))) #define __section_wall_jiffies __attribute__ ((unused, __section__ (".wall_jiffies"), aligned(16))) #define __section_jiffies __attribute__ ((unused, __section__ (".jiffies"), aligned(16))) #define __section_sys_tz __attribute__ ((unused, __section__ (".sys_tz"), aligned(16))) @@ -26,6 +28,9 @@ enum vsyscall_num { #define VXTIME_HPET 2 #define VXTIME_PMTMR 3 +#define VGETCPU_RDTSCP 1 +#define VGETCPU_LSL 2 + struct vxtime_data { long hpet_address; /* HPET base address */ int last; @@ -40,6 +45,7 @@ struct vxtime_data { /* vsyscall space (readonly) */ extern struct vxtime_data __vxtime; +extern int __vgetcpu_mode; extern struct timespec __xtime; extern volatile unsigned long __jiffies; extern unsigned long __wall_jiffies; @@ -48,6 +54,7 @@ extern seqlock_t __xtime_lock; /* kernel space (writeable) */ extern struct vxtime_data vxtime; +extern int vgetcpu_mode; extern unsigned long wall_jiffies; extern struct timezone sys_tz; extern int sysctl_vsyscall; @@ -55,6 +62,8 @@ extern seqlock_t xtime_lock; extern int sysctl_vsyscall; +extern void vsyscall_set_cpu(int cpu); + #define ARCH_HAVE_XTIME_LOCK 1 #endif /* __KERNEL__ */ |