diff options
Diffstat (limited to 'arch/ia64/kernel')
| -rw-r--r-- | arch/ia64/kernel/Makefile | 1 | ||||
| -rw-r--r-- | arch/ia64/kernel/efi.c | 32 | ||||
| -rw-r--r-- | arch/ia64/kernel/entry.S | 2 | ||||
| -rw-r--r-- | arch/ia64/kernel/entry.h | 2 | ||||
| -rw-r--r-- | arch/ia64/kernel/ivt.S | 13 | ||||
| -rw-r--r-- | arch/ia64/kernel/ptrace.c | 15 | ||||
| -rw-r--r-- | arch/ia64/kernel/signal.c | 5 | ||||
| -rw-r--r-- | arch/ia64/kernel/smp.c | 2 | ||||
| -rw-r--r-- | arch/ia64/kernel/uncached.c | 246 |
9 files changed, 304 insertions, 14 deletions
diff --git a/arch/ia64/kernel/Makefile b/arch/ia64/kernel/Makefile index c1a02bb..4c73d8b 100644 --- a/arch/ia64/kernel/Makefile +++ b/arch/ia64/kernel/Makefile @@ -20,6 +20,7 @@ obj-$(CONFIG_SMP) += smp.o smpboot.o domain.o obj-$(CONFIG_PERFMON) += perfmon_default_smpl.o obj-$(CONFIG_IA64_CYCLONE) += cyclone.o obj-$(CONFIG_IA64_MCA_RECOVERY) += mca_recovery.o +obj-$(CONFIG_IA64_UNCACHED_ALLOCATOR) += uncached.o mca_recovery-y += mca_drv.o mca_drv_asm.o # The gate DSO image is built using a special linker script. diff --git a/arch/ia64/kernel/efi.c b/arch/ia64/kernel/efi.c index 4a3b1aa..179f230 100644 --- a/arch/ia64/kernel/efi.c +++ b/arch/ia64/kernel/efi.c @@ -410,6 +410,38 @@ efi_memmap_walk (efi_freemem_callback_t callback, void *arg) } /* + * Walk the EFI memory map to pull out leftover pages in the lower + * memory regions which do not end up in the regular memory map and + * stick them into the uncached allocator + * + * The regular walk function is significantly more complex than the + * uncached walk which means it really doesn't make sense to try and + * marge the two. + */ +void __init +efi_memmap_walk_uc (efi_freemem_callback_t callback) +{ + void *efi_map_start, *efi_map_end, *p; + efi_memory_desc_t *md; + u64 efi_desc_size, start, end; + + efi_map_start = __va(ia64_boot_param->efi_memmap); + efi_map_end = efi_map_start + ia64_boot_param->efi_memmap_size; + efi_desc_size = ia64_boot_param->efi_memdesc_size; + + for (p = efi_map_start; p < efi_map_end; p += efi_desc_size) { + md = p; + if (md->attribute == EFI_MEMORY_UC) { + start = PAGE_ALIGN(md->phys_addr); + end = PAGE_ALIGN((md->phys_addr+(md->num_pages << EFI_PAGE_SHIFT)) & PAGE_MASK); + if ((*callback)(start, end, NULL) < 0) + return; + } + } +} + + +/* * Look for the PAL_CODE region reported by EFI and maps it using an * ITR to enable safe PAL calls in virtual mode. See IA-64 Processor * Abstraction Layer chapter 11 in ADAG diff --git a/arch/ia64/kernel/entry.S b/arch/ia64/kernel/entry.S index d99316c..b1d5d3d 100644 --- a/arch/ia64/kernel/entry.S +++ b/arch/ia64/kernel/entry.S @@ -1579,7 +1579,7 @@ sys_call_table: data8 sys_keyctl data8 sys_ni_syscall data8 sys_ni_syscall // 1275 - data8 sys_ni_syscall + data8 sys_set_zone_reclaim data8 sys_ni_syscall data8 sys_ni_syscall data8 sys_ni_syscall diff --git a/arch/ia64/kernel/entry.h b/arch/ia64/kernel/entry.h index 6d4ecec..78eeb07 100644 --- a/arch/ia64/kernel/entry.h +++ b/arch/ia64/kernel/entry.h @@ -60,7 +60,7 @@ .spillsp @priunat,SW(AR_UNAT)+16+(off); \ .spillsp ar.rnat,SW(AR_RNAT)+16+(off); \ .spillsp ar.bspstore,SW(AR_BSPSTORE)+16+(off); \ - .spillsp pr,SW(PR)+16+(off)) + .spillsp pr,SW(PR)+16+(off) #define DO_SAVE_SWITCH_STACK \ movl r28=1f; \ diff --git a/arch/ia64/kernel/ivt.S b/arch/ia64/kernel/ivt.S index d9c05d5..2bc085a 100644 --- a/arch/ia64/kernel/ivt.S +++ b/arch/ia64/kernel/ivt.S @@ -405,17 +405,22 @@ ENTRY(nested_dtlb_miss) * r30: continuation address * r31: saved pr * - * Clobbered: b0, r18, r19, r21, psr.dt (cleared) + * Clobbered: b0, r18, r19, r21, r22, psr.dt (cleared) */ rsm psr.dt // switch to using physical data addressing mov r19=IA64_KR(PT_BASE) // get the page table base address shl r21=r16,3 // shift bit 60 into sign bit + mov r18=cr.itir ;; shr.u r17=r16,61 // get the region number into r17 + extr.u r18=r18,2,6 // get the faulting page size ;; cmp.eq p6,p7=5,r17 // is faulting address in region 5? - shr.u r18=r16,PGDIR_SHIFT // get bits 33-63 of faulting address + add r22=-PAGE_SHIFT,r18 // adjustment for hugetlb address + add r18=PGDIR_SHIFT-PAGE_SHIFT,r18 ;; + shr.u r22=r16,r22 + shr.u r18=r16,r18 (p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place srlz.d @@ -428,7 +433,7 @@ ENTRY(nested_dtlb_miss) (p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // r17=PTA + IFA(33,42)*8 (p7) dep r17=r18,r17,3,(PAGE_SHIFT-6) // r17=PTA + (((IFA(61,63) << 7) | IFA(33,39))*8) cmp.eq p7,p6=0,r21 // unused address bits all zeroes? - shr.u r18=r16,PMD_SHIFT // shift L2 index into position + shr.u r18=r22,PMD_SHIFT // shift L2 index into position ;; ld8 r17=[r17] // fetch the L1 entry (may be 0) ;; @@ -436,7 +441,7 @@ ENTRY(nested_dtlb_miss) dep r17=r18,r17,3,(PAGE_SHIFT-3) // compute address of L2 page table entry ;; (p7) ld8 r17=[r17] // fetch the L2 entry (may be 0) - shr.u r19=r16,PAGE_SHIFT // shift L3 index into position + shr.u r19=r22,PAGE_SHIFT // shift L3 index into position ;; (p7) cmp.eq.or.andcm p6,p7=r17,r0 // was L2 entry NULL? dep r17=r19,r17,3,(PAGE_SHIFT-3) // compute address of L3 page table entry diff --git a/arch/ia64/kernel/ptrace.c b/arch/ia64/kernel/ptrace.c index 575a8f6..6d57aeb 100644 --- a/arch/ia64/kernel/ptrace.c +++ b/arch/ia64/kernel/ptrace.c @@ -945,6 +945,13 @@ access_uarea (struct task_struct *child, unsigned long addr, *data = (pt->cr_ipsr & IPSR_MASK); return 0; + case PT_AR_RSC: + if (write_access) + pt->ar_rsc = *data | (3 << 2); /* force PL3 */ + else + *data = pt->ar_rsc; + return 0; + case PT_AR_RNAT: urbs_end = ia64_get_user_rbs_end(child, pt, NULL); rnat_addr = (long) ia64_rse_rnat_addr((long *) @@ -996,9 +1003,6 @@ access_uarea (struct task_struct *child, unsigned long addr, case PT_AR_BSPSTORE: ptr = pt_reg_addr(pt, ar_bspstore); break; - case PT_AR_RSC: - ptr = pt_reg_addr(pt, ar_rsc); - break; case PT_AR_UNAT: ptr = pt_reg_addr(pt, ar_unat); break; @@ -1234,7 +1238,7 @@ ptrace_getregs (struct task_struct *child, struct pt_all_user_regs __user *ppr) static long ptrace_setregs (struct task_struct *child, struct pt_all_user_regs __user *ppr) { - unsigned long psr, ec, lc, rnat, bsp, cfm, nat_bits, val = 0; + unsigned long psr, rsc, ec, lc, rnat, bsp, cfm, nat_bits, val = 0; struct unw_frame_info info; struct switch_stack *sw; struct ia64_fpreg fpval; @@ -1267,7 +1271,7 @@ ptrace_setregs (struct task_struct *child, struct pt_all_user_regs __user *ppr) /* app regs */ retval |= __get_user(pt->ar_pfs, &ppr->ar[PT_AUR_PFS]); - retval |= __get_user(pt->ar_rsc, &ppr->ar[PT_AUR_RSC]); + retval |= __get_user(rsc, &ppr->ar[PT_AUR_RSC]); retval |= __get_user(pt->ar_bspstore, &ppr->ar[PT_AUR_BSPSTORE]); retval |= __get_user(pt->ar_unat, &ppr->ar[PT_AUR_UNAT]); retval |= __get_user(pt->ar_ccv, &ppr->ar[PT_AUR_CCV]); @@ -1365,6 +1369,7 @@ ptrace_setregs (struct task_struct *child, struct pt_all_user_regs __user *ppr) retval |= __get_user(nat_bits, &ppr->nat); retval |= access_uarea(child, PT_CR_IPSR, &psr, 1); + retval |= access_uarea(child, PT_AR_RSC, &rsc, 1); retval |= access_uarea(child, PT_AR_EC, &ec, 1); retval |= access_uarea(child, PT_AR_LC, &lc, 1); retval |= access_uarea(child, PT_AR_RNAT, &rnat, 1); diff --git a/arch/ia64/kernel/signal.c b/arch/ia64/kernel/signal.c index 499b7e5..edd9f07 100644 --- a/arch/ia64/kernel/signal.c +++ b/arch/ia64/kernel/signal.c @@ -94,7 +94,7 @@ sys_sigaltstack (const stack_t __user *uss, stack_t __user *uoss, long arg2, static long restore_sigcontext (struct sigcontext __user *sc, struct sigscratch *scr) { - unsigned long ip, flags, nat, um, cfm; + unsigned long ip, flags, nat, um, cfm, rsc; long err; /* Always make any pending restarted system calls return -EINTR */ @@ -106,7 +106,7 @@ restore_sigcontext (struct sigcontext __user *sc, struct sigscratch *scr) err |= __get_user(ip, &sc->sc_ip); /* instruction pointer */ err |= __get_user(cfm, &sc->sc_cfm); err |= __get_user(um, &sc->sc_um); /* user mask */ - err |= __get_user(scr->pt.ar_rsc, &sc->sc_ar_rsc); + err |= __get_user(rsc, &sc->sc_ar_rsc); err |= __get_user(scr->pt.ar_unat, &sc->sc_ar_unat); err |= __get_user(scr->pt.ar_fpsr, &sc->sc_ar_fpsr); err |= __get_user(scr->pt.ar_pfs, &sc->sc_ar_pfs); @@ -119,6 +119,7 @@ restore_sigcontext (struct sigcontext __user *sc, struct sigscratch *scr) err |= __copy_from_user(&scr->pt.r15, &sc->sc_gr[15], 8); /* r15 */ scr->pt.cr_ifs = cfm | (1UL << 63); + scr->pt.ar_rsc = rsc | (3 << 2); /* force PL3 */ /* establish new instruction pointer: */ scr->pt.cr_iip = ip & ~0x3UL; diff --git a/arch/ia64/kernel/smp.c b/arch/ia64/kernel/smp.c index 953095e..b49d4dd 100644 --- a/arch/ia64/kernel/smp.c +++ b/arch/ia64/kernel/smp.c @@ -269,7 +269,7 @@ smp_call_function_single (int cpuid, void (*func) (void *info), void *info, int int me = get_cpu(); /* prevent preemption and reschedule on another processor */ if (cpuid == me) { - printk("%s: trying to call self\n", __FUNCTION__); + printk(KERN_INFO "%s: trying to call self\n", __FUNCTION__); put_cpu(); return -EBUSY; } diff --git a/arch/ia64/kernel/uncached.c b/arch/ia64/kernel/uncached.c new file mode 100644 index 0000000..490dfc9 --- /dev/null +++ b/arch/ia64/kernel/uncached.c @@ -0,0 +1,246 @@ +/* + * Copyright (C) 2001-2005 Silicon Graphics, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of version 2 of the GNU General Public License + * as published by the Free Software Foundation. + * + * A simple uncached page allocator using the generic allocator. This + * allocator first utilizes the spare (spill) pages found in the EFI + * memmap and will then start converting cached pages to uncached ones + * at a granule at a time. Node awareness is implemented by having a + * pool of pages per node. + */ + +#include <linux/types.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/errno.h> +#include <linux/string.h> +#include <linux/slab.h> +#include <linux/efi.h> +#include <linux/genalloc.h> +#include <asm/page.h> +#include <asm/pal.h> +#include <asm/system.h> +#include <asm/pgtable.h> +#include <asm/atomic.h> +#include <asm/tlbflush.h> +#include <asm/sn/arch.h> + +#define DEBUG 0 + +#if DEBUG +#define dprintk printk +#else +#define dprintk(x...) do { } while (0) +#endif + +void __init efi_memmap_walk_uc (efi_freemem_callback_t callback); + +#define MAX_UNCACHED_GRANULES 5 +static int allocated_granules; + +struct gen_pool *uncached_pool[MAX_NUMNODES]; + + +static void uncached_ipi_visibility(void *data) +{ + int status; + + status = ia64_pal_prefetch_visibility(PAL_VISIBILITY_PHYSICAL); + if ((status != PAL_VISIBILITY_OK) && + (status != PAL_VISIBILITY_OK_REMOTE_NEEDED)) + printk(KERN_DEBUG "pal_prefetch_visibility() returns %i on " + "CPU %i\n", status, get_cpu()); +} + + +static void uncached_ipi_mc_drain(void *data) +{ + int status; + status = ia64_pal_mc_drain(); + if (status) + printk(KERN_WARNING "ia64_pal_mc_drain() failed with %i on " + "CPU %i\n", status, get_cpu()); +} + + +static unsigned long +uncached_get_new_chunk(struct gen_pool *poolp) +{ + struct page *page; + void *tmp; + int status, i; + unsigned long addr, node; + + if (allocated_granules >= MAX_UNCACHED_GRANULES) + return 0; + + node = poolp->private; + page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, + IA64_GRANULE_SHIFT-PAGE_SHIFT); + + dprintk(KERN_INFO "get_new_chunk page %p, addr %lx\n", + page, (unsigned long)(page-vmem_map) << PAGE_SHIFT); + + /* + * Do magic if no mem on local node! XXX + */ + if (!page) + return 0; + tmp = page_address(page); + + /* + * There's a small race here where it's possible for someone to + * access the page through /dev/mem halfway through the conversion + * to uncached - not sure it's really worth bothering about + */ + for (i = 0; i < (IA64_GRANULE_SIZE / PAGE_SIZE); i++) + SetPageUncached(&page[i]); + + flush_tlb_kernel_range(tmp, tmp + IA64_GRANULE_SIZE); + + status = ia64_pal_prefetch_visibility(PAL_VISIBILITY_PHYSICAL); + + dprintk(KERN_INFO "pal_prefetch_visibility() returns %i on cpu %i\n", + status, get_cpu()); + + if (!status) { + status = smp_call_function(uncached_ipi_visibility, NULL, 0, 1); + if (status) + printk(KERN_WARNING "smp_call_function failed for " + "uncached_ipi_visibility! (%i)\n", status); + } + + if (ia64_platform_is("sn2")) + sn_flush_all_caches((unsigned long)tmp, IA64_GRANULE_SIZE); + else + flush_icache_range((unsigned long)tmp, + (unsigned long)tmp+IA64_GRANULE_SIZE); + + ia64_pal_mc_drain(); + status = smp_call_function(uncached_ipi_mc_drain, NULL, 0, 1); + if (status) + printk(KERN_WARNING "smp_call_function failed for " + "uncached_ipi_mc_drain! (%i)\n", status); + + addr = (unsigned long)tmp - PAGE_OFFSET + __IA64_UNCACHED_OFFSET; + + allocated_granules++; + return addr; +} + + +/* + * uncached_alloc_page + * + * Allocate 1 uncached page. Allocates on the requested node. If no + * uncached pages are available on the requested node, roundrobin starting + * with higher nodes. + */ +unsigned long +uncached_alloc_page(int nid) +{ + unsigned long maddr; + + maddr = gen_pool_alloc(uncached_pool[nid], PAGE_SIZE); + + dprintk(KERN_DEBUG "uncached_alloc_page returns %lx on node %i\n", + maddr, nid); + + /* + * If no memory is availble on our local node, try the + * remaining nodes in the system. + */ + if (!maddr) { + int i; + + for (i = MAX_NUMNODES - 1; i >= 0; i--) { + if (i == nid || !node_online(i)) + continue; + maddr = gen_pool_alloc(uncached_pool[i], PAGE_SIZE); + dprintk(KERN_DEBUG "uncached_alloc_page alternate search " + "returns %lx on node %i\n", maddr, i); + if (maddr) { + break; + } + } + } + + return maddr; +} +EXPORT_SYMBOL(uncached_alloc_page); + + +/* + * uncached_free_page + * + * Free a single uncached page. + */ +void +uncached_free_page(unsigned long maddr) +{ + int node; + + node = nasid_to_cnodeid(NASID_GET(maddr)); + + dprintk(KERN_DEBUG "uncached_free_page(%lx) on node %i\n", maddr, node); + + if ((maddr & (0XFUL << 60)) != __IA64_UNCACHED_OFFSET) + panic("uncached_free_page invalid address %lx\n", maddr); + + gen_pool_free(uncached_pool[node], maddr, PAGE_SIZE); +} +EXPORT_SYMBOL(uncached_free_page); + + +/* + * uncached_build_memmap, + * + * Called at boot time to build a map of pages that can be used for + * memory special operations. + */ +static int __init +uncached_build_memmap(unsigned long start, unsigned long end, void *arg) +{ + long length; + unsigned long vstart, vend; + int node; + + length = end - start; + vstart = start + __IA64_UNCACHED_OFFSET; + vend = end + __IA64_UNCACHED_OFFSET; + + dprintk(KERN_ERR "uncached_build_memmap(%lx %lx)\n", start, end); + + memset((char *)vstart, 0, length); + + node = nasid_to_cnodeid(NASID_GET(start)); + + for (; vstart < vend ; vstart += PAGE_SIZE) { + dprintk(KERN_INFO "sticking %lx into the pool!\n", vstart); + gen_pool_free(uncached_pool[node], vstart, PAGE_SIZE); + } + + return 0; +} + + +static int __init uncached_init(void) { + int i; + + for (i = 0; i < MAX_NUMNODES; i++) { + if (!node_online(i)) + continue; + uncached_pool[i] = gen_pool_create(0, IA64_GRANULE_SHIFT, + &uncached_get_new_chunk, i); + } + + efi_memmap_walk_uc(uncached_build_memmap); + + return 0; +} + +__initcall(uncached_init); |