From 858918b77b29d0e9ce7f524d1b57d602d85f5d64 Mon Sep 17 00:00:00 2001 From: Matt Fleming Date: Sun, 7 Feb 2010 12:40:36 +0000 Subject: sh: Optimise FDE/CIE lookup by using red-black trees Now that the DWARF unwinder is being used to provide perf callstacks unwinding speed is an issue. It is no longer being used in exceptional circumstances where we don't care about runtime performance, e.g. when panicing, so it makes sense improve performance is possible. With this patch I saw a 42% improvement in unwind time when calling return_address(1). Greater improvements will be seen as the number of levels unwound increases as each unwind is now cheaper. Note that insertion time has doubled but that's just the price we pay for keeping the trees balanced. However, this is a one-time cost for kernel boot/module load and so the improvements in lookup time dominate the extra time we spend keeping the trees balanced. Signed-off-by: Matt Fleming Signed-off-by: Paul Mundt --- arch/sh/kernel/dwarf.c | 174 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 120 insertions(+), 54 deletions(-) (limited to 'arch/sh/kernel') diff --git a/arch/sh/kernel/dwarf.c b/arch/sh/kernel/dwarf.c index e511680..bd1c497 100644 --- a/arch/sh/kernel/dwarf.c +++ b/arch/sh/kernel/dwarf.c @@ -39,10 +39,10 @@ static mempool_t *dwarf_frame_pool; static struct kmem_cache *dwarf_reg_cachep; static mempool_t *dwarf_reg_pool; -static LIST_HEAD(dwarf_cie_list); +static struct rb_root cie_root; static DEFINE_SPINLOCK(dwarf_cie_lock); -static LIST_HEAD(dwarf_fde_list); +static struct rb_root fde_root; static DEFINE_SPINLOCK(dwarf_fde_lock); static struct dwarf_cie *cached_cie; @@ -301,7 +301,8 @@ static inline int dwarf_entry_len(char *addr, unsigned long *len) */ static struct dwarf_cie *dwarf_lookup_cie(unsigned long cie_ptr) { - struct dwarf_cie *cie; + struct rb_node **rb_node = &cie_root.rb_node; + struct dwarf_cie *cie = NULL; unsigned long flags; spin_lock_irqsave(&dwarf_cie_lock, flags); @@ -315,16 +316,24 @@ static struct dwarf_cie *dwarf_lookup_cie(unsigned long cie_ptr) goto out; } - list_for_each_entry(cie, &dwarf_cie_list, link) { - if (cie->cie_pointer == cie_ptr) { - cached_cie = cie; - break; + while (*rb_node) { + struct dwarf_cie *cie_tmp; + + cie_tmp = rb_entry(*rb_node, struct dwarf_cie, node); + BUG_ON(!cie_tmp); + + if (cie_ptr == cie_tmp->cie_pointer) { + cie = cie_tmp; + cached_cie = cie_tmp; + goto out; + } else { + if (cie_ptr < cie_tmp->cie_pointer) + rb_node = &(*rb_node)->rb_left; + else + rb_node = &(*rb_node)->rb_right; } } - /* Couldn't find the entry in the list. */ - if (&cie->link == &dwarf_cie_list) - cie = NULL; out: spin_unlock_irqrestore(&dwarf_cie_lock, flags); return cie; @@ -336,25 +345,34 @@ out: */ struct dwarf_fde *dwarf_lookup_fde(unsigned long pc) { - struct dwarf_fde *fde; + struct rb_node **rb_node = &fde_root.rb_node; + struct dwarf_fde *fde = NULL; unsigned long flags; spin_lock_irqsave(&dwarf_fde_lock, flags); - list_for_each_entry(fde, &dwarf_fde_list, link) { - unsigned long start, end; + while (*rb_node) { + struct dwarf_fde *fde_tmp; + unsigned long tmp_start, tmp_end; - start = fde->initial_location; - end = fde->initial_location + fde->address_range; + fde_tmp = rb_entry(*rb_node, struct dwarf_fde, node); + BUG_ON(!fde_tmp); - if (pc >= start && pc < end) - break; - } + tmp_start = fde_tmp->initial_location; + tmp_end = fde_tmp->initial_location + fde_tmp->address_range; - /* Couldn't find the entry in the list. */ - if (&fde->link == &dwarf_fde_list) - fde = NULL; + if (pc < tmp_start) { + rb_node = &(*rb_node)->rb_left; + } else { + if (pc < tmp_end) { + fde = fde_tmp; + goto out; + } else + rb_node = &(*rb_node)->rb_right; + } + } +out: spin_unlock_irqrestore(&dwarf_fde_lock, flags); return fde; @@ -552,8 +570,8 @@ extern void ret_from_irq(void); * on the callstack. Each of the lower (older) stack frames are * linked via the "prev" member. */ -struct dwarf_frame * dwarf_unwind_stack(unsigned long pc, - struct dwarf_frame *prev) +struct dwarf_frame *dwarf_unwind_stack(unsigned long pc, + struct dwarf_frame *prev) { struct dwarf_frame *frame; struct dwarf_cie *cie; @@ -708,6 +726,8 @@ bail: static int dwarf_parse_cie(void *entry, void *p, unsigned long len, unsigned char *end, struct module *mod) { + struct rb_node **rb_node = &cie_root.rb_node; + struct rb_node *parent; struct dwarf_cie *cie; unsigned long flags; int count; @@ -802,11 +822,30 @@ static int dwarf_parse_cie(void *entry, void *p, unsigned long len, cie->initial_instructions = p; cie->instructions_end = end; - cie->mod = mod; - /* Add to list */ spin_lock_irqsave(&dwarf_cie_lock, flags); - list_add_tail(&cie->link, &dwarf_cie_list); + + while (*rb_node) { + struct dwarf_cie *cie_tmp; + + cie_tmp = rb_entry(*rb_node, struct dwarf_cie, node); + + parent = *rb_node; + + if (cie->cie_pointer < cie_tmp->cie_pointer) + rb_node = &parent->rb_left; + else if (cie->cie_pointer >= cie_tmp->cie_pointer) + rb_node = &parent->rb_right; + else + WARN_ON(1); + } + + rb_link_node(&cie->node, parent, rb_node); + rb_insert_color(&cie->node, &cie_root); + + if (mod != NULL) + list_add_tail(&cie->link, &mod->arch.cie_list); + spin_unlock_irqrestore(&dwarf_cie_lock, flags); return 0; @@ -816,6 +855,8 @@ static int dwarf_parse_fde(void *entry, u32 entry_type, void *start, unsigned long len, unsigned char *end, struct module *mod) { + struct rb_node **rb_node = &fde_root.rb_node; + struct rb_node *parent; struct dwarf_fde *fde; struct dwarf_cie *cie; unsigned long flags; @@ -863,11 +904,38 @@ static int dwarf_parse_fde(void *entry, u32 entry_type, fde->instructions = p; fde->end = end; - fde->mod = mod; - /* Add to list. */ spin_lock_irqsave(&dwarf_fde_lock, flags); - list_add_tail(&fde->link, &dwarf_fde_list); + + while (*rb_node) { + struct dwarf_fde *fde_tmp; + unsigned long tmp_start, tmp_end; + unsigned long start, end; + + fde_tmp = rb_entry(*rb_node, struct dwarf_fde, node); + + start = fde->initial_location; + end = fde->initial_location + fde->address_range; + + tmp_start = fde_tmp->initial_location; + tmp_end = fde_tmp->initial_location + fde_tmp->address_range; + + parent = *rb_node; + + if (start < tmp_start) + rb_node = &parent->rb_left; + else if (start >= tmp_end) + rb_node = &parent->rb_right; + else + WARN_ON(1); + } + + rb_link_node(&fde->node, parent, rb_node); + rb_insert_color(&fde->node, &fde_root); + + if (mod != NULL) + list_add_tail(&fde->link, &mod->arch.fde_list); + spin_unlock_irqrestore(&dwarf_fde_lock, flags); return 0; @@ -912,19 +980,29 @@ static struct unwinder dwarf_unwinder = { static void dwarf_unwinder_cleanup(void) { - struct dwarf_cie *cie, *cie_tmp; - struct dwarf_fde *fde, *fde_tmp; + struct rb_node **fde_rb_node = &fde_root.rb_node; + struct rb_node **cie_rb_node = &cie_root.rb_node; /* * Deallocate all the memory allocated for the DWARF unwinder. * Traverse all the FDE/CIE lists and remove and free all the * memory associated with those data structures. */ - list_for_each_entry_safe(cie, cie_tmp, &dwarf_cie_list, link) - kfree(cie); + while (*fde_rb_node) { + struct dwarf_fde *fde; - list_for_each_entry_safe(fde, fde_tmp, &dwarf_fde_list, link) + fde = rb_entry(*fde_rb_node, struct dwarf_fde, node); + rb_erase(*fde_rb_node, &fde_root); kfree(fde); + } + + while (*cie_rb_node) { + struct dwarf_cie *cie; + + cie = rb_entry(*cie_rb_node, struct dwarf_cie, node); + rb_erase(*cie_rb_node, &cie_root); + kfree(cie); + } kmem_cache_destroy(dwarf_reg_cachep); kmem_cache_destroy(dwarf_frame_cachep); @@ -1024,6 +1102,8 @@ int module_dwarf_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, /* Did we find the .eh_frame section? */ if (i != hdr->e_shnum) { + INIT_LIST_HEAD(&me->arch.cie_list); + INIT_LIST_HEAD(&me->arch.fde_list); err = dwarf_parse_section((char *)start, (char *)end, me); if (err) { printk(KERN_WARNING "%s: failed to parse DWARF info\n", @@ -1044,38 +1124,26 @@ int module_dwarf_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, */ void module_dwarf_cleanup(struct module *mod) { - struct dwarf_fde *fde; - struct dwarf_cie *cie; + struct dwarf_fde *fde, *ftmp; + struct dwarf_cie *cie, *ctmp; unsigned long flags; spin_lock_irqsave(&dwarf_cie_lock, flags); -again_cie: - list_for_each_entry(cie, &dwarf_cie_list, link) { - if (cie->mod == mod) - break; - } - - if (&cie->link != &dwarf_cie_list) { + list_for_each_entry_safe(cie, ctmp, &mod->arch.cie_list, link) { list_del(&cie->link); + rb_erase(&cie->node, &cie_root); kfree(cie); - goto again_cie; } spin_unlock_irqrestore(&dwarf_cie_lock, flags); spin_lock_irqsave(&dwarf_fde_lock, flags); -again_fde: - list_for_each_entry(fde, &dwarf_fde_list, link) { - if (fde->mod == mod) - break; - } - - if (&fde->link != &dwarf_fde_list) { + list_for_each_entry_safe(fde, ftmp, &mod->arch.fde_list, link) { list_del(&fde->link); + rb_erase(&fde->node, &fde_root); kfree(fde); - goto again_fde; } spin_unlock_irqrestore(&dwarf_fde_lock, flags); @@ -1094,8 +1162,6 @@ again_fde: static int __init dwarf_unwinder_init(void) { int err; - INIT_LIST_HEAD(&dwarf_cie_list); - INIT_LIST_HEAD(&dwarf_fde_list); dwarf_frame_cachep = kmem_cache_create("dwarf_frames", sizeof(struct dwarf_frame), 0, -- cgit v1.1