kmem_cache_init()的疑问

aweii

内核版本：v2.6.24
slub内存管理模块
假定环境：SMP +NUMA

问题：kmem_cache_init()对通用缓存进行初始化，其中调用了create_kmalloc_cache->kmem_cache_open
->alloc_kmem_cache_cpus->alloc_kmem_cache_cpu,对缓存的cpu_slab[]进行了初始化，但是后来又
调用了register_cpu_notifier(&slab_notifier); 这会使系统调用其中包含的函数slab_cpuup_callback（），
这个函数对slab_caches链表中的缓存(包括以上的通用缓存)重新调用alloc_kmem_cache_cpu（），这不是重复了吗？
这个问题上次提过，不过是针对2.6.11版本的，一直没有得到答复，现在在2.6.24中有同样的疑问。望大侠们不吝指教。


代码摘录如下：
//---------------------------------------------linux/mm/slub.c -------------------------------
static struct notifier_block __cpuinitdata slab_notifier =
        { &slab_cpuup_callback, NULL, 0 };



void __init kmem_cache_init(void)
{
        int i;
        int caches = 0;

        init_alloc_cpu();

#ifdef CONFIG_NUMA
        /*
         * Must first have the slab cache available for the allocations of the
         * struct kmem_cache_node's. There is special bootstrap code in
         * kmem_cache_open for slab_state == DOWN.
         */
        create_kmalloc_cache(&kmalloc_caches[0], "kmem_cache_node",
                sizeof(struct kmem_cache_node), GFP_KERNEL);
        kmalloc_caches[0].refcount = -1;
        caches++;

        hotplug_memory_notifier(slab_memory_callback, 1);
#endif

        /* Able to allocate the per node structures */
        slab_state = PARTIAL;

        /* Caches that are not of the two-to-the-power-of size */
        if (KMALLOC_MIN_SIZE <= 64) {
                create_kmalloc_cache(&kmalloc_caches[1],
                                "kmalloc-96", 96, GFP_KERNEL);
                caches++;
        }
        if (KMALLOC_MIN_SIZE <= 12 {
                create_kmalloc_cache(&kmalloc_caches[2],
                                "kmalloc-192", 192, GFP_KERNEL);
                caches++;
        }

        for (i = KMALLOC_SHIFT_LOW; i < PAGE_SHIFT; i++) {
                create_kmalloc_cache(&kmalloc_caches[i],
                        "kmalloc", 1 << i, GFP_KERNEL);
                caches++;
        }


        /*
         * Patch up the size_index table if we have strange large alignment
         * requirements for the kmalloc array. This is only the case for
         * mips it seems. The standard arches will not generate any code here.
         *
         * Largest permitted alignment is 256 bytes due to the way we
         * handle the index determination for the smaller caches.
         *
         * Make sure that nothing crazy happens if someone starts tinkering
         * around with ARCH_KMALLOC_MINALIGN
         */
        BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
                (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));

        for (i = 8; i < KMALLOC_MIN_SIZE; i +=
                size_index[(i - 1) / 8] = KMALLOC_SHIFT_LOW;

        slab_state = UP;

        /* Provide the correct kmalloc names now that the caches are up */
        for (i = KMALLOC_SHIFT_LOW; i < PAGE_SHIFT; i++)
                kmalloc_caches[i]. name =
                        kasprintf(GFP_KERNEL, "kmalloc-%d", 1 << i);

#ifdef CONFIG_SMP
        register_cpu_notifier(&slab_notifier);
        kmem_size = offsetof(struct kmem_cache, cpu_slab) +
                                nr_cpu_ids * sizeof(struct kmem_cache_cpu *);
#else
        kmem_size = sizeof(struct kmem_cache);
#endif


        printk(KERN_INFO "SLUB: Genslabs=%d, HWalign=%d, Order=%d-%d, MinObjects=%d,"
                " CPUs=%d, Nodes=%d\n",
                caches, cache_line_size(),
                slub_min_order, slub_max_order, slub_min_objects,
                nr_cpu_ids, nr_node_ids);
}

瀚海书香

回复 1# aweii
你说的这个代码在v2.6.34中已经解决了。
git log ff12059ed14b0773d7bbef86f98218ada6c20770 -p
commit ff12059ed14b0773d7bbef86f98218ada6c20770
Author: Christoph Lameter <cl@linux-foundation.org>
Date:   Fri Dec 18 16:26:22 2009 -0600

    SLUB: this_cpu: Remove slub kmem_cache fields
   
    Remove the fields in struct kmem_cache_cpu that were used to cache data from
    struct kmem_cache when they were in different cachelines. The cacheline that
    holds the per cpu array pointer now also holds these values. We can cut down
    the struct kmem_cache_cpu size to almost half.
   
    The get_freepointer() and set_freepointer() functions that used to be only
    intended for the slow path now are also useful for the hot path since access
    to the size field does not require accessing an additional cacheline anymore.
    This results in consistent use of functions for setting the freepointer of
    objects throughout SLUB.
   
    Also we initialize all possible kmem_cache_cpu structures when a slab is
    created. No need to initialize them when a processor or node comes online.
   
    Signed-off-by: Christoph Lameter <cl@linux-foundation.org>
    Signed-off-by: Pekka Enberg <penberg@cs.helsinki.fi>

   

firkraag

In version 2.6.24, I saw function 'alloc_kmem_cache_cpus()'  would  just work on  the online cpu, like this:
static int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags)
{
        int cpu;

        for_each_online_cpu(cpu) {
                struct kmem_cache_cpu *c = get_cpu_slab(s, cpu);

                if (c)
                        continue;

                c = alloc_kmem_cache_cpu(s, cpu, flags);
                if (!c) {
                        free_kmem_cache_cpus(s);
                        return 0;
                }
                s->cpu_slab[cpu] = c;
        }
        return 1;
}

Function 'alloc_kmem_cache_cpu' in called function slab_cpuup_callbackwill be called when some CPUs is coming up as they was off when initialized the cpu_slab[] array.

I guess this mechanism is for cpu state changing.

As 瀚海书香 said:

Also we initialize all possible kmem_cache_cpu structures when a slab is
created. No need to initialize them when a processor or node comes online.

'alloc_kmem_cache_cpus' has been  changed in version 2.6.34.

aweii

两位说的都很中肯，我茅塞顿开。非常感谢！