[linux内存管理] 第044篇 per-CPU基础知识以及per-CPU分配器的初始化

struct pcpu_group_info {
	int			nr_units;	/* aligned # of units */
	unsigned long		base_offset;	/* base address offset */
	unsigned int		*cpu_map;	/* unit->cpu map, empty
						 * entries contain NR_CPUS */
};

struct pcpu_alloc_info {
	size_t			static_size;
	size_t			reserved_size;
	size_t			dyn_size;
	size_t			unit_size;
	size_t			atom_size;
	size_t			alloc_size;
	size_t			__ai_size;	/* internal, don't use */
	int			nr_groups;	/* 0 if grouping unnecessary */
	struct pcpu_group_info	groups[];
};

struct pcpu_chunk {
#ifdef CONFIG_PERCPU_STATS
	int			nr_alloc;	/* # of allocations */
	size_t			max_alloc_size; /* largest allocation size */
#endif

	struct list_head	list;		/* linked to pcpu_slot lists */
	int			free_bytes;	/* free bytes in the chunk */
	struct pcpu_block_md	chunk_md;
	void			*base_addr;	/* base address of this chunk */

	unsigned long		*alloc_map;	/* allocation map */
	unsigned long		*bound_map;	/* boundary map */
	struct pcpu_block_md	*md_blocks;	/* metadata blocks */

	void			*data;		/* chunk data */
	bool			immutable;	/* no [de]population allowed */
	bool			isolated;	/* isolated from active chunk
						   slots */
	int			start_offset;	/* the overlap with the previous
						   region to have a page aligned
						   base_addr */
	int			end_offset;	/* additional area required to
						   have the region end page
						   aligned */
#ifdef CONFIG_MEMCG_KMEM
	struct obj_cgroup	**obj_cgroups;	/* vector of object cgroups */
#endif

	int			nr_pages;	/* # of pages served by this chunk */
	int			nr_populated;	/* # of populated pages */
	int                     nr_empty_pop_pages; /* # of empty populated pages */
	unsigned long		populated[];	/* populated bitmap */
};

struct pcpu_block_md {
	int			scan_hint;	/* scan hint for block */
	int			scan_hint_start; /* block relative starting
						    position of the scan hint */
	int                     contig_hint;    /* contig hint for block */
	int                     contig_hint_start; /* block relative starting
						      position of the contig hint */
	int                     left_free;      /* size of free space along
						   the left side of the block */
	int                     right_free;     /* size of free space along
						   the right side of the block */
	int                     first_free;     /* block position of first free */
	int			nr_bits;	/* total bits responsible for */
};

asmlinkage __visible void __init __no_sanitize_address start_kernel(void)
{
    //...
	setup_nr_cpu_ids();
	setup_per_cpu_areas();
    //...
}

void __init setup_nr_cpu_ids(void)
{
	set_nr_cpu_ids(find_last_bit(cpumask_bits(cpu_possible_mask), NR_CPUS) + 1);
}

void __init setup_per_cpu_areas(void)
{
	unsigned long delta;
	unsigned int cpu;
	int rc;

	/*
	 * Always reserve area for module percpu variables.  That's
	 * what the legacy allocator did.
	 */
	rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, PERCPU_DYNAMIC_RESERVE,
				    PAGE_SIZE, NULL, NULL);
	if (rc < 0)
		panic("Failed to initialize percpu areas.");

	delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
	for_each_possible_cpu(cpu)
		__per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
}

int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size,
				  size_t atom_size,
				  pcpu_fc_cpu_distance_fn_t cpu_distance_fn,
				  pcpu_fc_cpu_to_node_fn_t cpu_to_nd_fn)
{
	void *base = (void *)ULONG_MAX;
	void **areas = NULL;
	struct pcpu_alloc_info *ai;
	size_t size_sum, areas_size;
	unsigned long max_distance;
	int group, i, highest_group, rc = 0;

	ai = pcpu_build_alloc_info(reserved_size, dyn_size, atom_size,
				   cpu_distance_fn);
	if (IS_ERR(ai))
		return PTR_ERR(ai);

	size_sum = ai->static_size + ai->reserved_size + ai->dyn_size;
	areas_size = PFN_ALIGN(ai->nr_groups * sizeof(void *));

	areas = memblock_alloc(areas_size, SMP_CACHE_BYTES);
	if (!areas) {
		rc = -ENOMEM;
		goto out_free;
	}

	/* allocate, copy and determine base address & max_distance */
	highest_group = 0;
	for (group = 0; group < ai->nr_groups; group++) {
		struct pcpu_group_info *gi = &ai->groups[group];
		unsigned int cpu = NR_CPUS;
		void *ptr;

		for (i = 0; i < gi->nr_units && cpu == NR_CPUS; i++)
			cpu = gi->cpu_map[i];
		BUG_ON(cpu == NR_CPUS);

		/* allocate space for the whole group */
		ptr = pcpu_fc_alloc(cpu, gi->nr_units * ai->unit_size, atom_size, cpu_to_nd_fn);
		if (!ptr) {
			rc = -ENOMEM;
			goto out_free_areas;
		}
		/* kmemleak tracks the percpu allocations separately */
		kmemleak_ignore_phys(__pa(ptr));
		areas[group] = ptr;

		base = min(ptr, base);
		if (ptr > areas[highest_group])
			highest_group = group;
	}
	max_distance = areas[highest_group] - base;
	max_distance += ai->unit_size * ai->groups[highest_group].nr_units;

	/* warn if maximum distance is further than 75% of vmalloc space */
	if (max_distance > VMALLOC_TOTAL * 3 / 4) {
		pr_warn("max_distance=0x%lx too large for vmalloc space 0x%lx\n",
				max_distance, VMALLOC_TOTAL);
#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
		/* and fail if we have fallback */
		rc = -EINVAL;
		goto out_free_areas;
#endif
	}

	/*
	 * Copy data and free unused parts.  This should happen after all
	 * allocations are complete; otherwise, we may end up with
	 * overlapping groups.
	 */
	for (group = 0; group < ai->nr_groups; group++) {
		struct pcpu_group_info *gi = &ai->groups[group];
		void *ptr = areas[group];

		for (i = 0; i < gi->nr_units; i++, ptr += ai->unit_size) {
			if (gi->cpu_map[i] == NR_CPUS) {
				/* unused unit, free whole */
				pcpu_fc_free(ptr, ai->unit_size);
				continue;
			}
			/* copy and return the unused part */
			memcpy(ptr, __per_cpu_load, ai->static_size);
			pcpu_fc_free(ptr + size_sum, ai->unit_size - size_sum);
		}
	}

	/* base address is now known, determine group base offsets */
	for (group = 0; group < ai->nr_groups; group++) {
		ai->groups[group].base_offset = areas[group] - base;
	}

	pr_info("Embedded %zu pages/cpu s%zu r%zu d%zu u%zu\n",
		PFN_DOWN(size_sum), ai->static_size, ai->reserved_size,
		ai->dyn_size, ai->unit_size);

	pcpu_setup_first_chunk(ai, base);
	goto out_free;

out_free_areas:
	for (group = 0; group < ai->nr_groups; group++)
		if (areas[group])
			pcpu_fc_free(areas[group],
				ai->groups[group].nr_units * ai->unit_size);
out_free:
	pcpu_free_alloc_info(ai);
	if (areas)
		memblock_free(areas, areas_size);
	return rc;
}

ai = pcpu_build_alloc_info(reserved_size, dyn_size, atom_size, cpu_distance_fn);

static struct pcpu_alloc_info * __init __flatten pcpu_build_alloc_info(
				size_t reserved_size, size_t dyn_size,
				size_t atom_size,
				pcpu_fc_cpu_distance_fn_t cpu_distance_fn)
{
	static int group_map[NR_CPUS] __initdata;
	static int group_cnt[NR_CPUS] __initdata;
	static struct cpumask mask __initdata;
    // 静态变量总大小，从 __per_cpu_start ~ __per_cpu_end
	const size_t static_size = __per_cpu_end - __per_cpu_start;
	int nr_groups = 1, nr_units = 0;
	size_t size_sum, min_unit_size, alloc_size;
	int upa, max_upa, best_upa;	/* units_per_alloc */
	int last_allocs, group, unit;
	unsigned int cpu, tcpu;
	struct pcpu_alloc_info *ai;
	unsigned int *cpu_map;

	/* this function may be called multiple times */
	memset(group_map, 0, sizeof(group_map));
	memset(group_cnt, 0, sizeof(group_cnt));
	cpumask_clear(&mask);
    // //计算每个cpu所占有的percpu内存空间，
    //  包括: 静态空间 + 预留空间 + 动态空间
	/* calculate size_sum and ensure dyn_size is enough for early alloc */
	size_sum = PFN_ALIGN(static_size + reserved_size +
			    max_t(size_t, dyn_size, PERCPU_DYNAMIC_EARLY_SIZE));
    // 实际的动态空间，相当于上面size_sum是按照页对齐的，三者和多出来的部分最终都算到了dyn_size中
	dyn_size = size_sum - static_size - reserved_size;

	/*
	 * Determine min_unit_size, alloc_size and max_upa such that
	 * alloc_size is multiple of atom_size and is the smallest
	 * which can accommodate 4k aligned segments which are equal to
	 * or larger than min_unit_size.
	 */
    //理论上每个cpu unit的大小是按照size_sum，
    //但系统设计要求每个cpu unit的大小，不能小于PCPU_MIN_UNIT_SIZE (32KB)
	min_unit_size = max_t(size_t, size_sum, PCPU_MIN_UNIT_SIZE);

	/* determine the maximum # of units that can fit in an allocation */
    // 每次实际从内存分配器获取的连续内存块大小，按 atom_size 向上取整
	alloc_size = roundup(min_unit_size, atom_size);
	upa = alloc_size / min_unit_size;
	while (alloc_size % upa || (offset_in_page(alloc_size / upa)))
		upa--;
	max_upa = upa;

	cpumask_copy(&mask, cpu_possible_mask);

	/* group cpus according to their proximity */
    // 该循环将所有可能的 CPU 按距离函数分组：同组的 CPU 之间距离为 LOCAL_DISTANCE
	for (group = 0; !cpumask_empty(&mask); group++) {
		/* pop the group's first cpu */
		cpu = cpumask_first(&mask);
		group_map[cpu] = group;
		group_cnt[group]++;
		cpumask_clear_cpu(cpu, &mask);

		for_each_cpu(tcpu, &mask) {
			if (!cpu_distance_fn ||
			    (cpu_distance_fn(cpu, tcpu) == LOCAL_DISTANCE &&
			     cpu_distance_fn(tcpu, cpu) == LOCAL_DISTANCE)) {
				group_map[tcpu] = group;
				group_cnt[group]++;
				cpumask_clear_cpu(tcpu, &mask);
			}
		}
	}
	nr_groups = group;

	/*
	 * Wasted space is caused by a ratio imbalance of upa to group_cnt.
	 * Expand the unit_size until we use >= 75% of the units allocated.
	 * Related to atom_size, which could be much larger than the unit_size.
	 */
	last_allocs = INT_MAX;
	best_upa = 0;
	for (upa = max_upa; upa; upa--) {
		int allocs = 0, wasted = 0;

		if (alloc_size % upa || (offset_in_page(alloc_size / upa)))
			continue;

		for (group = 0; group < nr_groups; group++) {
			int this_allocs = DIV_ROUND_UP(group_cnt[group], upa);
			allocs += this_allocs;
			wasted += this_allocs * upa - group_cnt[group];
		}

		/*
		 * Don't accept if wastage is over 1/3.  The
		 * greater-than comparison ensures upa==1 always
		 * passes the following check.
		 */
		if (wasted > num_possible_cpus() / 3)
			continue;

		/* and then don't consume more memory */
		if (allocs > last_allocs)
			break;
		last_allocs = allocs;
		best_upa = upa;
	}
	BUG_ON(!best_upa);
	upa = best_upa;

	/* allocate and fill alloc_info */
    // 计算总单元数并分配 alloc_info 结构
	for (group = 0; group < nr_groups; group++)
		nr_units += roundup(group_cnt[group], upa);

	ai = pcpu_alloc_alloc_info(nr_groups, nr_units);
	if (!ai)
		return ERR_PTR(-ENOMEM);
	cpu_map = ai->groups[0].cpu_map;

	for (group = 0; group < nr_groups; group++) {
		ai->groups[group].cpu_map = cpu_map;
		cpu_map += roundup(group_cnt[group], upa);
	}
    // 填充 alloc_info 的基本字段
	ai->static_size = static_size;
	ai->reserved_size = reserved_size;
	ai->dyn_size = dyn_size;
	ai->unit_size = alloc_size / upa;
	ai->atom_size = atom_size;
	ai->alloc_size = alloc_size;
    // 填充每个组的详细信息
	for (group = 0, unit = 0; group < nr_groups; group++) {
		struct pcpu_group_info *gi = &ai->groups[group];

		/*
		 * Initialize base_offset as if all groups are located
		 * back-to-back.  The caller should update this to
		 * reflect actual allocation.
		 */
		gi->base_offset = unit * ai->unit_size;

		for_each_possible_cpu(cpu)
			if (group_map[cpu] == group)
				gi->cpu_map[gi->nr_units++] = cpu;
		gi->nr_units = roundup(gi->nr_units, upa);
		unit += gi->nr_units;
	}
	BUG_ON(unit != nr_units);

	return ai;
}

size_sum = ai->static_size + ai->reserved_size + ai->dyn_size;
areas_size = PFN_ALIGN(ai->nr_groups * sizeof(void *));
areas = memblock_alloc(areas_size, SMP_CACHE_BYTES);

for (group = 0; group < ai->nr_groups; group++) {
    ...
    ptr = pcpu_fc_alloc(cpu, gi->nr_units * ai->unit_size, atom_size, cpu_to_nd_fn);
    areas[group] = ptr;
    base = min(ptr, base);
    ...
}

for (group = 0; group < ai->nr_groups; group++) {
    for (i = 0; i < gi->nr_units; i++, ptr += ai->unit_size) {
        if (gi->cpu_map[i] == NR_CPUS) {
            pcpu_fc_free(ptr, ai->unit_size);  // 未使用的单元（空洞），整块释放
            continue;
        }
        memcpy(ptr, __per_cpu_load, ai->static_size);          // 复制内核静态数据
        pcpu_fc_free(ptr + size_sum, ai->unit_size - size_sum); // 释放单元尾部未使用的填充部分
    }
}

void __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
                                   void *base_addr)

size_t size_sum = ai->static_size + ai->reserved_size + ai->dyn_size;
// ... 其他变量
#define PCPU_SETUP_BUG_ON(cond) ...

alloc_size = ai->nr_groups * sizeof(group_offsets[0]);
group_offsets = memblock_alloc(alloc_size, SMP_CACHE_BYTES);
...
group_sizes = memblock_alloc(...);
unit_map = memblock_alloc(nr_cpu_ids * sizeof(unit_map[0]), ...);
unit_off = memblock_alloc(nr_cpu_ids * sizeof(unit_off[0]), ...);

for (cpu = 0; cpu < nr_cpu_ids; cpu++)
    unit_map[cpu] = UINT_MAX;
pcpu_low_unit_cpu = pcpu_high_unit_cpu = NR_CPUS;

for (group = 0, unit = 0; group < ai->nr_groups; group++, unit += i) {
    const struct pcpu_group_info *gi = &ai->groups[group];
    group_offsets[group] = gi->base_offset;
    group_sizes[group] = gi->nr_units * ai->unit_size;

    for (i = 0; i < gi->nr_units; i++) {
        cpu = gi->cpu_map[i];
        if (cpu == NR_CPUS)
            continue;   // 空洞单元（对齐预留），不映射到真实 CPU

        // 检查 CPU 合法性
        PCPU_SETUP_BUG_ON(cpu >= nr_cpu_ids);
        PCPU_SETUP_BUG_ON(!cpu_possible(cpu));
        PCPU_SETUP_BUG_ON(unit_map[cpu] != UINT_MAX); // 不能重复映射

        unit_map[cpu] = unit + i;                     // 全局单元编号
        unit_off[cpu] = gi->base_offset + i * ai->unit_size; // 单元偏移

        // 记录最低/最高偏移的 CPU
        if (pcpu_low_unit_cpu == NR_CPUS ||
            unit_off[cpu] < unit_off[pcpu_low_unit_cpu])
            pcpu_low_unit_cpu = cpu;
        if (pcpu_high_unit_cpu == NR_CPUS ||
            unit_off[cpu] > unit_off[pcpu_high_unit_cpu])
            pcpu_high_unit_cpu = cpu;
    }
}

for_each_possible_cpu(cpu)
    PCPU_SETUP_BUG_ON(unit_map[cpu] == UINT_MAX);

pcpu_nr_groups = ai->nr_groups;
pcpu_group_offsets = group_offsets;
pcpu_group_sizes = group_sizes;
pcpu_unit_map = unit_map;
pcpu_unit_offsets = unit_off;

pcpu_unit_pages = ai->unit_size >> PAGE_SHIFT;
pcpu_unit_size = pcpu_unit_pages << PAGE_SHIFT;  // 实际单元大小（页对齐）
pcpu_atom_size = ai->atom_size;
pcpu_chunk_struct_size = struct_size(chunk, populated,
                                     BITS_TO_LONGS(pcpu_unit_pages));

pcpu_sidelined_slot = __pcpu_size_to_slot(pcpu_unit_size) + 1;
pcpu_free_slot = pcpu_sidelined_slot + 1;
pcpu_to_depopulate_slot = pcpu_free_slot + 1;
pcpu_nr_slots = pcpu_to_depopulate_slot + 1;
pcpu_chunk_lists = memblock_alloc(pcpu_nr_slots * sizeof(pcpu_chunk_lists[0]),
                                  SMP_CACHE_BYTES);
...
for (i = 0; i < pcpu_nr_slots; i++)
    INIT_LIST_HEAD(&pcpu_chunk_lists[i]);

static_size = ALIGN(ai->static_size, PCPU_MIN_ALLOC_SIZE);
dyn_size = ai->dyn_size - (static_size - ai->static_size);

tmp_addr = (unsigned long)base_addr + static_size;
map_size = ai->reserved_size ?: dyn_size;
chunk = pcpu_alloc_first_chunk(tmp_addr, map_size);

if (ai->reserved_size) {
    pcpu_reserved_chunk = chunk;   // 将第一个 chunk 设为保留 chunk
    tmp_addr = (unsigned long)base_addr + static_size + ai->reserved_size;
    map_size = dyn_size;
    chunk = pcpu_alloc_first_chunk(tmp_addr, map_size); // 再创建一个动态 chunk
}

pcpu_first_chunk = chunk;   // 指向动态区域 chunk（如果没有保留区域，则指向第一个 chunk）
pcpu_nr_empty_pop_pages = pcpu_first_chunk->nr_empty_pop_pages;
pcpu_chunk_relocate(pcpu_first_chunk, -1);  // 将 chunk 放入合适的 slot

pcpu_nr_populated += PFN_DOWN(size_sum);   // 统计已 populate 的页数
pcpu_stats_chunk_alloc();
trace_percpu_create_chunk(base_addr);

pcpu_base_addr = base_addr;   // 保存基地址

// percpu_test.c
#include "linux/printk.h"
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/percpu.h>
#include <linux/smp.h>
#include <linux/cpu.h>

MODULE_LICENSE("GPL");
MODULE_AUTHOR("percpu-test");
MODULE_DESCRIPTION("percpu memory layout test");

/* 定义一个 per-cpu 变量 */
DEFINE_PER_CPU(int, percpu_var);
DEFINE_PER_CPU(int, percpu_var2);

static int __init percpu_test_init(void)
{
    int cpu;

    pr_info("percpu_test: module loaded\n");

    pr_info("__per_cpu_start: %px\n", __per_cpu_start);
    pr_info("__per_cpu_end: %px\n", __per_cpu_end);
    pr_info("pcpu_base_addr: %px\n", pcpu_base_addr);

    pr_info("percpu_test: printing percpu addresses and values\n");
    for_each_possible_cpu(cpu) {
        int *addr;
        int val;
        addr = &per_cpu(percpu_var, cpu);
        val = per_cpu(percpu_var, cpu);

        pr_info("percpu_var - CPU %d: addr=%px value=%d\n", cpu, addr, val);

        int *addr2;
        int val2;
        addr2 = &per_cpu(percpu_var2, cpu);
        val2 = per_cpu(percpu_var2, cpu);

        pr_info("percpu_var2 - CPU %d: addr=%px value=%d\n", cpu, addr2, val2);
    }

    /* 给每个 CPU 写入不同的值 */
    for_each_possible_cpu(cpu) {
        int *addr;
        int val;
        addr = &per_cpu(percpu_var, cpu);
        val = per_cpu(percpu_var, cpu);
        per_cpu(percpu_var, cpu) = cpu * 100;
    }

    pr_info("percpu_test: printing percpu addresses and values\n");

    for_each_possible_cpu(cpu) {

        int *addr;
        int val;

        addr = &per_cpu(percpu_var, cpu);
        val = per_cpu(percpu_var, cpu);

        pr_info("percpu_var - CPU %d: addr=%px value=%d\n", cpu, addr, val);
    }

    return 0;
}

static void __exit percpu_test_exit(void)
{
    pr_info("percpu_test: module unloaded\n");
}

module_init(percpu_test_init);
module_exit(percpu_test_exit);

&pcpu_unit_offsets[0]=0xffffff803fdf6100
	*(&pcpu_unit_offsets[0])=0
&pcpu_unit_offsets[1]=0xffffff803fdf6108
	*(&pcpu_unit_offsets[1])=0x15000
&pcpu_unit_offsets[2]=0xffffff803fdf6110
	*(&pcpu_unit_offsets[2])=0x2a000=0x15000*2
&pcpu_unit_offsets[3]=0xffffff803fdf6118
	*(&pcpu_unit_offsets[3])=0x3f000=0x15000*3

addr = (unsigned long)(&percpu_var) + __per_cpu_offset[cpu];

[    0.878314] __per_cpu_start: ffffffc009af0000
[    0.878565] __per_cpu_end: ffffffc009afbb28
[    0.878771] pcpu_base_addr: ffffff803fd8b000
[    0.878993] percpu_test: printing percpu addresses and values
[    0.879350] percpu_var - CPU 0: addr=ffffff803fd94c24 value=0
[    0.879659] percpu_var2 - CPU 0: addr=ffffff803fd94c28 value=0

[    0.844582] percpu_test: printing percpu addresses and values
[    0.845437] CPU 0: addr=ffffff803fd94c24 value=0
[    0.845693] CPU 1: addr=ffffff803fda9c24 value=0
[    0.845919] CPU 2: addr=ffffff803fdbec24 value=0
[    0.846138] CPU 3: addr=ffffff803fdd3c24 value=0
[    0.846458] percpu_test: printing percpu addresses and values
[    0.902621] CPU 0: addr=ffffff803fd94c24 value=0
[    0.919498] CPU 1: addr=ffffff803fda9c24 value=100
[    0.931415] CPU 2: addr=ffffff803fdbec24 value=200
[    0.940844] CPU 3: addr=ffffff803fdd3c24 value=300