林渡
AI智能摘要
本文分析了Linux内核中因`Non secure wdt`导致的死机问题。通过分析ramdump，发现所有CPU都在等待一个spin lock，且锁的持有者是`kworker/u17:12`。进一步分析发现，该进程在获取锁后出现了data abort，并在异常处理流程中再次尝试获取锁，导致死锁。根本原因是`nvt_update_firmware`函数中使用了未初始化的指针作为`printk`的参数，导致打印异常。解决方案是将`kmalloc`改为`kzalloc`，以确保内存被清零。实验验证了当`printk`的参数为非法指针时，会导致死锁。
此摘要由AI分析文章内容生成，仅供参考。
{% tip success %}
本章建议先看[linux内存管理] 第020篇 Linux内核slab内存的越界检查——SLUB_DEBUG的原理剖析
{% endtip %}
1. linux ramdump parser解析dump
查看死机原因，是 Non secure wdt
CPU |Reset Reason |Reset Count
0 |0x00000000 (TZBSP_ERR_FATAL_NONE ) |0x00000000
1 |0x00000001 (TZBSP_ERR_FATAL_NON_SECURE_WDT ) |0x00000001 // 报错
2 |0x00000000 (TZBSP_ERR_FATAL_NONE ) |0x00000000
3 |0x00000000 (TZBSP_ERR_FATAL_NONE ) |0x00000000
4 |0x00000000 (TZBSP_ERR_FATAL_NONE ) |0x00000000
5 |0x00000000 (TZBSP_ERR_FATAL_NONE ) |0x00000000
6 |0x00000000 (TZBSP_ERR_FATAL_NONE ) |0x00000000
7 |0x00000000 (TZBSP_ERR_FATAL_NONE ) |0x00000000
查看⼀下喂狗时间，15.84秒最后⼀次喂狗
crash-20201127> p wdog_data
wdog_data = $1 = (struct msm_watchdog_data *) 0xfffffff431ac7c80
crash-20201127> struct msm_watchdog_data 0xfffffff431ac7c80
struct msm_watchdog_data {
phys_base = 398524416,
size = 4096,
base = 0xffffff8008065000,
wdog_absent_base = 0x0,
dev = 0xfffffff431b4b090,
pet_time = 15000,
bark_time = 20000,
bark_irq = 41,
bite_irq = 42,
do_ipi_ping = true,
wakeup_irq_enable = true,
last_pet = 15840100412, // 15.84 最后一次喂狗
看⼀下所有cpu最后跑的进程，发现都在等spin lock
crash-20201127> bt -a
PID: 925 TASK: fffffff40d68cc80 CPU: 0 COMMAND: "audio.service
#0 [ffffff800ed7baf0] do_raw_spin_lock at ffffff97dc343c1c
#1 [ffffff800ed7bb20] _raw_spin_lock at ffffff97dd4550b8 //等spinlock
#2 [ffffff800ed7bb60] vprintk_emit at ffffff97dc34b398
#3 [ffffff800ed7bbf0] vprintk_default at ffffff97dc34bc68
#4 [ffffff800ed7bc90] vprintk_func at ffffff97dc34e3c0
#5 [ffffff800ed7bdd0] printk at ffffff97dc34a0ac
#6 [ffffff800ed7bdf0] msm_pcm_path_latency_ctl_get at ffffffa1f9e068d0
[platform_dlkm]
#7 [ffffff800ed7be30] snd_ctl_ioctl_compat at ffffff97dd11d048
#8 [ffffff800ed7be80] compat_sys_ioctl at ffffff97dc504b7c
#9 [ffffff800ed7bff0] el0_svc_naked at ffffff97dc283cfc
PC: ea53c224 LR: ea50a133 SP: ff91c4f8 PSTATE: 800c0010
X12: e8681720 X11: e8ec0f70 X10: 00000004 X9: ea1d2778
X8: ff91c518 X7: 00000036 X6: ea2d0d60 X5: ea55c25c
X4: 653eb4a1 X3: ff91c514 X2: ff91c518 X1: c2c85512
X0: 00000009
PID: 681 TASK: fffffff41ae1a680 CPU: 1 COMMAND: "logd.auditd"
#0 [ffffff801bdabae0] do_raw_spin_lock at ffffff97dc343c1c
#1 [ffffff801bdabb10] _raw_spin_lock at ffffff97dd4550b8 //等spinlock
#2 [ffffff801bdabb50] vprintk_emit at ffffff97dc34b398
#3 [ffffff801bdabc90] printk_emit at ffffff97dc34bc00
#4 [ffffff801bdabcb0] devkmsg_write at ffffff97dc349d9c
#5 [ffffff801bdabd10] do_iter_readv_writev at ffffff97dc491038
#6 [ffffff801bdabd30] do_iter_write at ffffff97dc48e758
#7 [ffffff801bdabe30] vfs_writev at ffffff97dc491438
#8 [ffffff801bdabe70] do_writev at ffffff97dc4912ac
#9 [ffffff801bdabeb0] sys_writev at ffffff97dc48e93c
#10 [ffffff801bdabff0] el0_svc_naked at ffffff97dc283cfc
PC: 0000007cb1626ad8 LR: 00000055c9ea1490 SP: 0000007c2b7f9620
X29: 0000007c2b7f97e0 X28: 0000007c2b7fc000 X27: b400007cb0e67a00
X26: 0000000000000000 X25: b400007c2f5493c0 X24: 00000000000000b9
X23: b400007c2f566ae0 X22: 00000000000000c6 X21: 00000000000004ca
X20: 00000000000004ca X19: 0000007c2b7f9620 X18: 0000007c2afc4000
X17: 0000007cb1626ad0 X16: 00000055c9eb1e40 X15: 0000000000000100
X14: 00000000000000c0 X13: 646c616d72656874 X12: 0000000000092580
X11: 0000007c00000000 X10: 0000000000000001 X9: 00000055c9e8d62f
X8: 0000000000000042 X7: 7f7f7f7f7f7f7f7f X6: 647568727267ff2f
X5: 00000000000000b8 X4: 0000000000000008 X3: 6576697373690030
X2: 0000000000000004 X1: 0000007c2b7f9790 X0: 000000000000001b
ORIG_X0: 000000000000001b SYSCALLNO: 42 PSTATE: 20000000
PID: 608 TASK: fffffff41d8d1380 CPU: 2 COMMAND: "kworker/2:2"
#0 [ffffff801bd8b930] do_raw_spin_lock at ffffff97dc343c1c
#1 [ffffff801bd8b960] _raw_spin_lock at ffffff97dd4550b8 //等spinlock
#2 [ffffff801bd8b9a0] vprintk_emit at ffffff97dc34b398
#3 [ffffff801bd8bab0] dev_vprintk_emit at ffffff97dcae2b60
#4 [ffffff801bd8bbf0] dev_printk_emit at ffffff97dcae2c20
#5 [ffffff801bd8bd20] __dynamic_dev_dbg at ffffff97dc71a8d8
#6 [ffffff801bd8bd50] tavil_codec_power_gate_digital_core at ffffffa1fa2a673c
[wcd934x_dlkm]
#7 [ffffff801bd8bd80] tavil_codec_power_gate_work at ffffffa1fa2a56bc
[wcd934x_dlkm]
#8 [ffffff801bd8bd90] process_one_work at ffffff97dc2e4af0
#9 [ffffff801bd8be00] worker_thread at ffffff97dc2e4f40
#10 [ffffff801bd8be60] kthread at ffffff97dc2ea440
PID: 0 TASK: fffffff4397e3980 CPU: 3 COMMAND: "swapper/3"
#0 [ffffff800801b9e0] do_raw_spin_lock at ffffff97dc343c1c
#1 [ffffff800801ba10] _raw_spin_lock at ffffff97dd4550b8 //等spinlock
#2 [ffffff800801ba50] vprintk_emit at ffffff97dc34b398
#3 [ffffff800801bae0] vprintk_default at ffffff97dc34bc68
#4 [ffffff800801bb80] vprintk_func at ffffff97dc34e3c0
#5 [ffffff800801bcc0] printk at ffffff97dc34a0ac
#6 [ffffff800801bd30] rcu_check_callbacks at ffffff97dc35e9b0
#7 [ffffff800801bd90] update_process_times at ffffff97dc369d48
#8 [ffffff800801bdc0] tick_sched_timer at ffffff97dc37d98c
#9 [ffffff800801be30] __hrtimer_run_queues at ffffff97dc36c438
#10 [ffffff800801bea0] hrtimer_interrupt at ffffff97dc36c0a4
#11 [ffffff800801bf00] arch_timer_handler_virt at ffffff97dd01ea80
#12 [ffffff800801bf10] handle_percpu_devid_irq at ffffff97dc353f98
#13 [ffffff800801bf60] __handle_domain_irq at ffffff97dc34e6b8
#14 [ffffff800801bfa0] gic_handle_irq at ffffff97dc281860
--- <IRQ stack> ---
#15 [ffffff80080e3e50] el1_irq at ffffff97dc283424
PC: ffffff97dcfd1f80 [lpm_cpuidle_enter+1264]
LR: ffffff97dcfd1efc [lpm_cpuidle_enter+1132]
SP: ffffff80080e3e60 PSTATE: a0c00145
X29: ffffff80080e3e80 X28: fffffff43fb798e8 X27: ffffff97de6e58e0
X26: ffffff97dec16b10 X25: 0000000000124f2a X24: 0000000000000000
X23: ffffff97deda6000 X22: fffffff421b11500 X21: fffffff421b11c10
X20: fffffff421aa4900 X19: 0000000000000000 X18: 0000000000000003
X17: 0000000000000000 X16: 0000000000000000 X15: 0000000000000022
X14: 0000000000000010 X13: 0000000000001360 X12: 0000000034155555
X11: 003178cb75c3e200 X10: ffffff97de6dc018 X9: 0000000000000001
X8: 0000000000000000 X7: 0000000000000000 X6: 0000000000000018
X5: 0000000000000001 X4: 0000000a946f618d X3: 0000000000000001
X2: 0000000000000000 X1: 00000000000001c0 X0: fffffff56b909c02
#16 [ffffff80080e3e80] lpm_cpuidle_enter at ffffff97dcfd1f7c
#17 [ffffff80080e3ee0] cpuidle_enter_state at ffffff97dcfcaee8
#18 [ffffff80080e3f40] cpuidle_enter at ffffff97dcfcb09c
#19 [ffffff80080e3f60] do_idle at ffffff97dc326834
#20 [ffffff80080e3fc0] cpu_startup_entry at ffffff97dc3268f4
#21 [ffffff80080e3fe0] secondary_start_kernel at ffffff97dc294d40
PID: 975 TASK: fffffff40d689380 CPU: 4 COMMAND: "HwBinder:925_2"
#0 [ffffff80147eb9f0] do_raw_spin_lock at ffffff97dc343c1c
#1 [ffffff80147eba20] _raw_spin_lock at ffffff97dd4550b8 //等spinlock
#2 [ffffff80147eba60] vprintk_emit at ffffff97dc34b398
#3 [ffffff80147ebb70] dev_vprintk_emit at ffffff97dcae2b60
#4 [ffffff80147ebcb0] dev_printk_emit at ffffff97dcae2c20
#5 [ffffff80147ebde0] __dynamic_dev_dbg at ffffff97dc71a8d8
#6 [ffffff80147ebe10] wm_adsp_cal_ambient_get at ffffffa1fa1e49b8 [cs35l41_dlkm]
#7 [ffffff80147ebe30] snd_ctl_ioctl_compat at ffffff97dd11d048
#8 [ffffff80147ebe80] compat_sys_ioctl at ffffff97dc504b7c
#9 [ffffff80147ebff0] el0_svc_naked at ffffff97dc283cfc
PC: ea53c224 LR: ea50a133 SP: e94f42d0 PSTATE: 800c0010
X12: e82b6d70 X11: ea8d11f8 X10: 00000000 X9: ea1d2738
X8: e94f42f0 X7: 00000036 X6: ea260170 X5: ea55c25c
X4: 653eb4a1 X3: e94f42ec X2: e94f42f0 X1: c2c85512
X0: 0000000b
PID: 930 TASK: fffffff40ab88080 CPU: 5 COMMAND: "kworker/u17:20"
#0 [ffffff800edeba80] do_raw_spin_lock at ffffff97dc343c1c
#1 [ffffff800edebab0] _raw_spin_lock at ffffff97dd4550b8 //等spinlock
#2 [ffffff800edebaf0] vprintk_emit at ffffff97dc34b398
#3 [ffffff800edebb80] vprintk_default at ffffff97dc34bc68
#4 [ffffff800edebc20] vprintk_func at ffffff97dc34e3c0
#5 [ffffff800edebd60] printk at ffffff97dc34a0ac
#6 [ffffff800edebd80] keyboard_resume_work at ffffff97dcd670b4
#7 [ffffff800edebd90] process_one_work at ffffff97dc2e4af0
#8 [ffffff800edebe00] worker_thread at ffffff97dc2e4f40
#9 [ffffff800edebe60] kthread at ffffff97dc2ea440
PID: 858 TASK: fffffff400fe1380 CPU: 6 COMMAND: "kworker/u17:12" // 锁的持有者
#0 [ffffff80219eb500] do_raw_spin_lock at ffffff97dc343c1c
#1 [ffffff80219eb530] _raw_spin_lock at ffffff97dd4550b8 //等spinlock
#2 [ffffff80219eb560] console_unlock at ffffff97dc34b620
#3 [ffffff80219eb5c0] console_unblank at ffffff97dc34c330
#4 [ffffff80219eb5e0] bust_spinlocks at ffffff97dc6f74c4
#5 [ffffff80219eb5f0] die at ffffff97dc28de9c
#6 [ffffff80219eb640] __do_kernel_fault at ffffff97dc2a8728
#7 [ffffff80219eb670] do_translation_fault at ffffff97dc2a7de8
#8 [ffffff80219eb710] do_mem_abort at ffffff97dc281078
#9 [ffffff80219eb880] el1_ia at ffffff97dc283144
PC: ffffff97dd4457bc [string+60]
LR: ffffff97dd4450f0 [vsnprintf+1072]
SP: ffffff80219eb890 PSTATE: 20c001c5
X29: ffffff80219eb890 X28: ffffff80219eb940 X27: ffffff97dde2ff80
X26: ffffff97de27390a X25: ffffff97de27390c X24: 00000000ffffffff
X23: ffffff97df0ac194 X22: 0000000000000002 X21: ffffff97df0ac540
X20: ffffff80219eb928 X19: ffffff97df0ac160 X18: ffffff97dec40000
X17: 00000000fff9393c X16: 000000000000002a X15: ffffff97dd445618
X14: ffffff97dde30227 X13: 000000000000004e X12: ffffffffffffffff
X11: ffffff97df0ac193 X10: 6b6b6b6b6b6b6b6b X9: 0000000000000000
X8: ffffff97df0ac540 X7: 0000000000000000 X6: ffffff97df0ac194
X5: ffffff80219eb9f8 X4: ffff0a00ffffff04 X3: ffff0a00ffffff04
X2: 6b6b6b6b6b6b6b6b X1: ffffffffffffffff X0: ffffff97df0ac194
#10 [ffffff80219eb890] string at ffffff97dd4457b8
#11 [ffffff80219eb8c0] vsnprintf at ffffff97dd4450ec
#12 [ffffff80219eb950] vscnprintf at ffffff97dd445f4c
#13 [ffffff80219eb9a0] vprintk_store at ffffff97dc34b168
#14 [ffffff80219eba20] vprintk_emit at ffffff97dc34b3c4
#15 [ffffff80219ebab0] vprintk_default at ffffff97dc34bc68
#16 [ffffff80219ebb50] vprintk_func at ffffff97dc34e3c0
#17 [ffffff80219ebc90] printk at ffffff97dc34a0ac
#18 [ffffff80219ebcf0] nvt_update_firmware at ffffff97dcd56700
#19 [ffffff80219ebd50] nvt_ts_resume at ffffff97dcd55a48
#20 [ffffff80219ebd80] nvt_resume_work at ffffff97dcd547d0
#21 [ffffff80219ebd90] process_one_work at ffffff97dc2e4af0
#22 [ffffff80219ebe00] worker_thread at ffffff97dc2e4f40
#23 [ffffff80219ebe60] kthread at ffffff97dc2ea440
PID: 452 TASK: fffffff423414c80 CPU: 7 COMMAND: "kworker/u16:12"
#0 [ffffff801a73ba20] do_raw_spin_lock at ffffff97dc343c1c
#1 [ffffff801a73ba50] _raw_spin_lock at ffffff97dd4550b8 //等spinlock
#2 [ffffff801a73ba90] vprintk_emit at ffffff97dc34b398
#3 [ffffff801a73bb20] vprintk_default at ffffff97dc34bc68
#4 [ffffff801a73bbc0] vprintk_func at ffffff97dc34e3c0
#5 [ffffff801a73bd00] printk at ffffff97dc34a0ac
#6 [ffffff801a73bd20] nvt_match_fw at ffffff97dcd5156c
#7 [ffffff801a73bd70] Boot_Update_Firmware at ffffff97dcd575bc
#8 [ffffff801a73bd90] process_one_work at ffffff97dc2e4af0
#9 [ffffff801a73be00] worker_thread at ffffff97dc2e4f40
#10 [ffffff801a73be60] kthread at ffffff97dc2ea440
2. 确认spinlock的持有者
持锁的是kworker/u17:12，从23.67秒开始被调度后⼀直占着cpu6
crash-20201127> dis -l ffffff97dc34b398
/home/work/data/miui_codes/build_home_rom/kernel/msm-4.14/kernel/printk/printk.c:1913
0xffffff97dc34b398 <vprintk_emit+112>: bl 0xffffff97dd4550a0
1896asmlinkage int vprintk_emit(int facility, int level,
1897 const char *dict, size_t dictlen,
1898 const char *fmt, va_list args)
1899{
1900 int printed_len;
1901 bool in_sched = false;
1902 unsigned long flags;
1903
1904 if (level == LOGLEVEL_SCHED) {
1905 level = LOGLEVEL_DEFAULT;
1906 in_sched = true;
1907 }
1908
1909 boot_delay_msec(level);
1910 printk_delay();
1911
1912 /* This stops the holder of console_sem just where we want him */
1913 logbuf_lock_irqsave(flags); //这里持锁
395#define logbuf_lock_irqsave(flags) \
396 do { \
397 printk_safe_enter_irqsave(flags); \
398 raw_spin_lock(&logbuf_lock); \ //持锁
399 } while (0)
crash-20201127> p logbuf_lock
logbuf_lock = $2 = {
raw_lock = {
owner = 27944,
next = 27953
},
magic = 3735899821,
owner_cpu = 6,
owner = 0xfffffff400fe1380 //持锁者
}
crash-20201127> task 0xfffffff400fe1380 //根据owner查找持锁进程
PID: 858 TASK: fffffff400fe1380 CPU: 6 COMMAND: "kworker/u17:12"
struct task_struct {
last_arrival = 23675532988,
3. 查找持锁为什么没有释放的原因
在第14帧的时候，拿到logbuf_lock，然后在第10帧时出现data abort，el1_ia异常处理流程中，也就是第⼆帧地⽅要再次拿logbuf_lock，⾃⼰就把⾃⼰锁死了。
crash-20201127> bt 858
PID: 858 TASK: fffffff400fe1380 CPU: 6 COMMAND: "kworker/u17:12"
#0 [ffffff80219eb500] do_raw_spin_lock at ffffff97dc343c1c
#1 [ffffff80219eb530] _raw_spin_lock at ffffff97dd4550b8
#2 [ffffff80219eb560] console_unlock at ffffff97dc34b620
#3 [ffffff80219eb5c0] console_unblank at ffffff97dc34c330
#4 [ffffff80219eb5e0] bust_spinlocks at ffffff97dc6f74c4
#5 [ffffff80219eb5f0] die at ffffff97dc28de9c
#6 [ffffff80219eb640] __do_kernel_fault at ffffff97dc2a8728
#7 [ffffff80219eb670] do_translation_fault at ffffff97dc2a7de8
#8 [ffffff80219eb710] do_mem_abort at ffffff97dc281078
#9 [ffffff80219eb880] el1_ia at ffffff97dc283144
PC: ffffff97dd4457bc [string+60]
LR: ffffff97dd4450f0 [vsnprintf+1072]
SP: ffffff80219eb890 PSTATE: 20c001c5
X29: ffffff80219eb890 X28: ffffff80219eb940 X27: ffffff97dde2ff80
X26: ffffff97de27390a X25: ffffff97de27390c X24: 00000000ffffffff
X23: ffffff97df0ac194 X22: 0000000000000002 X21: ffffff97df0ac540
X20: ffffff80219eb928 X19: ffffff97df0ac160 X18: ffffff97dec40000
X17: 00000000fff9393c X16: 000000000000002a X15: ffffff97dd445618
X14: ffffff97dde30227 X13: 000000000000004e X12: ffffffffffffffff
X11: ffffff97df0ac193 X10: 6b6b6b6b6b6b6b6b X9: 0000000000000000
X8: ffffff97df0ac540 X7: 0000000000000000 X6: ffffff97df0ac194
X5: ffffff80219eb9f8 X4: ffff0a00ffffff04 X3: ffff0a00ffffff04
X2: 6b6b6b6b6b6b6b6b X1: ffffffffffffffff X0: ffffff97df0ac194
#10 [ffffff80219eb890] string at ffffff97dd4457b8 // 开始出现data abort
#11 [ffffff80219eb8c0] vsnprintf at ffffff97dd4450ec
#12 [ffffff80219eb950] vscnprintf at ffffff97dd445f4c
#13 [ffffff80219eb9a0] vprintk_store at ffffff97dc34b168
#14 [ffffff80219eba20] vprintk_emit at ffffff97dc34b3c4 // 拿到logbuf_lock
#15 [ffffff80219ebab0] vprintk_default at ffffff97dc34bc68
#16 [ffffff80219ebb50] vprintk_func at ffffff97dc34e3c0
#17 [ffffff80219ebc90] printk at ffffff97dc34a0ac
#18 [ffffff80219ebcf0] nvt_update_firmware at ffffff97dcd56700
#19 [ffffff80219ebd50] nvt_ts_resume at ffffff97dcd55a48
#20 [ffffff80219ebd80] nvt_resume_work at ffffff97dcd547d0
#21 [ffffff80219ebd90] process_one_work at ffffff97dc2e4af0
#22 [ffffff80219ebe00] worker_thread at ffffff97dc2e4f40
crash-20201127> dis -l ffffff97dc34b3c4
/home/work/data/miui_codes/build_home_rom/kernel/msm-4.14/kernel/printk/printk.c:1914
0xffffff97dc34b3c4 <vprintk_emit+156>: bl 0xffffff97dc34b108
1896asmlinkage int vprintk_emit(int facility, int level,
1897 const char *dict, size_t dictlen,
1898 const char *fmt, va_list args)
1899{
1900 int printed_len;
1901 bool in_sched = false;
1902 unsigned long flags;
1903
1904 if (level == LOGLEVEL_SCHED) {
1905 level = LOGLEVEL_DEFAULT;
1906 in_sched = true;
1907 }
1908
1909 boot_delay_msec(level);
1910 printk_delay();
1911
1912 /* This stops the holder of console_sem just where we want him */
1913 logbuf_lock_irqsave(flags);
1914 printed_len = vprintk_store(facility, level, dict, dictlen, fmt, gs);
crash-20201127> dis -l ffffff97dd4457bc
/home/work/data/miui_codes/build_home_rom/kernel/msm-4.14/lib/vsprintf.c: 595
0xffffff97dd4457bc <string+60>: ldrb w14, [x10,x9] //string函数+60的地方有问题
X10: 6b6b6b6b6b6b6b6b //x10寄存器很异常
4. 根本原因
从函数调⽤来看，其实就是调⽤printk进⾏打印⽇志输出，最后出现data abort，只能是打印的⼊参有问题
x10的值是从x2来的，x2是第三个⼊参
crash-20201127> dis string
0xffffff97dd445780 <string>: stp x29, x30, [sp,#-16]!
0xffffff97dd445784 <string+4>: mov x29, sp
0xffffff97dd445788 <string+8>: mov x8, x1
0xffffff97dd44578c <string+12>: asr x1, x3, #48
0xffffff97dd445790 <string+16>: cbz x1, 0xffffff97dd4457ec
0xffffff97dd445794 <string+20>: adrp x10, 0xffffff97de1b2000
0xffffff97dd445798 <string+24>: cmp x2, #0x1, lsl #12
0xffffff97dd44579c <string+28>: add x10, x10, #0xce3
0xffffff97dd4457a0 <string+32>: mov x9, xzr
0xffffff97dd4457a4 <string+36>: csel x10, x10, x2, cc
0xffffff97dd4457a8 <string+40>: add x11, x0, x1
0xffffff97dd4457ac <string+44>: mov x12, x1
0xffffff97dd4457b0 <string+48>: b 0xffffff97dd4457bc
0xffffff97dd4457b4 <string+52>: add x9, x9, #0x1
0xffffff97dd4457b8 <string+56>: cbz x12, 0xffffff97dd4457dc
0xffffff97dd4457bc <string+60>: ldrb w14, [x10,x9] //crash here
查看string函数源码
char *string(char *buf, char *end, const char *s, struct printf_spec spec)
{
int len = 0;
size_t lim = spec.precision;
if ((unsigned long)s < PAGE_SIZE)
s = "(null)";
while (lim--) {
char c = *s++;
if (!c)
break;
if (buf < end)
*buf = c;
++buf;
++len;
}
return widen_string(buf, len, end, spec);
}
那么从最开头调⽤printk的地⽅开始查，nvt_update_firmware这个函数⾥⾯进⾏打印输出
-> #18 [ffffff80219ebcf0] nvt_update_firmware at ffffff97dcd56700
crash-20201127> dis ffffff97dcd56700 -l
o/home/work/data/miui_codes/build_home_rom/kernel/msm-4.14/drivers/input/touchscreen/nt36523/nt36xxx_fw_update.c: 334
0xffffff97dcd56700 <nvt_update_firmware+168>: bl 0xffffff97dc34a040
对应源码
324static int32_t update_firmware_request(const char *filename)
325{
326 uint8_t retry = 0;
327 int32_t ret = 0;
328
329 if (NULL == filename) {
330 return -ENOENT;
331 }
332
333 while (1) {
334 NVT_LOG("filename is %s\n", filename); //crash here
986int32_t nvt_update_firmware(const char *firmware_name)
987{
988 int32_t ret = 0;
989
990 // request bin file in "/etc/firmware"
991 ret = update_firmware_request(firmware_name);
继续往前推一个栈帧-> #19 [ffffff80219ebd50] nvt_ts_resume at ffffff97dcd55a48
crash-20201127> dis -l ffffff97dcd55a48
/home/work/data/miui_codes/build_home_rom/kernel/msm-4.14/drivers/input/touchscreen/nt36523/nt36xxx.c: 3448
0xffffff97dcd55a48 <nvt_ts_resume+216>: bl 0xffffff97dcd56658
3413static int32_t nvt_ts_resume(struct device *dev)
3414{
3415 int ret = 0;
3416 if (bTouchIsAwake) {
3417 NVT_LOG("Touch is already resume\n");
3418 return 0;
3419 }
3420
3421 if (ts->dev_pm_suspend)
3422 pm_stay_awake(dev);
3423
3424 mutex_lock(&ts->lock);
3425
3426 NVT_LOG("resume start\n");
3427 ts->ic_state = NVT_IC_RESUME_IN;
3428 if (!ts->db_wakeup) {
3429 if (ts->ts_pinctrl) {
3430 ret = pinctrl_select_state(ts->ts_pinctrl, ts-
inctrl_state_active);
3431 if (ret < 0) {
3432 NVT_ERR("Failed to select %s pinstate %d\n",
3433 PINCTRL_STATE_ACTIVE, ret);
3434 }
3435 } else {
3436 NVT_ERR("Failed to init pinctrl\n");
3437 }
3438 }
3439
3440 // please make sure display reset(RESX) sequence and mipi dsi cmds sent before this
3441#if NVT_TOUCH_SUPPORT_HW_RST
3442 gpio_set_value(ts->reset_gpio, 1);
3443#endif
3444 if (nvt_get_dbgfw_status()) {
3445 ret = nvt_update_firmware(DEFAULT_DEBUG_FW_NAME);
3446 if (ret < 0) {
3447 NVT_ERR("use built-in fw");
3448 ret = nvt_update_firmware(ts->fw_name); // 指向这里
从上⾯函数调⽤关系可以看到，其实filename，是从ts->fw_name传过来的，这个值0x6b6b6b6b6b6b6b6b是有问题的，导致打印异常，然后前⾯的判空处理⼀点意义都没，直接被跳过了。
crash-20201127> p ts
ts = $3 = (struct nvt_ts_data *) 0xfffffff421873680
crash-20201127> struct nvt_ts_data.fw_name 0xfffffff421873680
fw_name = 0x6b6b6b6b6b6b6b6b <Address 0x6b6b6b6b6b6b6b6b out of bounds>
从内存中看⼀下ts对应的内容，是slab的⼀个object
crash-20201127> struct nvt_ts_data -ox
struct nvt_ts_data {
[0x0] struct spi_device *client;
[0x8] struct input_dev *input_dev;
[0x10] struct delayed_work nvt_fwu_work;
[0x70] struct delayed_work nvt_lockdown_work;
[0xd0] struct work_struct switch_mode_work;
[0xf0] uint16_t addr;
[0xf2] int8_t phys[32];
[0x118] struct notifier_block drm_notif;
[0x130] uint32_t config_array_size;
[0x138] struct nvt_config_info *config_array;
[0x140] const u8 *fw_name; //偏移0x140
crash-20201127> rd 0xfffffff421873680 200 //读取附近0x200的内存
fffffff421873680: fffffff425a5e480 fffffff42184b280 ...%.......!....
fffffff421873690: 0000000000000200 fffffff421873698 .........6.!....
fffffff4218736a0: fffffff421873698 ffffff97dcd575b0 .6.!.....u......
fffffff4218736b0: dead000000000200 0000000000000000 ................
fffffff4218736c0: 00000000ffff9132 ffffff97dc2dfba0 2.........-.....
fffffff4218736d0: fffffff421873690 6b6b6b6b21600000 .6.!......`!kkkk
fffffff4218736e0: fffffff42195f880 6b6b6b6b00000008 ...!........kkkk
fffffff4218736f0: 0000000000000200 fffffff4218736f8 .........6.!....
fffffff421873700: fffffff4218736f8 ffffff97dcd54520 .6.!.... E......
fffffff421873710: dead000000000200 0000000000000000 ................
fffffff421873720: 00000000ffff8d4a ffffff97dc2dfba0 J.........-.....
fffffff421873730: fffffff4218736f0 6b6b6b6b1aa00001 .6.!........kkkk
fffffff421873740: fffffff42195e480 6b6b6b6b00000008 ...!........kkkk
fffffff421873750: 0000000fffffffe0 fffffff421873758 ........X7.!....
fffffff421873760: fffffff421873758 ffffff97dcd54470 X7.!....pD......
fffffff421873770: 2f7475706e696b6b 6b6b6b6b6b007374 kkinput/ts.kkkkk
fffffff421873780: 6b6b6b6b6b6b6b6b 6b6b6b6b6b6b6b6b kkkkkkkkkkkkkkkk
fffffff421873790: 6b6b6b6b6b6b6b6b ffffff97dcd54800 kkkkkkkk.H......
fffffff4218737a0: ffffff97df491820 6b6b6b6b6b6b6b6b .I.....kkkkkkkk
fffffff4218737b0: 6b6b6b6b00000002 fffffff421a0ba00 ....kkkk...!....
fffffff4218737c0: 6b6b6b6b6b6b6b6b 6b6b6b6b6b6b6b6b kkkkkkkkkkkkkkkk //0x140的地方
fffffff4218737d0: 3158383131425301 0000000232200000 .SB118X1.. 2....
fffffff4218737e0: 6b6b000a0a000640 0000000200000001 @.....kk........
fffffff4218737f0: 0000000100000002 0000000100000002 ................
fffffff421873800: 0000000200000002 0000000200000002 ................
fffffff421873810: 0000000200000001 0000000300000002 ................
fffffff421873820: 0000000100000002 0000200100000027 ........'.... ..
fffffff421873830: 6b6b6b6b6b6b6b6b fffffff400fe1380 kkkkkkkk........
fffffff421873840: dead4ead00000000 6b6b6b6bffffffff .....N......kkkk
fffffff421873850: ffffffffffffffff 6b6b6b6b00000000 ............kkkk
fffffff421873860: fffffff421873860 fffffff421873860 `8.!....`8.!....
fffffff421873870: fffffff421873838 ffffff97ddc9a198 88.!............
fffffff421873880: 6b6b6b6b6b6b0202 fffffff42184ee80 ..kkkkkk...!....
fffffff421873890: fffffff421a20000 0000000000000000 ...!............
fffffff4218738a0: dead4ead00000000 6b6b6b6bffffffff .....N......kkkk
fffffff4218738b0: ffffffffffffffff 6b6b6b6b00000000 ............kkkk
fffffff4218738c0: fffffff4218738c0 fffffff4218738c0 .8.!.....8.!....
fffffff4218738d0: fffffff421873898 6b6b060401016b00 .8.!.....k....kk
fffffff4218738e0: fffffff421a30a80 702f7475706e6900 ...!.....input/p
fffffff4218738f0: 6b6b6b6b6b006e65 6b6b6b6b6b6b6b6b en.kkkkkkkkkkkkk
fffffff421873900: 6b6b6b6b6b6b6b6b 6b6b6b6b6b6b6b6b kkkkkkkkkkkkkkkk
fffffff421873910: fffffff42195c880 0000000000000220 ...!.... .......
fffffff421873920: fffffff421873920 fffffff421873920 9.!.... 9.!....
fffffff421873930: ffffff97dcd547e0 0000000000000220 .G...... .......
fffffff421873940: fffffff421873940 fffffff421873940 @9.!....@9.!....
fffffff421873950: ffffff97dcd547c0 fffffff421a8f700 .G.........!....
fffffff421873960: 6b6b6b6b6b6b6b6b 0000000000927c00 kkkkkkkk.|......
fffffff421873970: fffffff4225dab28 6b6b6b6b6b00006b (.]"....k..kkkkk
fffffff421873980: 6b6b6b6b00000000 dead4ead00000000 ....kkkk.....N..
fffffff421873990: 6b6b6b6bffffffff ffffffffffffffff ....kkkk........
fffffff4218739a0: fffffff4218739a0 fffffff4218739a0 .9.!.....9.!....
fffffff4218739b0: ffffffff6b6b6b6b fffffff421a1ad80 kkkk.......!....
fffffff4218739c0: fffffff421a18300 fffffff421a18a80 ...!.......!....
fffffff4218739d0: fffffff421959880 6b6b6b6b6b6b6b6b ...!....kkkkkkkk
fffffff4218739e0: 6b6b6b6b6b6b6b6b 6b6b6b6b6b6b6b6b kkkkkkkkkkkkkkkk
fffffff4218739f0: 6b6b6b6b6b6b6b6b 6b6b6b6b6b6b6b6b kkkkkkkkkkkkkkkk
fffffff421873a00: 6b6b6b6b6b6b6b6b 6b6b6b6b6b6b6b6b kkkkkkkkkkkkkkkk
fffffff421873a10: 6b6b6b6b6b6b6b6b 6b6b6b6b6b6b6b6b kkkkkkkkkkkkkkkk
fffffff421873a20: 6b6b6b6b6b6b6b6b 6b6b6b6b6b6b6b6b kkkkkkkkkkkkkkkk
fffffff421873a30: 6b6b6b6b6b6b6b6b 6b6b6b6b6b6b6b6b kkkkkkkkkkkkkkkk
fffffff421873a40: 6b6b6b6b6b6b6b6b 6b6b6b6b6b6b6b6b kkkkkkkkkkkkkkkk
fffffff421873a50: 6b6b6b6b6b6b6b6b 6b6b6b6b6b6b6b6b kkkkkkkkkkkkkkkk
fffffff421873a60: 6b6b6b6b6b6b6b6b 6b6b6b6b6b6b6b6b kkkkkkkkkkkkkkkk
fffffff421873a70: 6b6b6b6b6b6b6b6b a56b6b6b6b6b6b6b kkkkkkkkkkkkkkk. // 找到a5的地方
fffffff421873a80: cccccccccccccccc c027ee5c18a50909 ............\.'. // 填充0xcc的地方
fffffff421873a90: ffffff97dcd51940 ffffff97dc46d418 @.........F..... // alloc track
fffffff421873aa0: ffffff97dcd51940 ffffff97dcbc04e4 @...............
fffffff421873ab0: ffffff97dcae64ac ffffff97dcae6938 .d......8i......
fffffff421873ac0: ffffff97dcae3dc8 ffffff97dcae6838 .=......8h......
fffffff421873ad0: ffffff97dcae45c0 ffffff97dcae7a6c .E......lz......
fffffff421873ae0: ffffff97dcbc0458 ffffff97de65736c X.......lse.....
fffffff421873af0: ffffff97dc283e58 ffffff97de6011c8 X>(.......`.....
fffffff421873b00: ffffff97dd449a40 ffffff97dc2853d4 @.D......S(.....
fffffff421873b10: 0000000000000000 0000000100000002 ................
fffffff421873b20: 00000000ffff8bb8 0000000000000000 ................
fffffff421873b30: 0000000000000000 0000000000000000 ................
fffffff421873b40: 0000000000000000 0000000000000000 ................
fffffff421873b50: 0000000000000000 0000000000000000 ................
fffffff421873b60: 0000000000000000 0000000000000000 ................
fffffff421873b70: 0000000000000000 0000000000000000 ................
fffffff421873b80: 0000000000000000 0000000000000000 ................
fffffff421873b90: 0000000000000000 0000000000000000 ................
fffffff421873ba0: 0000000000000000 0000000000000000 ................
fffffff421873bb0: 0000000000000000 0000000000000000 ................
fffffff421873bc0: 5a5a5a5a5a5a5a5a 5a5a5a5a5a5a5a5a ZZZZZZZZZZZZZZZZ
fffffff421873bd0: 5a5a5a5a5a5a5a5a 5a5a5a5a5a5a5a5a ZZZZZZZZZZZZZZZZ
fffffff421873be0: 5a5a5a5a5a5a5a5a 5a5a5a5a5a5a5a5a ZZZZZZZZZZZZZZZZ
fffffff421873bf0: 5a5a5a5a5a5a5a5a 5a5a5a5a5a5a5a5a ZZZZZZZZZZZZZZZZ
crash-20201127> struct track fffffff421873a90 -x // 查看slab的alloc track
struct track {
addr = 0xffffff97dcd51940, // 申请的地址
addrs = {0xffffff97dc46d418, 0xffffff97dcd51940, 0xffffff97dcbc04e4,
0xffffff97dcae64ac, 0xffffff97dcae6938, 0xffffff97dcae3dc8, 0xffffff97dcae6838,
0xffffff97dcae45c0, 0xffffff97dcae7a6c, 0xffffff97dcbc0458, 0xffffff97de65736c,
0xffffff97dc283e58, 0xffffff97de6011c8, 0xffffff97dd449a40, 0xffffff97dc2853d4,
0x0},
cpu = 0x2,
pid = 0x1,
when = 0xffff8bb8
}
crash-20201127> dis 0xffffff97dcd51940 -l // 查看slab alloc的代码
/home/work/data/miui_codes/build_home_rom/kernel/msm-4.14/drivers/input/touchscreen/nt36523/nt36xxx.c: 2664
0xffffff97dcd51940 <nvt_ts_probe+112>: adrp x26, 0xffffff97df1d0000
查看源代码
2654static int32_t nvt_ts_probe(struct spi_device *client)
2655{
2656 int32_t ret = 0;
2657#if ((TOUCH_KEY_NUM > 0) || WAKEUP_GESTURE)
2658 int32_t retry = 0;
2659#endif
2660 struct attribute_group *attrs_p = NULL;
2661
2662 NVT_LOG("probe start\n");
2663
2664 ts = kmalloc(sizeof(struct nvt_ts_data), GFP_KERNEL);
2665 if (ts == NULL) {
2666 NVT_ERR("failed to allocated memory for nvt ts data\n");
2667 return -ENOMEM;
2668 }
2669
2670 ts->xbuf = (uint8_t *)kzalloc((NVT_TRANSFER_LEN+1+DUMMY_BYTES),GFP_KERNEL);
2671 if(ts->xbuf == NULL) {
2672 NVT_ERR("kzalloc for xbuf failed!\n");
2673 ret = -ENOMEM;
2674 goto err_malloc_xbuf;
2675 }
2676
2677 ts->rbuf = (uint8_t *)kzalloc(NVT_READ_LEN, GFP_KERNEL);
2678 if(ts->rbuf == NULL) {
2679 NVT_ERR("kzalloc for rbuf failed!\n");
2680 ret = -ENOMEM;
2681 goto err_malloc_rbuf;
2682 }
2683
代码写的不规范，kmalloc之后没有对内存清零，导致使⽤了默认的脏数据。⽽对filename赋值的地⽅，还没来得及跑到。其实，说⽩了了就是两work queue没有同步约束，刚好在反复重启压⼒测试的时候，出现了极端情况，kworker/u16:12 延迟14秒，没成想超过了14秒，⽽且被调度在kworker/u17:12之后。
PID: 452 TASK: fffffff423414c80 CPU: 7 COMMAND: "kworker/u16:12"
#0 [ffffff801a73ba20] do_raw_spin_lock at ffffff97dc343c1c
#1 [ffffff801a73ba50] _raw_spin_lock at ffffff97dd4550b8
#2 [ffffff801a73ba90] vprintk_emit at ffffff97dc34b398
#3 [ffffff801a73bb20] vprintk_default at ffffff97dc34bc68
#4 [ffffff801a73bbc0] vprintk_func at ffffff97dc34e3c0
#5 [ffffff801a73bd00] printk at ffffff97dc34a0ac
#6 [ffffff801a73bd20] nvt_match_fw at ffffff97dcd5156c
#7 [ffffff801a73bd70] Boot_Update_Firmware at ffffff97dcd575bc
#8 [ffffff801a73bd90] process_one_work at ffffff97dc2e4af0
#9 [ffffff801a73be00] worker_thread at ffffff97dc2e4f40
#10 [ffffff801a73be60] kthread at ffffff97dc2ea440
1309void nvt_match_fw(void)
1310{
1311 NVT_LOG("start match fw name");//卡在这⾥
1312 if (is_lockdown_empty(ts->lockdown_info))
1313 flush_delayed_work(&ts->nvt_lockdown_work);
1314 if (nvt_get_panel_type(ts) < 0) {
1315 ts->fw_name = DEFAULT_BOOT_UPDATE_FIRMWARE_NAME;
1316 ts->mp_name = DEFAULT_MP_UPDATE_FIRMWARE_NAME;
1317 } else {
1318 ts->fw_name = ts->config_array[ts->panel_index].nvt_fw_name;
1319 ts->mp_name = ts->config_array[ts->panel_index].nvt_mp_name;
1320 }
1321}
kworker/u17:12
3010 INIT_WORK(&ts->resume_work, nvt_resume_work);
3494static int nvt_drm_notifier_callback(struct notifier_block *self, unsigned long event, void *data)
3495{
3496 struct drm_notify_data *evdata = data;
3497 int *blank;
3498 struct nvt_ts_data *ts_data =
3499 container_of(self, struct nvt_ts_data, drm_notif);
3500
3501 if (!evdata)
3502 return 0;
3503
3504 if (evdata && ts_data) {
3505 blank = evdata->data;
3506 if (event == DRM_EARLY_EVENT_BLANK) {
3507 if (*blank == DRM_BLANK_POWERDOWN) {
3508 NVT_LOG("event=%lu, *blank=%d\n", event, *blank);
3509 flush_workqueue(ts_data->event_wq);
3510 queue_work(ts_data->event_wq, &ts_data->suspend_work);
3511 }
3512 } else if (event == DRM_R_EARLY_EVENT_BLANK) {
3513 if (*blank == DRM_BLANK_POWERDOWN) {
3514 NVT_LOG("event=%lu, *blank=%d\n", event, *blank);
3515 nvt_enable_doubleclick();
3516 }
3517 } else if (event == DRM_EVENT_BLANK) {
3518 if (*blank == DRM_BLANK_UNBLANK) {
3519 NVT_LOG("event=%lu, *blank=%d\n", event, *blank);
3520 flush_workqueue(ts_data->event_wq);
3521 queue_work(ts_data->event_wq, &ts_data->resume_work);
3522 }
3523 }
3524
3525 }
3526
3527 return 0;
3528}
kworker/u16:12
2942 INIT_DELAYED_WORK(&ts->nvt_fwu_work, Boot_Update_Firmware);
2943 // please make sure boot update start after display reset(RESX) sequence
2944 queue_delayed_work(nvt_fwu_wq, &ts->nvt_fwu_work, msecs_to_jiffies(14000));
Task name PID Exec_Started_at Last_Queued_at Total_wait_time No_of_times_exec Prio
kworker/u17:12 858 23.675532988 0.000000000 0.001926303 37 100
Task name PID Exec_Started_at Last_Queued_at Total_wait_time No_of_times_exec Prio
kworker/u16:12 452 23.760054291 0.000000000 0.427718656 3487 120
5. 解决方案
kmalloc->kzalloc
2654static int32_t nvt_ts_probe(struct spi_device *client)
2655{
2656 int32_t ret = 0;
2657#if ((TOUCH_KEY_NUM > 0) || WAKEUP_GESTURE)
2658 int32_t retry = 0;
2659#endif
2660 struct attribute_group *attrs_p = NULL;
2661
2662 NVT_LOG("probe start\n");
2663
2664 ts = kmalloc(sizeof(struct nvt_ts_data), GFP_KERNEL);
6. 小实验
给printk的⼊参传⼊⼀个⾮空的⾮法指针会怎么样？测试结果和预想的是⼀样的，会⾃⼰把⾃⼰锁死，只能等狗咬
diff --git a/drivers/input/touchscreen/nt36523/nt36xxx.c
b/drivers/input/touchscreen/nt36523/nt36xxx.c
index aeec43b..2f17e6a 100644
--- a/drivers/input/touchscreen/nt36523/nt36xxx.c
+++ b/drivers/input/touchscreen/nt36523/nt36xxx.c
@@ -2666,6 +2666,9 @@ static int32_t nvt_ts_probe(struct spi_device *client)
NVT_ERR("failed to allocated memory for nvt ts data\n");
return -ENOMEM;
}
ts = kmalloc(sizeof(struct nvt_ts_data), GFP_KERNEL);
if (ts == NULL) {
NVT_ERR("failed to allocated memory for nvt ts data\n");
return -ENOMEM;
}
+ kfree(ts);
+
+ NVT_LOG("probe start %s\n",ts->fw_name);
ts->xbuf = (uint8_t *)kzalloc((NVT_TRANSFER_LEN+1+DUMMY_BYTES), GFP_KERNEL);
if(ts->xbuf == NULL) {
crash-20201127> bt
PID: 1 TASK: ffffffee38692100 CPU: 0 COMMAND: "swapper/0"
#0 [ffffff800805b360] _raw_spin_lock at ffffff8788c261e4 //等logbuf_lock
#1 [ffffff800805b3a0] console_unlock at ffffff8787d3932c
#2 [ffffff800805b400] console_unblank at ffffff8787d39ddc
#3 [ffffff800805b420] bust_spinlocks at ffffff878807f6b8
#4 [ffffff800805b430] die at ffffff8787c8dc90
#5 [ffffff800805b470] __do_kernel_fault at ffffff8787ca5fc4
#6 [ffffff800805b4a0] do_bad_area at ffffff8787ca5c74
#7 [ffffff800805b4b0] do_translation_fault at ffffff8787ca56bc
#8 [ffffff800805b550] do_mem_abort at ffffff8787c8167c
#9 [ffffff800805b6c0] el1_ia at ffffff8787c83944
PC: ffffff8788c19c00 [string+44]
LR: ffffff8788c19430 [vsnprintf+892]
SP: ffffff800805b6d0 PSTATE: a08000c5
X29: ffffff800805b6d0 X28: ffffff8789623978 X27: 00000000ffffffff
X26: ffffff800805b780 X25: 0000000000000002 X24: ffffff8789a7ac66
X23: ffffff8789a7ac68 X22: ffffff878a87b08a X21: ffffff878a87b440
X20: ffffff800805b768 X19: ffffff878a87b060 X18: 0000000000000000
X17: 0000000000000029 X16: ffffff8788c19938 X15: 0000000000000004
X14: ffff0000ffffff00 X13: ffffff8789623c0e X12: 0000000000000000
X11: ffffffffffffffff X10: afafafafafafafaf X9: 0000000000000000
X8: ffffff878a87b440 X7: 0000000000000000 X6: ffffff878a87b08a
X5: ffffff800805b838 X4: ffff0a00ffffff04 X3: ffff0a00ffffff04
X2: afafafafafafafaf X1: ffffffffffffffff X0: ffffff878a87b08a
#10 [ffffff800805b6d0] string at ffffff8788c19bfc
#11 [ffffff800805b700] vsnprintf at ffffff8788c1942c
#12 [ffffff800805b790] vscnprintf at ffffff8788c1a328
#13 [ffffff800805b7e0] vprintk_store at ffffff8787d38f1c
#14 [ffffff800805b860] vprintk_emit at ffffff8787d39148 //持logbuf_lock
#15 [ffffff800805b8f0] vprintk_default at ffffff8787d39874
#16 [ffffff800805b990] vprintk_func at ffffff8787d3b870
#17 [ffffff800805bad0] printk at ffffff8787d3839c
#18 [ffffff800805bb10] nvt_ts_probe at ffffff87885eb390
#19 [ffffff800805bb70] spi_drv_probe at ffffff87884a01e8
#20 [ffffff800805bba0] driver_probe_device at ffffff87883e3fe8
#21 [ffffff800805bbe0] __driver_attach at ffffff87883e43bc
#22 [ffffff800805bc30] bus_for_each_dev at ffffff87883e2078
#23 [ffffff800805bc60] driver_attach at ffffff87883e4324
#24 [ffffff800805bc70] bus_add_driver at ffffff87883e27c0
#25 [ffffff800805bca0] driver_register at ffffff87883e543c
#26 [ffffff800805bcc0] __spi_register_driver at ffffff87884a015c
#27 [ffffff800805bce0] nvt_driver_init at ffffff8789e548ac
#28 [ffffff800805be00] do_one_initcall at ffffff8787c84610
#29 [ffffff800805be40] kernel_init_freeable at ffffff8789e00f9c
#30 [ffffff800805bea0] kernel_init at ffffff8788c1d528
#31 [ffffff800805bec0] ret_from_fork at ffffff8787c85a80
crash-20201127> dis -l ffffff8788c19c00
/home/gumingtao/work/code/k82/kernel/msm-4.14/lib/vsprintf.c: 595
0xffffff8788c19c00 <string+44>: ldrb w13, [x10,x9]
crash-20201127> p ts
ts = $1 = (struct nvt_ts_data *) 0xffffffee2afd5000
crash-20201127> struct nvt_ts_data.fw_name 0xffffffee2afd5000
fw_name = 0xafafafafafafafaf <Address 0xafafafafafafafaf out of bounds>