Hi,
I use iMX8MN to make signal acquisition and processing on M7 side (baremetal) and then send some results to Linux on A53. Firmware on M7 is fairly big (use of CMSIS DSP, protobuf for multicore comm, rpmsg, etc) so memory footprint must be optimized. Both TCM data and text are full and I have fairly big heap that I don't know where to allocate. I tried to put the heap in the DDR like in the following linker script :
Since I crawled the web for a similar problem I will leave here one of my thoughts.
I guess the best way to fix this is to make `memcpy` function handle unaligned section sizes, something like this.
+/*
+ * Custom memory copy implementation for i.MX DSP Cores
+ *
+ * The IRAM is part of the HiFi DSP.
+ * According to hw specs only 32-bits writes are allowed.
+ */
+static int imx_dsp_rproc_memcpy(void *dest, const void *src, size_t size)
+{
+ const u8 *src_byte = src;
+ u32 affected_mask;
+ u32 tmp;
+ int q, r;
+
+ q = size / 4;
+ r = size % 4;
+
+ /* __iowrite32_copy use 32bit size values so divide by 4 */
+ __iowrite32_copy(dest, src, q);
+
+ if (r) {
+ affected_mask = (1 << (8 * r)) - 1;
+
+ /* first read the 32bit data of dest, then change affected
+ * bytes, and write back to dest.
+ * For unaffected bytes, it should not be changed
+ */
+ tmp = ioread32(dest + q * 4);
+ tmp &= ~affected_mask;
+
+ tmp |= *(u32 *)(src_byte + q * 4) & affected_mask;
+ iowrite32(tmp, dest + q * 4);
+ }
+
+ return 0;
+}
+
+/**
+ * imx_dsp_rproc_elf_load_segments() - load firmware segments to memory
+ * @rproc: remote processor which will be booted using these fw segments
+ * @fw: the ELF firmware image
+ *
+ * This function loads the firmware segments to memory, where the remote
+ * processor expects them.
+ *
+ * Return: 0 on success and an appropriate error code otherwise
+ */
+static int imx_dsp_rproc_elf_load_segments(struct rproc *rproc, const struct firmware *fw)
+{
+ struct device *dev = &rproc->dev;
+ const void *ehdr, *phdr;
+ int i, ret = 0;
+ u16 phnum;
+ const u8 *elf_data = fw->data;
+ u8 class = fw_elf_get_class(fw);
+ u32 elf_phdr_get_size = elf_size_of_phdr(class);
+
+ ehdr = elf_data;
+ phnum = elf_hdr_get_e_phnum(class, ehdr);
+ phdr = elf_data + elf_hdr_get_e_phoff(class, ehdr);
+
+ /* go through the available ELF segments */
+ for (i = 0; i < phnum; i++, phdr += elf_phdr_get_size) {
+ u64 da = elf_phdr_get_p_paddr(class, phdr);
+ u64 memsz = elf_phdr_get_p_memsz(class, phdr);
+ u64 filesz = elf_phdr_get_p_filesz(class, phdr);
+ u64 offset = elf_phdr_get_p_offset(class, phdr);
+ u32 type = elf_phdr_get_p_type(class, phdr);
+ bool is_iomem = false;
+ void *ptr;
+
+ if (type != PT_LOAD || !memsz)
+ continue;
+
+ dev_dbg(dev, "phdr: type %d da 0x%llx memsz 0x%llx filesz 0x%llx\n",
+ type, da, memsz, filesz);
+
+ if (filesz > memsz) {
+ dev_err(dev, "bad phdr filesz 0x%llx memsz 0x%llx\n",
+ filesz, memsz);
+ ret = -EINVAL;
+ break;
+ }
+
+ if (offset + filesz > fw->size) {
+ dev_err(dev, "truncated fw: need 0x%llx avail 0x%zx\n",
+ offset + filesz, fw->size);
+ ret = -EINVAL;
+ break;
+ }
+
+ if (!rproc_u64_fit_in_size_t(memsz)) {
+ dev_err(dev, "size (%llx) does not fit in size_t type\n",
+ memsz);
+ ret = -EOVERFLOW;
+ break;
+ }
+
+ /* grab the kernel address for this device address */
+ ptr = rproc_da_to_va(rproc, da, memsz, &is_iomem);
+ if (!ptr) {
+ dev_err(dev, "bad phdr da 0x%llx mem 0x%llx\n", da,
+ memsz);
+ ret = -EINVAL;
+ break;
+ }
+
+ /* put the segment where the remote processor expects it */
+ if (filesz) {
+ ret = imx_dsp_rproc_memcpy(ptr, elf_data + offset, filesz);
+ if (ret) {
+ dev_err(dev, "memory copy failed for da 0x%llx memsz 0x%llx\n",
+ da, memsz);
+ break;
+ }
+ }
+ }
+
+ return ret;
+}
Hi @Sanket_Parekh ,
Sorry for the delay.
I use iMX8M Nano + 512MB DDR with SDK 2.11 and I modified MIMX8MN4xxxxx_cm7_ram.ld as shown in the original post with the following intends :
- put data and code in TCMs as much as possible for performance
- increase heap size from 0x400 to 0x4000 and reallocate it in DDR (which separate it from stack)
I reserved memory segments in DTS (Linux 5.15.32) to match TCMs and first part of DDR (4000_0000 - 4048_0000 as shown in ld script also) and try to load the elf with remoteproc, which produce the following kernel panic at memset in imx_rproc_elf_load_segments :
[ 182.030838] remoteproc remoteproc0: powering up imx-rproc
[ 182.045600] remoteproc remoteproc0: Booting fw image fw_m7.elf, size 648712
[ 182.055082] Unable to handle kernel paging request at virtual address ffff80000aa06040
[ 182.063038] Mem abort info:
[ 182.065831] ESR = 0x96000061
[ 182.068949] EC = 0x25: DABT (current EL), IL = 32 bits
[ 182.074269] SET = 0, FnV = 0
[ 182.077324] EA = 0, S1PTW = 0
[ 182.080483] FSC = 0x21: alignment fault
[ 182.084499] Data abort info:
[ 182.087386] ISV = 0, ISS = 0x00000061
[ 182.091239] CM = 0, WnR = 1
[ 182.094217] swapper pgtable: 4k pages, 48-bit VAs, pgdp=00000000412da000
[ 182.100926] [ffff80000aa06040] pgd=100000005ffff003, p4d=100000005ffff003, pud=100000005fffe003, pmd=0068000040200711
[ 182.111572] Internal error: Oops: 96000061 [#1] PREEMPT SMP
[ 182.117148] Modules linked in: imx_rpmsg_tty
[ 182.121431] CPU: 1 PID: 864 Comm: mtr_app Not tainted 5.15.32-
[ 182.129090] Hardware name: proto (DT)
[ 182.136489] pstate: 80000005 (Nzcv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
[ 182.143452] pc : __memset+0x16c/0x188
[ 182.147139] lr : rproc_elf_load_segments+0x12c/0x250
[ 182.152125] sp : ffff80000a37bb20
[ 182.155437] x29: ffff80000a37bb20 x28: 0000000000008000 x27: 0000000000000000
[ 182.162580] x26: 0000000040206018 x25: 0000000000006018 x24: 0000000000000004
[ 182.169721] x23: ffff0000021bf000 x22: 0000000000000020 x21: 0000000000000005
[ 182.176863] x20: 0000000000000001 x19: ffff8000094310b4 x18: 0000000000000000
[ 182.184004] x17: 0000000000000008 x16: 0000100058000000 x15: 0000000000000000
[ 182.191145] x14: 0000000000000001 x13: ffff0000021bf4c0 x12: 000000000000000d
[ 182.198286] x11: 0000000000206018 x10: 0000000040206018 x9 : 0000000000000000
[ 182.205426] x8 : ffff80000aa06040 x7 : 0000000000000000 x6 : 000000000000003f
[ 182.212568] x5 : 0000000000000040 x4 : ffffffffffffffe0 x3 : 0000000000007fd8
[ 182.219709] x2 : 0000000000007f98 x1 : 0000000000000000 x0 : ffff80000aa06018
[ 182.226851] Call trace:
[ 182.229298] __memset+0x16c/0x188
[ 182.232615] imx_rproc_elf_load_segments+0x24/0x40
[ 182.237408] rproc_start+0x30/0x168
[ 182.240905] rproc_boot+0x344/0x5f0
[ 182.244397] state_store+0x44/0x104
[ 182.247889] dev_attr_store+0x1c/0x30
[ 182.251565] sysfs_kf_write+0x48/0x60
[ 182.255235] kernfs_fop_write_iter+0x118/0x1b0
[ 182.259681] new_sync_write+0xe8/0x184
[ 182.263439] vfs_write+0x20c/0x284
[ 182.266842] ksys_write+0x68/0xf4
[ 182.270159] __arm64_sys_write+0x20/0x2c
[ 182.274084] invoke_syscall+0x48/0x114
[ 182.277842] el0_svc_common.constprop.0+0x44/0xfc
[ 182.282548] do_el0_svc+0x20/0x30
[ 182.285863] el0_svc+0x18/0x54
[ 182.288922] el0t_64_sync_handler+0xa4/0x130
[ 182.293196] el0t_64_sync+0x1a0/0x1a4
[ 182.296870] Code: 91010108 54ffff4a 8b040108 cb050042 (d50b7428)
[ 182.302968] ---[ end trace cde97d89a7f35765 ]---
[ 182.307589] Kernel panic - not syncing: Oops: Fatal exception
[ 182.313336] SMP: stopping secondary CPUs
[ 182.317564] Kernel Offset: disabled
[ 182.321052] CPU features: 0x00002001,20000842
[ 182.325412] Memory Limit: none
[ 182.328510] ------------[ cut here ]------------
Thanks,
Yann
Hi @ycx ,
Hi @Sanket_Parekh ,
Thank you for your point.
You are right, however most of this size is debug symbols that are not copied by remoteproc when loaded (sorry, I should have shown release version). As per driver code, the elf loader only parse program headers which fit in targeted memory regions :
fw_m7.elf: file format elf32-little
Program Header:
LOAD off 0x00010000 vaddr 0x00000000 paddr 0x00000000 align 2**16 // this is ITCM
filesz 0x00000240 memsz 0x00000240 flags r--
LOAD off 0x00010240 vaddr 0x00000240 paddr 0x00000240 align 2**16 // still ITCM
filesz 0x000111e0 memsz 0x000111e0 flags rwx
LOAD off 0x00030000 vaddr 0x20000000 paddr 0x20000000 align 2**16 // DTCM
filesz 0x00000074 memsz 0x0001fb88 flags rw-
LOAD off 0x00040000 vaddr 0x40000000 paddr 0x40000000 align 2**16 // DDR data
filesz 0x00006018 memsz 0x00006018 flags rw-
LOAD off 0x00006018 vaddr 0x40006018 paddr 0x40006018 align 2**16 // DDR relocated heap
filesz 0x00000000 memsz 0x00004000 flags rw-
When I try with the release build it produces the same behaviour.
Thanks,
Yann
Hi @ycx ,
/* * Zero out remaining memory for this segment. * * This isn't strictly required since dma_alloc_coherent already * did this for us. albeit harmless, we may consider removing * this. */ if (memsz > filesz) { if (is_iomem) memset_io((void __iomem *)(ptr + filesz), 0, memsz - filesz); else memset(ptr + filesz, 0, memsz - filesz); }
Hi @Sanket_Parekh ,
Thank you for the advice. It works when memset is commented in the driver.
However I tried also after a fixed alignment of the data (align(8)), with such :
Program Headers:
Type Offset VirtAddr PhysAddr FileSiz MemSiz Flg Align
LOAD 0x010000 0x00000000 0x00000000 0x00240 0x00240 R 0x10000
LOAD 0x010240 0x00000240 0x00000240 0x11218 0x11218 RWE 0x10000
LOAD 0x030000 0x20000000 0x20000000 0x00078 0x1fba8 RW 0x10000
LOAD 0x040000 0x40000000 0x40000000 0x06020 0x06020 RW 0x10000
LOAD 0x006020 0x40006020 0x40006020 0x00000 0x04000 RW 0x10000
And I still get the following panic. By curiosity I would like to know why if the problem is not bad align ?
[ 59.076947] remoteproc remoteproc0: powering up imx-rproc
[ 59.084182] remoteproc remoteproc0: Booting fw image fw_m7.elf, size 329056
[ 59.093339] Unable to handle kernel paging request at virtual address ffff80000a006040
[ 59.101289] Mem abort info:
[ 59.104133] ESR = 0x96000061
[ 59.107204] EC = 0x25: DABT (current EL), IL = 32 bits
[ 59.112534] SET = 0, FnV = 0
[ 59.115607] EA = 0, S1PTW = 0
[ 59.118761] FSC = 0x21: alignment fault
[ 59.122780] Data abort info:
[ 59.125660] ISV = 0, ISS = 0x00000061
[ 59.129505] CM = 0, WnR = 1
[ 59.132479] swapper pgtable: 4k pages, 48-bit VAs, pgdp=00000000412da000
[ 59.139204] [ffff80000a006040] pgd=100000005ffff003, p4d=100000005ffff003, pud=100000005fffe003, pmd=0068000040000711
[ 59.149851] Internal error: Oops: 96000061 [#1] PREEMPT SMP
[ 59.155426] Modules linked in: imx_rpmsg_tty
[ 59.159707] CPU: 1 PID: 883 Comm: app Not tainted 5.15.32- #1
[ 59.167367] Hardware name: Prototype (DT)
[ 59.174765] pstate: 80000005 (Nzcv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
[ 59.181730] pc : __memset+0x16c/0x188
[ 59.185417] lr : rproc_elf_load_segments+0x12c/0x250
[ 59.190402] sp : ffff8000093cbb20
[ 59.193715] x29: ffff8000093cbb20 x28: 0000000000004000 x27: 0000000000000000
[ 59.200858] x26: 0000000040006020 x25: 0000000000006020 x24: 0000000000000004
[ 59.207998] x23: ffff000002e36000 x22: 0000000000000020 x21: 0000000000000005
[ 59.215139] x20: 0000000000000001 x19: ffff8000094bd0b4 x18: 0000000000000000
[ 59.222280] x17: 0000000000000008 x16: 0000100058000000 x15: 0000000000000000
[ 59.229422] x14: 0000000000000001 x13: ffff000002e364c0 x12: 000000000000000d
[ 59.236563] x11: 0000000000006020 x10: 0000000040006020 x9 : 0000000000000000
[ 59.243704] x8 : ffff80000a006040 x7 : 0000000000000000 x6 : 000000000000003f
[ 59.250845] x5 : 0000000000000040 x4 : ffffffffffffffe0 x3 : 0000000000003fe0
[ 59.257986] x2 : 0000000000003fa0 x1 : 0000000000000000 x0 : ffff80000a006020
[ 59.265129] Call trace:
[ 59.267574] __memset+0x16c/0x188
[ 59.270890] imx_rproc_elf_load_segments+0x24/0x40
[ 59.275683] rproc_start+0x30/0x168
[ 59.279181] rproc_boot+0x344/0x5f0
[ 59.282674] state_store+0x44/0x104
[ 59.286165] dev_attr_store+0x1c/0x30
[ 59.289842] sysfs_kf_write+0x48/0x60
[ 59.293512] kernfs_fop_write_iter+0x118/0x1b0
[ 59.297957] new_sync_write+0xe8/0x184
[ 59.301716] vfs_write+0x20c/0x284
[ 59.305120] ksys_write+0x68/0xf4
[ 59.308437] __arm64_sys_write+0x20/0x2c
[ 59.312363] invoke_syscall+0x48/0x114
[ 59.316120] el0_svc_common.constprop.0+0x44/0xfc
[ 59.320826] do_el0_svc+0x20/0x30
[ 59.324142] el0_svc+0x18/0x54
[ 59.327202] el0t_64_sync_handler+0xa4/0x130
[ 59.331475] el0t_64_sync+0x1a0/0x1a4
[ 59.335150] Code: 91010108 54ffff4a 8b040108 cb050042 (d50b7428)
[ 59.341249] ---[ end trace cb9b3b48f641547f ]---
[ 59.345870] Kernel panic - not syncing: Oops: Fatal exception
[ 59.351617] SMP: stopping secondary CPUs
[ 59.355843] Kernel Offset: disabled
[ 59.359330] CPU features: 0x00002001,20000842
[ 59.363690] Memory Limit: none
[ 59.366784] ------------[ cut here ]------------
[ 59.371400] Voluntary context switch within RCU read-side critical section!
[ 59.371435] WARNING: CPU: 1 PID: 883 at kernel/rcu/tree_plugin.h:316 rcu_note_context_switch+0x344/0x3a4
[ 59.387888] Modules linked in: imx_rpmsg_tty
[ 59.392162] CPU: 1 PID: 883 Comm: app Tainted: G D 5.15.32- #1
[ 59.401207] Hardware name: Prototype (DT)
[ 59.408602] pstate: 600000c5 (nZCv daIF -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
[ 59.415565] pc : rcu_note_context_switch+0x344/0x3a4
[ 59.420531] lr : rcu_note_context_switch+0x344/0x3a4
[ 59.425498] sp : ffff8000093cb1d0
[ 59.428812] x29: ffff8000093cb1d0 x28: 0000000000000002 x27: ffff000002eb8880
[ 59.435954] x26: 0000000000000000 x25: 0000000000000002 x24: 0000000000000002
[ 59.443095] x23: 0000000000000000 x22: 0000000000000000 x21: ffff0000029f8000
[ 59.450236] x20: 0000000000000000 x19: ffff00001feb2300 x18: ffffffffffff9f38
[ 59.457377] x17: 3230363030613030 x16: 3030386666666620 x15: 0000000000000048
[ 59.464519] x14: 0000000000000000 x13: 000000000000053c x12: ffff8000093caee0
[ 59.471660] x11: ffffffffffff9f38 x10: ffffffffffff9ef0 x9 : ffff800008ed8b48
[ 59.478801] x8 : 00000000ffffefff x7 : ffff800008f30b48 x6 : 000000000001cca0
[ 59.485943] x5 : 0000000000000001 x4 : 0000000000000000 x3 : 0000000000000027
[ 59.493084] x2 : 0000000000000000 x1 : 0000000000000000 x0 : ffff0000029f8000
[ 59.500225] Call trace:
[ 59.502670] rcu_note_context_switch+0x344/0x3a4
[ 59.507289] __schedule+0x8c/0x530
[ 59.510694] schedule+0x44/0xf0
[ 59.513836] schedule_hrtimeout_range_clock+0xa4/0x144
[ 59.518982] schedule_hrtimeout_range+0x14/0x20
[ 59.523515] usleep_range_state+0x74/0xb0
[ 59.527527] i2c_imx_start+0xb0/0x180
[ 59.531197] i2c_imx_xfer_common+0x38/0xee4
[ 59.535381] i2c_imx_xfer+0x19c/0x350
[ 59.539044] __i2c_transfer+0x164/0x4d4
[ 59.542890] i2c_smbus_xfer_emulated+0x10c/0x710
[ 59.547509] __i2c_smbus_xfer+0x108/0x1f0
[ 59.551519] i2c_smbus_xfer+0x80/0x120
[ 59.555270] i2c_smbus_read_byte_data+0x3c/0x70
[ 59.559801] rn5t618_trigger_poweroff_sequence+0x24/0xa4
[ 59.565119] rn5t618_restart+0x14/0x30
[ 59.568868] atomic_notifier_call_chain+0x60/0x90
[ 59.573582] do_kernel_restart+0x28/0x3c
[ 59.577508] machine_restart+0x24/0x34
[ 59.581261] emergency_restart+0x1c/0x30
[ 59.585187] panic+0x2a0/0x314
[ 59.588249] die+0x1b4/0x210
[ 59.591131] die_kernel_fault+0x64/0x74
[ 59.594976] __do_kernel_fault+0x90/0x18c
[ 59.598988] do_alignment_fault+0x68/0x7c
[ 59.603000] do_mem_abort+0x44/0xb4
[ 59.606491] el1_abort+0x40/0x6c
[ 59.609721] el1h_64_sync_handler+0xa4/0xd0
[ 59.613905] el1h_64_sync+0x78/0x7c
[ 59.617395] __memset+0x16c/0x188
[ 59.620713] imx_rproc_elf_load_segments+0x24/0x40
[ 59.625507] rproc_start+0x30/0x168
[ 59.628997] rproc_boot+0x344/0x5f0
[ 59.632489] state_store+0x44/0x104
[ 59.635980] dev_attr_store+0x1c/0x30
[ 59.639651] sysfs_kf_write+0x48/0x60
[ 59.643314] kernfs_fop_write_iter+0x118/0x1b0
[ 59.647760] new_sync_write+0xe8/0x184
[ 59.651511] vfs_write+0x20c/0x284
[ 59.654916] ksys_write+0x68/0xf4
[ 59.658232] __arm64_sys_write+0x20/0x2c
[ 59.662158] invoke_syscall+0x48/0x114
[ 59.665908] el0_svc_common.constprop.0+0x44/0xfc
[ 59.670613] do_el0_svc+0x20/0x30
[ 59.673930] el0_svc+0x18/0x54
[ 59.676986] el0t_64_sync_handler+0xa4/0x130
[ 59.681258] el0t_64_sync+0x1a0/0x1a4
[ 59.684922] ---[ end trace cb9b3b48f6415480 ]---
Thank you
Hi @ycx ,
Hi @ycx ,
I hope you are doing well.
can you please share the step you have followed and the error message you got?
Thanks & Regards
Sanket Parekh