Meaning of backtrace - Debugging imx6 kernel crash

schadalavada · ‎06-08-2020

On OpenWRT 5.4.42 kernel on an imx6, I'm seeing a kernel crash when
loading builtin modules of igb driver and mv88e6xxx driver over PCI
bus. I've searched for the messages in the kernel code and Google but
its all over the place. What does this backtrace mean?

Modules linked in:
[ 2.631594][ T28] CPU: 2 PID: 28 Comm: kworker/u8:1 Not tainted
5.4.42 #0
[ 2.631599][ T28] Hardware name: Freescale i.MX6 Quad/DualLite
(Device Tree)
2.670498][ T32] Workqueue: events_unbound async_run_entry_fn
[ 2.676560][ T32] [<80016424>] (unwind_backtrace) from
[<80012a84>] (show_stack+0x10/0x14)
[ 2.685057][ T32] [<80012a84>] (show_stack) from [<8053cb30>]
(dump_stack+0x90/0xa4)
[ 2.693033][ T32] [<8053cb30>] (dump_stack) from [<800241fc>]
(__warn+0xbc/0xd8)
[ 2.700636][ T32] [<800241fc>] (__warn) from [<80024268>]
(warn_slowpath_fmt+0x50/0x94)
[ 2.708849][ T32] [<80024268>] (warn_slowpath_fmt) from
[<8004331c>] (__request_module+0xac/0x37c)
[ 2.718020][ T32] [<8004331c>] (__request_module) from
[<802eaeac>] (phy_request_driver_module+0x118/0x158)
[ 2.727967][ T32] [<802eaeac>] (phy_request_driver_module) from
[<802ec4ac>] (phy_device_create+0x1e4/0x204)
[ 2.737997][ T32] [<802ec4ac>] (phy_device_create) from
[<802ec628>] (get_phy_device+0x15c/0x1b0)
[ 2.747098][ T32] [<802ec628>] (get_phy_device) from [<803d7ab4>]
(of_mdiobus_register_phy+0x74/0x170)
[ 2.756615][ T32] [<803d7ab4>] (of_mdiobus_register_phy) from
[<803d8040>] (of_mdiobus_register+0x120/0x32c)
[ 2.766665][ T32] [<803d8040>] (of_mdiobus_register) from
[<802ef248>] (mv88e6xxx_mdio_register+0xdc/0x190)
[ 2.776617][ T32] [<802ef248>] (mv88e6xxx_mdio_register) from
[<802f10ac>] (mv88e6xxx_probe+0x618/0x714)
[ 2.786305][ T32] [<802f10ac>] (mv88e6xxx_probe) from
[<802edaf8>] (mdio_probe+0x30/0x54)
[ 2.794699][ T32] [<802edaf8>] (mdio_probe) from [<8027a034>]
(really_probe+0x1f0/0x364)
[ 2.802996][ T32] [<8027a034>] (really_probe) from [<8027a328>]
(driver_probe_device+0x60/0x170)
[ 2.811986][ T32] [<8027a328>] (driver_probe_device) from
[<802785c0>] (bus_for_each_drv+0x70/0x94)
[ 2.821231][ T32] [<802785c0>] (bus_for_each_drv) from
[<80279dd4>] (__device_attach+0xb4/0x11c)
[ 2.830218][ T32] [<80279dd4>] (__device_attach) from
[<802791d8>] (bus_probe_device+0x84/0x8c)
[ 2.839117][ T32] [<802791d8>] (bus_probe_device) from
[<802768d0>] (device_add+0x36c/0x614)
[ 2.847759][ T32] [<802768d0>] (device_add) from [<802edb9c>]
(mdio_device_register+0x24/0x48)
[ 2.856582][ T32] [<802edb9c>] (mdio_device_register) from
[<803d81b4>] (of_mdiobus_register+0x294/0x32c)
[ 2.866367][ T32] [<803d81b4>] (of_mdiobus_register) from
[<8032ccc8>] (igb_probe+0x1088/0x13cc)
[ 2.875385][ T32] [<8032ccc8>] (igb_probe) from [<8021f010>]
(pci_device_probe+0xd4/0x15c)
[ 2.883862][ T32] [<8021f010>] (pci_device_probe) from
[<8027a034>] (really_probe+0x1f0/0x364)
[ 2.892676][ T32] [<8027a034>] (really_probe) from [<8027a328>]
(driver_probe_device+0x60/0x170)
[ 2.901663][ T32] [<8027a328>] (driver_probe_device) from
[<802785c0>] (bus_for_each_drv+0x70/0x94)
[ 2.910910][ T32] [<802785c0>] (bus_for_each_drv) from
[<80279dd4>] (__device_attach+0xb4/0x11c)
[ 2.919917][ T32] [<80279dd4>] (__device_attach) from
[<80213bbc>] (pci_bus_add_device+0x44/0x90)
[ 2.928995][ T32] [<80213bbc>] (pci_bus_add_device) from
[<80213c34>] (pci_bus_add_devices+0x2c/0x70)
[ 2.938420][ T32] [<80213c34>] (pci_bus_add_devices) from
[<80213c68>] (pci_bus_add_devices+0x60/0x70)
[ 2.948007][ T32] ---[ end trace c8de08d4ca07a3ea ]---

with this device tree :
&pcie {
pinctrl-names = "default";
pinctrl-0 = <&pinctrl_pcie>;
reset-gpio =<&reset_ic 0 1>;
child-reset-gpios = <&reset_ic 3 1
&reset_ic 2 1
&reset_ic 1 1>;
#gpio-cells = <3>;
status = "okay";
fsl,max-link-speed = <2>;

/*
* PCI bridge: PLX Technology, Inc. PEX 8605 PCI Express
4-port Gen2 Switch
* Port 0 to CPU, Port 1 to igb and ports 2 and 3 are accessible through
* the half and full size mini PCIe slots on the board.
*/
pcie@1,0 {
/* Connection to CPU */
status = "okay";
fsl,max-link-speed = <2>;
};

pcie@2,1 {
/* Connection to Intel Gigabit Ethernet Controller*/
status = "okay";
fsl,max-link-speed = <1>;
pcie@3,0 {
/* The igb */
status = "okay";
fsl,max-link-speed = <1>;
eth0: igb0 {
compatible = "intel,igb";
/*pinctrl-names = "default";
pinctrl-0 = <&pinctrl_enet>;*/
phy-mode = "mii";
phy-handle = <&eth0>;
phy-reset-gpios = <&reset_ic 4 GPIO_ACTIVE_LOW>;
phy-reset-duration = <100>;
status = "okay";

mdio {
#address-cells = <1>;
#size-cells = <0>;
status = "okay";

switch: switch@0 {
compatible ="marvell,mv88e6085";
reg = <0>;
dsa,member = <0 0>;
eeprom-length = <512>;
interrupt-parent = <&gpio2>;
interrupts =<31IRQ_TYPE_LEVEL_LOW>;
interrupt-controller;
#interrupt-cells = <2>;

ports {
#address-cells = <1>;
#size-cells = <0>;

port@0 {
reg = <0>;
label = "port0";
phy-handle = <&switchphy0>;
};

port@1 {
reg = <1>;
label = "port1";
phy-handle = <&switchphy1>;
};

port@2 {
reg = <2>;
label = "port0";
phy-handle = <&switchphy2>;
};

port@5 {
reg = <5>;
label = "cpu";
ethernet = <&eth0>;
};
};

mdio {
#address-cells = <1>;
#size-cells = <0>;
switchphy0:switchphy@0 {
reg = <0>;
interrupt-parent = <&switch>;
interrupts =<0 IRQ_TYPE_LEVEL_HIGH>;
};

switchphy1:switchphy@1 {
reg = <1>;
interrupt-parent = <&switch>;
interrupts =<1 IRQ_TYPE_LEVEL_HIGH>;
};

switchphy2:switchphy@2 {
reg = <2>;
interrupt-parent = <&switch>;
interrupts =<2 IRQ_TYPE_LEVEL_HIGH>;
};
};
};
};

igorpadykov · ‎06-08-2020

Hi Sriram

one can try to debug it starting from probe() function using AN4553

Using Open Source Debugging Tools for Linux on i.MX Processors

As for meaning of backtrace logs, one can look at linux documentation

Tainted kernels — The Linux Kernel documentation

also it may be posted on kernel mail list

Majordomo Lists at VGER.KERNEL.ORG

Best regards
igor
-----------------------------------------------------------------------------------------------------------------------
Note: If this post answers your question, please click the Correct Answer button. Thank you!
-----------------------------------------------------------------------------------------------------------------------

emptyfridge

Hi all,

I think I'm facing similar issue with imx8mm kernel 5.10.72.

I try to introduce the LAN7430 via PCIe. So fare it works and I even get the LINK up and connected. But never the less I have the warning during the init in the request_module for mdio...

any experience with that issue? or any hints ?

thanks guys.

cheers

tom

[    4.733732] mdio_bus pci-0000:01:00.0:01: MDIO_MODULE_PREFIX: mdio:; phy_id: 2233909
[    4.747581] ------------[ cut here ]------------
[    4.752205] WARNING: CPU: 1 PID: 7 at kernel/kmod.c:136 __request_module+0x94/0x4a0
[    4.759858] Modules linked in:
[    4.762919] CPU: 1 PID: 7 Comm: kworker/u4:0 Not tainted 5.10.72-gf1cb49db931b-dirty #206
[    4.771093] Hardware name: Freescale i.MX8MM PORT.CORE (DT)
[    4.776671] Workqueue: events_unbound async_run_entry_fn
[    4.781985] pstate: 00000005 (nzcv daif -PAN -UAO -TCO BTYPE=--)
[    4.787990] pc : __request_module+0x94/0x4a0
[    4.792258] lr : __request_module+0x8c/0x4a0
[    4.796525] sp : ffff8000118632a0
[    4.799837] x29: ffff8000118632a0 x28: ffff000002a2af20
[    4.805151] x27: ffff800010e88e30 x26: ffff800010c39f80
[    4.810465] x25: ffff800010c39e08 x24: 0000000000000000
[    4.815778] x23: 0000000000000001 x22: ffff000002a37000
[    4.821091] x21: ffff000002f9a000 x20: ffff800011863590
[    4.826405] x19: ffff800010ebf2b8 x18: ffffffffffffffff
[    4.831718] x17: 0000000000000001 x16: 0000000000000000
[    4.837031] x15: 0000000000000000 x14: 0000000000000000
[    4.842344] x13: 0000000000000001 x12: 0000000000000001
[    4.847657] x11: 0000000000000000 x10: 0000000000000001
[    4.852970] x9 : 0000000000000000 x8 : ffff00000005d580
[    4.858283] x7 : 0000000000000000 x6 : 0000000000000000
[    4.863596] x5 : 0000000000000000 x4 : 0000000000000000
[    4.868909] x3 : 0000000000000000 x2 : 0000000000000000
[    4.874222] x1 : ffff8000100e7fb0 x0 : 0000000000000001
[    4.879536] Call trace:
[    4.881983] __request_module+0x94/0x4a0
[    4.885908] phy_request_driver_module+0x130/0x17c
[    4.890701] phy_device_create+0x280/0x2d0
[    4.894796] get_phy_device+0x358/0x3a0
[    4.898631] mdiobus_scan+0x88/0x258
[    4.902207] __mdiobus_register+0x2d4/0x39c
[    4.906391] lan743x_pcidev_probe+0x44c/0x504
[    4.910750] local_pci_probe+0x40/0xac
[    4.914499] pci_device_probe+0xec/0x1a0
[    4.918421] really_probe+0xec/0x4c0
[    4.921995] driver_probe_device+0x58/0xc0
[    4.926090] __device_attach_driver+0xa8/0x10c
[    4.930535] bus_for_each_drv+0x78/0xd0
[    4.934369] __device_attach+0xdc/0x180
[    4.938203] device_attach+0x14/0x20
[    4.941779] pci_bus_add_device+0xc0/0x14c
[    4.945875] pci_bus_add_devices+0x78/0x128
[    4.950058] pci_bus_add_devices+0xec/0x128
[    4.954240] pci_host_probe+0x12c/0x160
[    4.958076] dw_pcie_host_init+0x540/0x5f4
[    4.962174] imx6_pcie_probe+0x11ac/0x128c
[    4.966269] platform_drv_probe+0x54/0xb0
[    4.970277] really_probe+0xec/0x4c0
[    4.973852] driver_probe_device+0x58/0xc0
[    4.977947] __device_attach_driver+0xa8/0x10c
[    4.982391] bus_for_each_drv+0x78/0xd0
[    4.986225] __device_attach_async_helper+0xac/0xd0
[    4.991103] async_run_entry_fn+0x44/0x150
[    4.995201] process_one_work+0x1cc/0x350
[    4.999210] worker_thread+0x13c/0x470
[    5.002958] kthread+0x154/0x160
[    5.006187] ret_from_fork+0x10/0x30
[    5.009760] ---[ end trace d2ae575134abe57c ]---

DT:

&pcie0{
   pinctrl-names = "default";
   pinctrl-0 = <&pinctrl_pcie0>;
   //disable-gpio = <&gpio5 5 GPIO_ACTIVE_LOW>;
   reset-gpio = <&gpio2 1 GPIO_ACTIVE_LOW>;
   clocks = <&clk IMX8MM_CLK_PCIE1_ROOT>,
        <&clk IMX8MM_CLK_PCIE1_AUX>,
        <&clk IMX8MM_CLK_PCIE1_PHY>,
        <&pcie0_refclk>;
   clock-names = "pcie", "pcie_aux", "pcie_phy", "pcie_bus";
   assigned-clocks = <&clk IMX8MM_CLK_PCIE1_AUX>,
             <&clk IMX8MM_CLK_PCIE1_PHY>,
             <&clk IMX8MM_CLK_PCIE1_CTRL>;
   assigned-clock-rates = <10000000>, <100000000>, <250000000>;
   assigned-clock-parents = <&clk IMX8MM_SYS_PLL2_50M>,
                <&clk IMX8MM_SYS_PLL2_100M>,
                <&clk IMX8MM_SYS_PLL2_250M>;
   l1ss-disabled;
   ext_osc = <1>;
   status = "okay";
};

kernel config:

# CONFIG_FSL_PQ_MDIO is not set
# CONFIG_FSL_XGMAC_MDIO is not set
CONFIG_FSL_ENETC_MDIO=y
CONFIG_MDIO_DEVICE=y
CONFIG_MDIO_BUS=y
CONFIG_FWNODE_MDIO=y
CONFIG_OF_MDIO=y
CONFIG_MDIO_DEVRES=y
# CONFIG_MDIO_BITBANG is not set
# CONFIG_MDIO_BCM_UNIMAC is not set
# CONFIG_MDIO_HISI_FEMAC is not set
# CONFIG_MDIO_MVUSB is not set
# CONFIG_MDIO_MSCC_MIIM is not set
# CONFIG_MDIO_OCTEON is not set
# CONFIG_MDIO_IPQ4019 is not set
# CONFIG_MDIO_IPQ8064 is not set
# CONFIG_MDIO_THUNDER is not set
# MDIO Multiplexers
# CONFIG_MDIO_BUS_MUX_GPIO is not set
# CONFIG_MDIO_BUS_MUX_MULTIPLEXER is not set
# CONFIG_MDIO_BUS_MUX_MMIOREG is not set

CONFIG_BLK_MQ_PCI=y
CONFIG_HAVE_PCI=y
CONFIG_PCI=y
CONFIG_PCI_DOMAINS=y
CONFIG_PCI_DOMAINS_GENERIC=y
CONFIG_PCI_SYSCALL=y
CONFIG_PCIEASPM=y
CONFIG_PCIEASPM_DEFAULT=y
CONFIG_PCI_MSI=y
CONFIG_PCI_MSI_IRQ_DOMAIN=y
CONFIG_PCI_QUIRKS=y
CONFIG_PCIE_BUS_DEFAULT=y
CONFIG_PCIE_DW=y
CONFIG_PCIE_DW_HOST=y
CONFIG_PCI_IMX6=y
CONFIG_PCI_IMX6_HOST=y
CONFIG_SERIAL_8250_PCI=y
CONFIG_SND_PCI=y
CONFIG_USB_PCI=y
CONFIG_USB_XHCI_PCI=y
CONFIG_USB_EHCI_PCI=y
CONFIG_USB_OHCI_HCD_PCI=y
CONFIG_USB_CHIPIDEA_PCI=y
CONFIG_ARM_GIC_V3_ITS_PCI=y
CONFIG_PHY_FSL_IMX_PCIE=y
CONFIG_GENERIC_PCI_IOMAP=y

CONFIG_LAN743X=y

Meaning of backtrace - Debugging imx6 kernel crash

Meaning of backtrace - Debugging imx6 kernel crash

i.MXRT