Hi All,

We had a hanging qemu process:


https://autobuilder.yoctoproject.org/typhoon/#/builders/96/builds/1748

This is on ubuntu1804-arm-1.yocto.io.

I was able to ssh into the image on the worker with ssh root@....2
and it looks to be hung during fs ltp tests. I've dumped some 
output below, I'm not too familiar with ltp so struggling to understand
where it is at.

The processes are still around so we may be able to save data from
them but the servers will be rebooted tomorrow during maint at which
point we'd lose this unless we ask Michael to defer.

It looks like the proc01 -m 128 is hanging.

Cheers,

Richard

root@qemuarm64:~# ps ax
PID TTY STAT TIME COMMAND
1 ? Ss 1:45 init [5]
2 ? S 0:00 [kthreadd]
3 ? I< 0:00 [rcu_gp]
4 ? I< 0:00 [rcu_par_gp]
6 ? I< 0:00 [kworker/0:0H-events_highpri]
8 ? I< 0:00 [mm_percpu_wq]
9 ? S 0:00 [rcu_tasks_kthre]
10 ? S 0:00 [rcu_tasks_rude_]
11 ? S 0:00 [rcu_tasks_trace]
12 ? S 0:42 [ksoftirqd/0]
13 ? I 0:08 [rcu_preempt]
14 ? S 0:00 [migration/0]
15 ? S 0:00 [cpuhp/0]
16 ? S 0:00 [kdevtmpfs]
17 ? I< 0:00 [netns]
19 ? S 0:00 [oom_reaper]
20 ? I< 0:00 [writeback]
21 ? S 0:04 [kcompactd0]
54 ? I< 0:00 [kblockd]
55 ? I< 0:00 [blkcg_punt_bio]
56 ? I< 0:00 [md]
57 ? I< 0:06 [kworker/0:1H-kblockd]
58 ? I< 0:00 [rpciod]
59 ? I< 0:00 [kworker/u3:0]
60 ? I< 0:00 [xprtiod]
61 ? S 0:01 [kswapd0]
62 ? I< 0:00 [nfsiod]
63 ? I< 0:00 [cifsiod]
64 ? I< 0:00 [smb3decryptd]
65 ? I< 0:00 [cifsfileinfoput]
66 ? I< 0:00 [cifsoplockd]
68 ? S 0:05 [hwrng]
69 ? I< 0:00 [ttm_swap]
71 ? S 0:00 [khvcd]
73 ? I< 0:00 [raid5wq]
74 ? I< 0:00 [dm_bufio_cache]
75 ? I< 0:00 [ipv6_addrconf]
80 ? S 0:24 [jbd2/vda-8]
81 ? I< 0:00 [ext4-rsv-conver]
116 ? Ss 0:00 /sbin/udevd -d
445 ? Ss 0:00 /usr/bin/dbus-daemon --system
453 ? Ss 3:47 /usr/sbin/connmand
460 ? S 0:00 xinit /etc/X11/Xsession -- /usr/bin/Xorg :0 -br -pn
463 ? S 0:58 /usr/sbin/wpa_supplicant -u
467 ? S<sl 1:07 /usr/bin/Xorg :0 -br -pn
471 ? Ss 0:00 /usr/sbin/dropbear -r /etc/dropbear/dropbear_rsa_host_key -p 22 -B
476 ? Ss 0:25 /usr/sbin/rpcbind
491 ? S 0:00 matchbox-window-manager -theme Sato -use_cursor yes
496 ? S 0:00 dbus-launch --sh-syntax --exit-with-session
497 ? Ss 0:00 /usr/bin/dbus-daemon --syslog --fork --print-pid 5 --print-address 7 --session
514 ? Sl 0:00 /usr/libexec/at-spi-bus-launcher --launch-immediately
518 ? S 0:00 connman-applet
534 ? S 0:00 /usr/bin/dbus-daemon --config-file=/usr/share/defaults/at-spi2/accessibility.conf --nofork --print-address 3
553 ? S 0:00 matchbox-desktop
554 ? S 10:50 matchbox-panel --start-applets showdesktop,windowselector --end-applets clock,,systray,startup-notify,notify
557 ? Sl 0:00 /usr/sbin/console-kit-daemon --no-daemon
621 ? S 1:41 /usr/libexec/gconfd-2
623 ? S 0:00 /usr/bin/settings-daemon
638 ? S 0:00 avahi-daemon: running [qemuarm64.local]
639 ? S 0:00 avahi-daemon: chroot helper
643 ? Ss 0:00 /usr/sbin/ofonod
648 ? Sl 0:00 /usr/libexec/at-spi2-registryd --use-gnome-session
650 ? Ss 1:22 /usr/sbin/crond
660 ? Ss 0:00 /bin/sh /bin/start_getty 115200 ttyAMA0 vt102
661 ? Ss 0:00 /bin/sh /bin/start_getty 115200 hvc0 vt102
662 tty1 Ss+ 0:00 /sbin/getty 38400 tty1
665 hvc0 Ss+ 0:00 /sbin/getty -L 115200 hvc0 vt102
666 ttyAMA0 Ss+ 0:00 -sh
7112 ? S 0:02 /usr/sbin/dropbear -r /etc/dropbear/dropbear_rsa_host_key -p 22 -B
7113 ? S 0:00 /bin/sh /opt/ltp/runltp -f fs -p -q -r /opt/ltp -l /opt/ltp/results/fs -I 1 -d /opt/ltp
7155 ? S 0:00 /opt/ltp/bin/ltp-pan -q -e -S -a 7113 -n 7113 -p -f /opt/ltp/ltp-HZxaOUahq7/alltests -l /opt/ltp/results/fs -C /opt/ltp/output/LTP_RUN_ON-fs.failed -T /opt/ltp/output/LTP_RUN_ON-fs.tconf
9361 ? S 0:00 /sbin/syslogd -n -O /var/log/messages
9364 ? S 0:01 /sbin/klogd -n
11412 ? I 0:00 [kworker/u2:0-ext4-rsv-conversion]
11413 ? I 0:00 [kworker/u2:3-events_unbound]
11578 ? S 0:00 proc01 -m 128
11580 ? S 0:00 /usr/sbin/dropbear -r /etc/dropbear/dropbear_rsa_host_key -p 22 -B
11581 pts/0 Ss 0:00 -sh
11585 pts/0 R+ 0:00 ps ax
27489 ? I 0:00 [kworker/0:1-cgroup_pidlist_destroy]
27490 ? I 0:55 [kworker/0:4-events]

root@qemuarm64:~# tail /opt/ltp/results/fs
ftest02 PASS 0
ftest03 PASS 0
ftest04 PASS 0
ftest05 PASS 0
ftest06 PASS 0
ftest07 PASS 0
ftest08 PASS 0
lftest01 PASS 0
writetest01 PASS 0
fs_di PASS 0
root@qemuarm64:~# tail /opt/ltp/output/LTP_RUN_ON-fs.failed

Lots in dmesg but the last bit shows handy timestamp:

[10052.451191] cgroup: fork rejected by pids controller in /ltp_pids_9/child2
[10052.500603] new mount options do not match the existing superblock, will be ignored
[10052.543519] cgroup: fork rejected by pids controller in /ltp_pids_9/child2
[10052.619108] new mount options do not match the existing superblock, will be ignored
[10052.733366] cgroup: fork rejected by pids controller in /ltp_pids_9/child2
[10052.942680] new mount options do not match the existing superblock, will be ignored
[10053.146175] cgroup: fork rejected by pids controller in /ltp_pids_9/child2
[10092.243267] input: virtual-device-ltp as /devices/virtual/input/input2
[10092.327062] input: virtual-device-ltp as /devices/virtual/input/input3
[10092.422283] input: virtual-device-ltp as /devices/virtual/input/input4
[10092.436818] input: virtual-device-ltp as /devices/virtual/input/input5
[10092.517318] input: virtual-device-ltp as /devices/virtual/input/input6
[10092.596125] input: virtual-device-ltp as /devices/virtual/input/input7
[11205.107302] ICMPv6: process `proc01' is using deprecated sysctl (syscall) net.ipv6.neigh.default.base_reachable_time - use net.ipv6.neigh.default.base_reachable_time_ms instead