Unread portion of the kernel message buffer: panic: mutex sleepq chain (0xffffffff8102c460) not owned at ../../../kern/subr_sleepqueue.c:641 cpuid = 0 KDB: enter: panic db> bt Tracing pid 92 tid 100091 td 0xffffff000274c6a0 kdb_enter_why() at kdb_enter_why+0x3d panic() at panic+0x176 _mtx_assert() at _mtx_assert+0xe2 sleepq_resume_thread() at sleepq_resume_thread+0xa3 sleepq_broadcast() at sleepq_broadcast+0x84 unlock_sx() at unlock_sx+0x69 _cv_timedwait() at _cv_timedwait+0x157 txg_thread_wait() at txg_thread_wait+0x47 txg_timelimit_thread() at txg_timelimit_thread+0x63 fork_exit() at fork_exit+0x12a fork_trampoline() at fork_trampoline+0xe --- trap 0, rip = 0, rsp = 0xffffffffbfcb5d30, rbp = 0 --- db> show lock 0xffffffff8102c460 class: spin mutex name: sleepq chain flags: {SPIN, RECURSE} state: {OWNED} owner: 0xffffff000274e000 (tid 100089, pid 90, "txg_thread_enter") db> bt 90 Tracing pid 90 tid 100089 td 0xffffff000274e000 cpustop_handler() at cpustop_handler+0x40 ipi_nmi_handler() at ipi_nmi_handler+0x30 trap() at trap+0x378 nmi_calltrap() at nmi_calltrap+0x8 --- trap 0x13, rip = 0xffffffff80485f57, rsp = 0xffffffffbcbddff0, rbp = 0xffffffffbfcabaa0 --- _thread_lock_flags() at _thread_lock_flags+0xb7 sleepq_wait() at sleepq_wait+0x3c _cv_wait() at _cv_wait+0x15e txg_thread_wait() at txg_thread_wait+0x5f txg_quiesce_thread() at txg_quiesce_thread+0x9c fork_exit() at fork_exit+0x12a fork_trampoline() at fork_trampoline+0xe --- trap 0, rip = 0, rsp = 0xffffffffbfcabd30, rbp = 0 --- db> show alllocks Process 6290 (sshd) thread 0xffffff001b9ae6a0 (100174) Process 491 (pflogd) thread 0xffffff0002745000 (100097) exclusive sx so_rcv_sx r = 0 (0xffffff001b9db3b8) locked @ kern/uipc_sockbuf.c:148 exclusive sx so_rcv_sx r = 0 (0xffffff0002b1b670) locked @ kern/uipc_sockbuf.c:148 db> bt 6290 Tracing pid 6290 tid 100174 td 0xffffff001b9ae6a0 sched_switch() at sched_switch+0x2a1 mi_switch() at mi_switch+0x233 sleepq_switch() at sleepq_switch+0xe5 sleepq_catch_signals() at sleepq_catch_signals+0x299 sleepq_wait_sig() at sleepq_wait_sig+0x16 _sleep() at _sleep+0x309 soreceive_generic() at soreceive_generic+0xffd dofileread() at dofileread+0xa1 kern_readv() at kern_readv+0x4c read() at read+0x54 syscall() at syscall+0x1ce Xfast_syscall() at Xfast_syscall+0xab --- syscall (3, FreeBSD ELF64, read), rip = 0x80170cefc, rsp = 0x7fffffffe548, rbp = 0x4 --- db> bt 491 Tracing pid 491 tid 100097 td 0xffffff0002745000 sched_switch() at sched_switch+0x2a1 mi_switch() at mi_switch+0x233 sleepq_switch() at sleepq_switch+0xe5 sleepq_catch_signals() at sleepq_catch_signals+0x299 sleepq_wait_sig() at sleepq_wait_sig+0x16 _sleep() at _sleep+0x309 soreceive_generic() at soreceive_generic+0xffd dofileread() at dofileread+0xa1 kern_readv() at kern_readv+0x4c read() at read+0x54 syscall() at syscall+0x1ce Xfast_syscall() at Xfast_syscall+0xab --- syscall (3, FreeBSD ELF64, read), rip = 0x800956efc, rsp = 0x7fffffffe938, rbp = 0x4 --- db> show sleepq 0xffffffff8102c460 Unable to locate a sleep queue via 0xffffffff8102c460 db> show lockq 0xffffffff8102c460 No such command db> show lock queue 0xffffffff8102c460 Symbol not found db> show lock 0xffffffff8102c460 class: spin mutex name: sleepq chain flags: {SPIN, RECURSE} state: {OWNED} owner: 0xffffff000274e000 (tid 100089, pid 90, "txg_thread_enter") db> show witness Sleep locks: 0 md bio queue -- last acquired @ dev/md/md.c:700 0 ls_lock -- last acquired @ kern/kern_lockf.c:617 2 owner graph lock -- last acquired @ kern/kern_lockf.c:1379 15 UMA zone -- last acquired @ vm/uma_core.c:2257 1 lock owners lock -- last acquired @ kern/kern_lockf.c:364 15 UMA zone -- (already displayed) 2 owner graph lock -- (already displayed) 15 UMA zone -- (already displayed) 0 so_rcv_sx -- last acquired @ kern/uipc_sockbuf.c:148 10 so_rcv -- last acquired @ kern/uipc_socket.c:1424 16 sellck -- last acquired @ kern/sys_generic.c:759 11 radix node head -- last acquired @ net/route.c:147 12 rtentry -- last acquired @ netinet/ip_output.c:596 13 ifaddr -- last acquired @ net/route.c:813 15 UMA zone -- (already displayed) 13 sctp-addr -- last acquired @ netinet/sctp_pcb.c:649 14 vm page queue free mutex -- last acquired @ vm/vm_page.c:1316 13 sctp_it_wq -- last acquired @ netinet/sctputil.c:1345 13 rts_inq -- last acquired @ net/netisr.c:140 12 ifnet -- last acquired @ contrib/altq/altq/altq_subr.c:461 15 UMA zone -- (already displayed) 13 eventhandler -- last acquired @ kern/subr_eventhandler.c:212 14 eventhandler list -- last acquired @ kern/kern_exit.c:233 14 if_addr_mtx -- last acquired @ netinet6/ip6_output.c:717 15 UMA zone -- (already displayed) 13 pf task mtx -- last acquired @ /usr/src/sys/modules/pf/../../contrib/pf/net/pf.c:980 15 UMA zone -- (already displayed) 14 vm page queue free mutex -- (already displayed) 14 arc4_mtx -- last acquired @ libkern/arc4random.c:137 14 if send queue -- last acquired @ dev/re/if_re.c:2351 14 random reseed -- last acquired @ dev/random/yarrow.c:278 14 eventhandler list -- (already displayed) 13 network driver -- last acquired @ kern/kern_timeout.c:241 15 UMA zone -- (already displayed) 14 vm page queue free mutex -- (already displayed) 14 UMA boot pages -- last acquired @ vm/uma_core.c:916 14 if_addr_mtx -- (already displayed) 14 taskqueue -- last acquired @ kern/subr_taskqueue.c:73 14 if send queue -- (already displayed) 14 bpf interface lock -- last acquired @ net/bpf.c:1358 15 bpf cdev lock -- last acquired @ net/bpf.c:488 16 sellck -- (already displayed) 14 arc4_mtx -- (already displayed) 15 UMA zone -- (already displayed) 14 vm page queue free mutex -- (already displayed) 11 process lock -- last acquired @ kern/subr_sleepqueue.c:390 12 session -- last acquired @ kern/kern_proc.c:588 13 uidinfo hash -- last acquired @ kern/kern_resource.c:1283 14 uidinfo struct -- last acquired @ order list:0 14 sleep mtxpool -- last acquired @ kern/kern_event.c:993 15 kqueue -- last acquired @ kern/kern_event.c:1182 16 struct mount mtx -- last acquired @ kern/vfs_vnops.c:1046 17 vnode interlock -- last acquired @ kern/vfs_subr.c:2304 18 cdev -- last acquired @ kern/kern_conf.c:71 18 vnode_free_list -- last acquired @ kern/vfs_subr.c:3012 18 Syncer mtx -- last acquired @ kern/vfs_subr.c:1681 13 tty -- last acquired @ kern/kern_event.c:1664 17 vnode interlock -- (already displayed) 12 sigacts -- last acquired @ kern/subr_sleepqueue.c:392 12 ktrace -- last acquired @ kern/kern_fork.c:600 14 sleep mtxpool -- (already displayed) 12 fdesc -- last acquired @ kern/kern_descrip.c:1539 14 sleep mtxpool -- (already displayed) 15 UMA zone -- (already displayed) 15 kqueue -- (already displayed) 14 vm page queue free mutex -- (already displayed) 11 process lock -- (already displayed) 2 unp_mtx -- last acquired @ kern/uipc_usrreq.c:725 8 accept -- last acquired @ kern/uipc_socket.c:685 9 so_snd -- last acquired @ kern/uipc_socket.c:2476 10 so_rcv -- (already displayed) 14 sleep mtxpool -- (already displayed) 11 process lock -- (already displayed) 11 radix node head -- (already displayed) 12 rtentry -- (already displayed) 10 tcp_hc_entry -- last acquired @ netinet/tcp_hostcache.c:346 15 UMA zone -- (already displayed) 14 vm page queue free mutex -- (already displayed) 16 sellck -- (already displayed) 15 UMA zone -- (already displayed) 15 kqueue -- (already displayed) 14 vm page queue free mutex -- (already displayed) 10 so_rcv -- (already displayed) 10 so_rcv -- (already displayed) 9 so_snd -- (already displayed) 15 UMA zone -- (already displayed) 4 filedesc structure -- last acquired @ kern/kern_descrip.c:2054 17 vnode interlock -- (already displayed) 12 fdesc -- (already displayed) 11 process lock -- (already displayed) 14 sleep mtxpool -- (already displayed) 6 Name Cache -- last acquired @ kern/vfs_cache.c:327 17 vnode interlock -- (already displayed) 15 UMA zone -- (already displayed) 14 vm page queue free mutex -- (already displayed) 15 UMA zone -- (already displayed) 18 cdev -- (already displayed) 5 Giant -- last acquired @ kern/kern_timeout.c:241 6 pipe mutex -- last acquired @ kern/sys_pipe.c:1331 7 sigio lock -- last acquired @ kern/kern_descrip.c:849 8 process group -- last acquired @ kern/kern_proc.c:439 11 process lock -- (already displayed) 12 session -- (already displayed) 15 UMA zone -- (already displayed) 9 ttylist -- last acquired @ kern/tty.c:2855 13 tty -- (already displayed) 16 sellck -- (already displayed) 15 kqueue -- (already displayed) 8 vm page queue mutex -- last acquired @ vm/vm_object.c:1816 17 vnode interlock -- (already displayed) 9 pmap -- last acquired @ amd64/amd64/pmap.c:1961 14 vm page queue free mutex -- (already displayed) 14 vm page queue free mutex -- (already displayed) 15 UMA zone -- (already displayed) 6 system map -- last acquired @ vm/vm_map.c:2421 8 vm page queue mutex -- (already displayed) 7 kmem object -- last acquired @ vm/vm_object.c:441 14 vm page queue free mutex -- (already displayed) 8 vm page queue mutex -- (already displayed) 17 vnode interlock -- (already displayed) 7 KMAP ENTRY -- last acquired @ vm/uma_core.c:2257 9 pmap -- (already displayed) 14 vm page queue free mutex -- (already displayed) 7 kernel object -- last acquired @ kern/vfs_bio.c:3680 14 vm page queue free mutex -- (already displayed) 8 vm page queue mutex -- (already displayed) 15 UMA zone -- (already displayed) 7 standard object -- last acquired @ vm/vnode_pager.c:110 14 vm page queue free mutex -- (already displayed) 8 vm page queue mutex -- (already displayed) 17 vnode interlock -- (already displayed) 8 vm object_list -- last acquired @ vm/vm_object.c:225 15 UMA zone -- (already displayed) 8 swap_pager swhash -- last acquired @ vm/swap_pager.c:1888 7 uma object -- last acquired @ vm/uma_core.c:986 14 vm page queue free mutex -- (already displayed) 14 UMA boot pages -- (already displayed) 14 vm page queue free mutex -- (already displayed) 6 UMA lock -- last acquired @ vm/uma_core.c:1492 15 UMA zone -- (already displayed) 7 KMAP ENTRY -- (already displayed) 13 eventhandler -- (already displayed) 14 eventhandler list -- (already displayed) 6 kobj -- last acquired @ kern/subr_kobj.c:258 7 kernel environment -- last acquired @ kern/kern_environment.c:301 6 malloc -- last acquired @ kern/kern_malloc.c:655 7 kernel object -- (already displayed) 8 vm object_list -- (already displayed) 7 KMAP ENTRY -- (already displayed) 13 uidinfo hash -- (already displayed) 11 process lock -- (already displayed) 14 sleep mtxpool -- (already displayed) 6 evclass_mtx -- last acquired @ security/audit/audit_bsm_klib.c:113 6 TID lock -- last acquired @ kern/subr_unit.c:752 7 standard object -- (already displayed) 6 intr event -- last acquired @ kern/kern_intr.c:454 18 cdev -- (already displayed) 6 GEOM orphanage -- last acquired @ geom/geom_event.c:201 9 ttylist -- (already displayed) 6 intr config -- last acquired @ kern/subr_autoconf.c:72 6 taskqueue list -- last acquired @ kern/subr_taskqueue.c:125 6 XPT lock -- last acquired @ cam/cam_xpt.c:2668 15 UMA zone -- (already displayed) 7 XPT topology lock -- last acquired @ cam/cam_xpt.c:2673 7 kernel environment -- (already displayed) 14 taskqueue -- (already displayed) 6 rman head -- last acquired @ kern/subr_rman.c:152 6 rman -- last acquired @ kern/subr_rman.c:539 15 UMA zone -- (already displayed) 14 vm page queue free mutex -- (already displayed) 6 devd -- last acquired @ kern/subr_bus.c:489 16 sellck -- (already displayed) 16 sellck -- (already displayed) 6 ACPI semaphore -- last acquired @ dev/acpica/Osd/OsdSynch.c:303 6 acpi subsystem HW lock -- last acquired @ dev/acpica/Osd/OsdSynch.c:377 6 acpi subsystem GPE lock -- last acquired @ dev/acpica/Osd/OsdSynch.c:377 14 taskqueue -- (already displayed) 13 network driver -- (already displayed) 6 bounce pages lock -- last acquired @ amd64/amd64/busdma_machdep.c:1070 12 ifnet -- (already displayed) 6 bpf global lock -- last acquired @ net/bpf.c:394 14 bpf interface lock -- (already displayed) 6 unit# allocation -- last acquired @ kern/subr_unit.c:623 18 vnode_free_list -- (already displayed) 6 pfs_node -- last acquired @ fs/pseudofs/pseudofs_internal.h:103 6 pfs_fileno -- last acquired @ kern/subr_unit.c:623 14 random reseed -- (already displayed) 14 arc4_mtx -- (already displayed) 7 mountlist -- last acquired @ ufs/ffs/ffs_softdep.c:747 16 struct mount mtx -- (already displayed) 6 nfsd_mtx -- last acquired @ nfsserver/nfs_srvsock.c:799 9 so_snd -- (already displayed) 14 sleep mtxpool -- (already displayed) 10 so_rcv -- (already displayed) 6 if_clone lock -- last acquired @ net/if_clone.c:164 6 if_cloners lock -- last acquired @ net/if_clone.c:252 6 domain list -- last acquired @ kern/uipc_domain.c:228 9 pfil_head_list lock -- last acquired @ net/pfil.c:159 8 PFil hook read/write mutex -- last acquired @ net/pfil.c:73 9 pfil_head_list lock -- (already displayed) 13 pf task mtx -- (already displayed) 15 UMA zone -- (already displayed) 6 isn_mtx -- last acquired @ netinet/tcp_subr.c:1426 14 random reseed -- (already displayed) 14 arc4_mtx -- (already displayed) 11 radix node head -- (already displayed) 6 lo_mtx -- last acquired @ net/if_loop.c:157 6 ATA queue lock -- last acquired @ dev/ata/ata-queue.c:177 7 ATA state lock -- last acquired @ dev/ata/ata-all.c:316 6 devstat -- last acquired @ kern/subr_devstat.c:83 7 XPT topology lock -- (already displayed) 6 NFS iod lock -- last acquired @ nfsclient/nfs_nfsiod.c:196 16 struct mount mtx -- (already displayed) 6 mntid -- last acquired @ kern/vfs_subr.c:460 7 mountlist -- (already displayed) 17 vnode interlock -- (already displayed) 6 Name Cache -- (already displayed) 6 knlist lock for lockless objects -- last acquired @ kern/kern_event.c:1664 6 bio queue -- last acquired @ geom/geom_io.c:68 6 devfs interlock -- last acquired @ fs/devfs/devfs_vnops.c:194 17 vnode interlock -- (already displayed) 18 cdev -- (already displayed) 8 vm page queue mutex -- (already displayed) 8 process group -- (already displayed) 13 tty -- (already displayed) 12 session -- (already displayed) 6 buf queue lock -- last acquired @ kern/vfs_bio.c:1468 17 vnode interlock -- (already displayed) 6 bdone lock -- last acquired @ kern/vfs_bio.c:3002 6 needsbuffer lock -- last acquired @ kern/vfs_bio.c:321 6 FFS Lock -- last acquired @ ufs/ffs/ffs_alloc.c:2075 14 arc4_mtx -- (already displayed) 6 vfs hash -- last acquired @ kern/vfs_hash.c:71 17 vnode interlock -- (already displayed) 6 dirhash list -- last acquired @ ufs/ufs/ufs_dirhash.c:349 7 dirhash -- last acquired @ ufs/ufs/ufs_dirhash.c:350 7 dirhash -- (already displayed) 6 pbuf mutex -- last acquired @ vm/vm_pager.c:413 6 buffer daemon lock -- last acquired @ kern/vfs_bio.c:2107 8 if_afdata -- last acquired @ netinet6/scope6.c:406 9 scope6_lock -- last acquired @ netinet6/scope6.c:435 14 if_addr_mtx -- (already displayed) 13 pf task mtx -- (already displayed) 7 sigio lock -- (already displayed) 8 ip6_inq -- last acquired @ net/netisr.c:140 6 udp -- last acquired @ netinet/udp_usrreq.c:1099 7 udpinp -- last acquired @ netinet/udp_usrreq.c:1100 8 in_multi_mtx -- last acquired @ netinet/in_mcast.c:320 9 igmp_mtx -- last acquired @ netinet/igmp.c:446 14 if_addr_mtx -- (already displayed) 14 if_addr_mtx -- (already displayed) 15 UMA zone -- (already displayed) 13 network driver -- (already displayed) 9 so_snd -- (already displayed) 12 ifnet -- (already displayed) 15 UMA zone -- (already displayed) 14 arc4_mtx -- (already displayed) 11 radix node head -- (already displayed) 12 rtentry -- (already displayed) 8 PFil hook read/write mutex -- (already displayed) 14 if send queue -- (already displayed) 13 network driver -- (already displayed) 10 so_rcv -- (already displayed) 8 accept -- (already displayed) 8 ip_inq -- last acquired @ net/netisr.c:140 14 bpf interface lock -- (already displayed) 8 if_afdata -- (already displayed) 8 addrsel_lock -- last acquired @ netinet6/in6_src.c:858 10 tcp_hc_entry -- (already displayed) 8 ip6_inq -- (already displayed) 14 random reseed -- (already displayed) 15 UMA zone -- (already displayed) 14 vm page queue free mutex -- (already displayed) 18 Syncer mtx -- (already displayed) 9 so_snd -- (already displayed) 6 pipe mutex -- (already displayed) 15 kqueue -- (already displayed) 15 bpf cdev lock -- (already displayed) 6 system map -- (already displayed) 10 so_rcv -- (already displayed) 5 protect sysfilt_ops -- last acquired @ kern/kern_event.c:774 15 UMA zone -- (already displayed) 4 user map -- last acquired @ vm/vm_glue.c:183 15 UMA zone -- (already displayed) 14 vm page queue free mutex -- (already displayed) 8 vm object_list -- (already displayed) 7 standard object -- (already displayed) 8 vm page queue mutex -- (already displayed) 9 pmap -- (already displayed) 17 vnode interlock -- (already displayed) 8 swap_pager swhash -- (already displayed) 6 buf queue lock -- (already displayed) 6 system map -- (already displayed) 6 bio queue -- (already displayed) 6 bdone lock -- (already displayed) 6 needsbuffer lock -- (already displayed) 16 struct mount mtx -- (already displayed) 14 random reseed -- (already displayed) 6 Name Cache -- (already displayed) 6 buffer daemon lock -- (already displayed) 2 tcpinp -- last acquired @ netinet/tcp_input.c:479 9 so_snd -- (already displayed) 15 UMA zone -- (already displayed) 14 vm page queue free mutex -- (already displayed) 10 so_rcv -- (already displayed) 12 ifnet -- (already displayed) 14 arc4_mtx -- (already displayed) 11 radix node head -- (already displayed) 12 rtentry -- (already displayed) 6 isn_mtx -- (already displayed) 8 PFil hook read/write mutex -- (already displayed) 14 if send queue -- (already displayed) 13 network driver -- (already displayed) 10 tcp_hc_entry -- (already displayed) 8 accept -- (already displayed) 8 ip_inq -- (already displayed) 3 tcp_sc_head -- last acquired @ kern/kern_timeout.c:241 15 UMA zone -- (already displayed) 14 vm page queue free mutex -- (already displayed) 14 arc4_mtx -- (already displayed) 11 radix node head -- (already displayed) 12 rtentry -- (already displayed) 10 tcp_hc_entry -- (already displayed) 14 random reseed -- (already displayed) 8 PFil hook read/write mutex -- (already displayed) 14 if send queue -- (already displayed) 13 network driver -- (already displayed) 3 so_glabel -- last acquired @ kern/uipc_socket.c:299 0 so_snd_sx -- last acquired @ kern/uipc_sockbuf.c:148 9 so_snd -- (already displayed) 15 UMA zone -- (already displayed) 1 unp_global_rwlock -- last acquired @ kern/uipc_usrreq.c:770 2 unp_mtx -- (already displayed) 8 accept -- (already displayed) 15 UMA zone -- (already displayed) 3 so_glabel -- (already displayed) 9 so_snd -- (already displayed) 14 vm page queue free mutex -- (already displayed) 4 filedesc structure -- (already displayed) 14 vm page queue free mutex -- (already displayed) 11 radix node head -- (already displayed) 12 rtentry -- (already displayed) 13 rts_inq -- (already displayed) 2 tcpinp -- (already displayed) 6 udp -- (already displayed) 11 process lock -- (already displayed) 4 user map -- (already displayed) 7 standard object -- (already displayed) 6 system map -- (already displayed) 14 random reseed -- (already displayed) 6 bio queue -- (already displayed) 17 vnode interlock -- (already displayed) 0 pf_statetbl_lock -- last acquired @ /usr/src/sys/modules/pf/../../contrib/pf/net/pf.c:979 13 pf task mtx -- (already displayed) 15 UMA zone -- (already displayed) 9 pfil_head_list lock -- (already displayed) 8 PFil hook read/write mutex -- (already displayed) 12 ifnet -- (already displayed) 14 if send queue -- (already displayed) 0 if_bridge -- last acquired @ kern/kern_timeout.c:241 1 bstp -- last acquired @ /usr/src/sys/modules/bridgestp/../../net/bridgestp.c:2163 12 ifnet -- (already displayed) 12 ifnet -- (already displayed) 15 UMA zone -- (already displayed) 0 if_bridge list -- last acquired @ /usr/src/sys/modules/if_bridge/../../net/if_bridge.c:610 0 bridgestp list -- last acquired @ /usr/src/sys/modules/bridgestp/../../net/bridgestp.c:2143 0 zvol -- last acquired @ /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c:348 0 zfs:vdev:geom:queue -- last acquired @ /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_geom.c:203 0 devfsmount -- last acquired @ fs/devfs/devfs_vnops.c:201 6 devfs interlock -- (already displayed) 18 vnode_free_list -- (already displayed) 15 UMA zone -- (already displayed) 17 vnode interlock -- (already displayed) 16 struct mount mtx -- (already displayed) 18 cdev -- (already displayed) 14 vm page queue free mutex -- (already displayed) 1 DEVFS ruleset lock -- last acquired @ fs/devfs/devfs_rule.c:144 15 UMA zone -- (already displayed) 11 process lock -- (already displayed) 4 user map -- (already displayed) 7 standard object -- (already displayed) 8 vm object_list -- (already displayed) 6 Name Cache -- (already displayed) 0 MD config lock -- last acquired @ dev/md/md.c:1161 15 UMA zone -- (already displayed) 13 eventhandler -- (already displayed) 2 proctree -- last acquired @ kern/kern_exit.c:686 3 allproc -- last acquired @ kern/kern_proc.c:249 4 allprison -- last acquired @ kern/kern_jail.c:951 14 sleep mtxpool -- (already displayed) 11 process lock -- (already displayed) 12 fdesc -- (already displayed) 4 filedesc structure -- (already displayed) 17 vnode interlock -- (already displayed) 4 user map -- (already displayed) 8 process group -- (already displayed) 11 process lock -- (already displayed) 5 Giant -- (already displayed) 12 session -- (already displayed) 7 sigio lock -- (already displayed) 3 clone events drain lock -- last acquired @ fs/devfs/devfs_vnops.c:626 13 eventhandler -- (already displayed) 14 eventhandler list -- (already displayed) 15 UMA zone -- (already displayed) 18 cdev -- (already displayed) 4 filedesc structure -- (already displayed) 11 process lock -- (already displayed) 8 vm object_list -- (already displayed) 7 standard object -- (already displayed) 1 GEOM topology -- last acquired @ geom/geom_event.c:233 6 GEOM orphanage -- (already displayed) 15 UMA zone -- (already displayed) 6 devstat -- (already displayed) 6 unit# allocation -- (already displayed) 18 cdev -- (already displayed) 6 bio queue -- (already displayed) 6 bdone lock -- (already displayed) 14 vm page queue free mutex -- (already displayed) 14 UMA boot pages -- (already displayed) 6 system map -- (already displayed) 2 swapdev -- last acquired @ vm/swap_pager.c:2235 8 vm object_list -- (already displayed) 17 vnode interlock -- (already displayed) 7 standard object -- (already displayed) 2 unique lock -- last acquired @ /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/unique.c:100 15 UMA zone -- (already displayed) 14 random reseed -- (already displayed) 13 eventhandler -- (already displayed) 2 proctree -- (already displayed) 4 filedesc structure -- (already displayed) 11 process lock -- (already displayed) 0 g_disk_done -- last acquired @ geom/geom_disk.c:198 15 UMA zone -- (already displayed) 6 bio queue -- (already displayed) 0 ip6qlock -- last acquired @ netinet6/frag6.c:688 0 iterator -- last acquired @ netinet/sctputil.c:1209 0 ipqlock -- last acquired @ netinet/ip_input.c:1086 0 msq -- last acquired @ kern/sysv_msg.c:1027 0 semid -- last acquired @ kern/sysv_sem.c:658 1 sem -- last acquired @ kern/sysv_sem.c:1288 0 intr sources -- last acquired @ amd64/amd64/intr_machdep.c:593 0 audit_mtx -- last acquired @ security/audit/audit_worker.c:393 0 p_peers -- last acquired @ kern/kern_exit.c:284 0 runningbufspace lock -- last acquired @ kern/vfs_bio.c:340 0 umtxql -- last acquired @ kern/kern_umtx.c:300 0 fifo mutex -- last acquired @ fs/fifofs/fifo_vnops.c:224 9 so_snd -- (already displayed) 10 so_rcv -- (already displayed) 0 Softdep Lock -- last acquired @ ufs/ffs/ffs_softdep.c:767 0 ACPI root bus -- last acquired @ dev/acpica/acpi.c:1022 6 rman -- (already displayed) 6 system map -- (already displayed) 15 UMA zone -- (already displayed) 6 ACPI semaphore -- (already displayed) 0 accept_filter_mtx -- last acquired @ kern/uipc_accf.c:116 0 ACPI PCI bus methods -- last acquired @ dev/acpica/acpi_pcib.c:221 0 rtsock route_cb lock -- last acquired @ net/rtsock.c:236 0 rawcb -- last acquired @ net/raw_cb.c:104 10 so_rcv -- (already displayed) 15 UMA zone -- (already displayed) 0 vm daemon -- last acquired @ vm/vm_pageout.c:1531 0 ACPI PCI link -- last acquired @ dev/acpica/acpi_pci_link.c:432 15 UMA zone -- (already displayed) 6 ACPI semaphore -- (already displayed) 0 lock states lock -- last acquired @ kern/kern_lockf.c:700 0 sysctl lock -- last acquired @ kern/kern_sysctl.c:1396 14 arc4_mtx -- (already displayed) 15 UMA zone -- (already displayed) 3 allproc -- (already displayed) 11 process lock -- (already displayed) 4 user map -- (already displayed) 18 cdev -- (already displayed) 4 filedesc structure -- (already displayed) 12 fdesc -- (already displayed) 1 kernel linker -- last acquired @ kern/kern_linker.c:1170 15 UMA zone -- (already displayed) 4 filedesc structure -- (already displayed) 17 vnode interlock -- (already displayed) 6 Name Cache -- (already displayed) 16 struct mount mtx -- (already displayed) 6 vfs hash -- (already displayed) 6 buf queue lock -- (already displayed) 6 system map -- (already displayed) 6 bio queue -- (already displayed) 6 bdone lock -- (already displayed) 5 Giant -- (already displayed) 2 module subsystem sx lock -- last acquired @ kern/kern_module.c:407 15 UMA zone -- (already displayed) 14 vm page queue free mutex -- (already displayed) 18 cdev -- (already displayed) 7 standard object -- (already displayed) 6 needsbuffer lock -- (already displayed) 6 buffer daemon lock -- (already displayed) 7 dirhash -- (already displayed) 18 vnode_free_list -- (already displayed) 14 vm page queue free mutex -- (already displayed) 8 vm object_list -- (already displayed) 6 kobj -- (already displayed) 6 pbuf mutex -- (already displayed) 1 filelist lock -- last acquired @ kern/kern_descrip.c:2246 4 filedesc structure -- (already displayed) 6 GEOM orphanage -- (already displayed) 5 Giant -- (already displayed) 12 ktrace -- (already displayed) 6 malloc -- (already displayed) 6 system map -- (already displayed) 6 devstat -- (already displayed) 9 ttylist -- (already displayed) 8 vm object_list -- (already displayed) 6 UMA lock -- (already displayed) 18 Syncer mtx -- (already displayed) 1 unp_global_rwlock -- (already displayed) 1 tcp -- last acquired @ netinet/tcp_timer.c:128 2 tcpinp -- (already displayed) 15 UMA zone -- (already displayed) 14 vm page queue free mutex -- (already displayed) 8 accept -- (already displayed) 14 sleep mtxpool -- (already displayed) 3 so_glabel -- (already displayed) 11 radix node head -- (already displayed) 8 PFil hook read/write mutex -- (already displayed) 12 rtentry -- (already displayed) 14 if send queue -- (already displayed) 13 network driver -- (already displayed) 8 ip_inq -- (already displayed) 3 tcp_sc_head -- (already displayed) 6 udp -- (already displayed) 1 sctp-info -- last acquired @ netinet/sctp_pcb.c:6138 1 rip -- last acquired @ netinet/raw_ip.c:613 15 UMA zone -- (already displayed) 14 vm page queue free mutex -- (already displayed) 2 rawinp -- last acquired @ netinet/in_pcb.c:221 1 addrsel_sxlock -- last acquired @ netinet6/in6_src.c:1023 8 addrsel_lock -- (already displayed) 1 db_capture_sx -- last acquired @ ddb/db_capture.c:181 1 db_script_mtx -- last acquired @ ddb/db_script.c:481 6 ACPI semaphore -- (already displayed) 4 allprison -- (already displayed) 1 cpufreq lock -- last acquired @ kern/kern_cpu.c:413 15 UMA zone -- (already displayed) 14 vm page queue free mutex -- (already displayed) 12 ifnet -- (already displayed) 1 ACPI CPU -- last acquired @ dev/acpica/acpi_cpu.c:1187 13 eventhandler -- (already displayed) 14 eventhandler list -- (already displayed) 14 vm page queue free mutex -- (already displayed) 2 swapdev -- (already displayed) 14 random reseed -- (already displayed) 0 ng_node -- last acquired @ order list:0 1 ng_worklist -- last acquired @ order list:0 0 802.11 com lock -- last acquired @ order list:0 0 ddp_list_mtx -- last acquired @ order list:0 1 ddp_mtx -- last acquired @ order list:0 0 slip_mtx -- last acquired @ order list:0 1 slip sc_mtx -- last acquired @ order list:0 0 unp -- last acquired @ order list:0 9 so_snd -- (already displayed) Spin locks: 0 ap boot -- last acquired @ amd64/amd64/mp_machdep.c:539 1 rm.mutex_mtx -- last acquired @ order list:0 2 sio -- last acquired @ dev/sio/sio.c:555 3 scrlock -- last acquired @ dev/syscons/syscons.c:2526 4 scc_hwmtx -- last acquired @ order list:0 5 uart_hwmtx -- last acquired @ order list:0 6 fast_taskqueue -- last acquired @ kern/subr_taskqueue.c:71 7 intr table -- last acquired @ order list:0 8 process slock -- last acquired @ kern/subr_sleepqueue.c:410 9 sleepq chain -- last acquired @ kern/subr_sleepqueue.c:232 10 umtx lock -- last acquired @ kern/kern_umtx.c:3189 11 turnstile chain -- last acquired @ kern/subr_turnstile.c:546 12 turnstile lock -- last acquired @ kern/subr_turnstile.c:555 13 sched lock -- last acquired @ kern/subr_sleepqueue.c:776 14 td_contested -- last acquired @ kern/subr_turnstile.c:895 15 callout -- last acquired @ kern/kern_timeout.c:419 16 entropy harvest mutex -- last acquired @ dev/random/randomdev_soft.c:249 17 syscons video lock -- last acquired @ dev/syscons/syscons.c:1802 18 time lock -- last acquired @ order list:0 19 smp rendezvous -- last acquired @ amd64/amd64/mp_machdep.c:845 20 icu -- last acquired @ order list:0 21 clk -- last acquired @ amd64/isa/clock.c:425 22 mprof lock -- last acquired @ order list:0 23 kse lock -- last acquired @ kern/kern_kse.c:92 24 zombie lock -- last acquired @ kern/kern_thread.c:298 25 ALD Queue -- last acquired @ order list:0 26 pcicfg -- last acquired @ amd64/pci/pci_cfgreg.c:133 27 NDIS thread lock -- last acquired @ order list:0 28 tw_osl_io_lock -- last acquired @ order list:0 29 tw_osl_q_lock -- last acquired @ order list:0 30 tw_cl_io_lock -- last acquired @ order list:0 31 tw_cl_intr_lock -- last acquired @ order list:0 32 tw_cl_gen_lock -- last acquired @ order list:0 33 blocked lock -- last acquired @ order list:0 15 callout -- (already displayed) 23 kse lock -- (already displayed) 24 zombie lock -- (already displayed) 14 td_contested -- (already displayed) 33 blocked lock -- (already displayed) 13 sched lock -- (already displayed) 13 sched lock -- (already displayed) 15 callout -- (already displayed) 33 blocked lock -- (already displayed) 11 turnstile chain -- (already displayed) 13 sched lock -- (already displayed) 10 umtx lock -- (already displayed) 12 turnstile lock -- (already displayed) 11 turnstile chain -- (already displayed) 33 blocked lock -- (already displayed) 13 sched lock -- (already displayed) 9 sleepq chain -- (already displayed) 33 blocked lock -- (already displayed) 1 intrcnt -- last acquired @ amd64/amd64/intr_machdep.c:471 3 scrlock -- (already displayed) 17 syscons video lock -- (already displayed) Locks which were never acquired: SCSI CD Changer List arp_inq pfs_vncache ppp_softc_list_mtx tunmtx gif_mtx nfs4dev state nfs4dev waitq nfs4dev newq ntfs nthash NFS xid lock NFS reqq lock ACPI global lock CAM SIMQ lock hptlock LED sx LED mtx audit_pipe_mtx nfslock msi audit_worker_sx audit_trigger_mtx ktrace_sx bpin lock zfs_debug pmc shared lock kqueue order securelevel mutex lock firmware table ACPI power resources encapmtx ACPI cmbat net80211 instances UUID generator mutex lock ACPI generic battery acct_sx ACPI PCI power methods ACPI lid ACPI HPET support ACPI embedded controller ACPI AC adapter ACPI thermal zone MSDOSFS fileno ACPI Smart Battery phys_pager list dev_pager list swap_pager list vm map sleep mutex db> show lock 0xffffffff8102c460 class: spin mutex name: sleepq chain flags: {SPIN, RECURSE} state: {OWNED} owner: 0xffffff000274e000 (tid 100089, pid 90, "txg_thread_enter") db> bt 90 Tracing pid 90 tid 100089 td 0xffffff000274e000 cpustop_handler() at cpustop_handler+0x40 ipi_nmi_handler() at ipi_nmi_handler+0x30 trap() at trap+0x378 nmi_calltrap() at nmi_calltrap+0x8 --- trap 0x13, rip = 0xffffffff80485f57, rsp = 0xffffffffbcbddff0, rbp = 0xffffffffbfcabaa0 --- _thread_lock_flags() at _thread_lock_flags+0xb7 sleepq_wait() at sleepq_wait+0x3c _cv_wait() at _cv_wait+0x15e txg_thread_wait() at txg_thread_wait+0x5f txg_quiesce_thread() at txg_quiesce_thread+0x9c fork_exit() at fork_exit+0x12a fork_trampoline() at fork_trampoline+0xe --- trap 0, rip = 0, rsp = 0xffffffffbfcabd30, rbp = 0 --- db> panic Script started on Tue Jun 3 12:08:24 2008 You have mail. pegasus# ls .htaccess capture.txt typescript pegasus# ggkgdb /boot/kernel/vkern kernel* kernel.symbols* pegasus# kgdb /boot/kernel/kernel /bootvarvar/rcrash/ bounds info.0 vmcore.0 pegasus# kgdb /boot/kernel/kernel /var/crash/ bounds info.0 vmcore.0 pegasus# kgdb /boot/kernel/kernel /var/crash/mvmcore.0 [GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so: Undefined symbol "ps_pglobal_lookup"] GNU gdb 6.1.1 [FreeBSD] Copyright 2004 Free Software Foundation, Inc. GDB is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain conditions. Type "show copying" to see the conditions. There is absolutely no warranty for GDB. Type "show warranty" for details. This GDB was configured as "amd64-marcel-freebsd". There is no member named pathname. Reading symbols from /boot/kernel/zfs.ko...Reading symbols from /bootdir/boot/kernel/zfs.ko.symbols...done. done. Loaded symbols for /boot/kernel/zfs.ko Reading symbols from /boot/kernel/speaker.ko...Reading symbols from /bootdir/boot/kernel/speaker.ko.symbols...done. done. Loaded symbols for /boot/kernel/speaker.ko Reading symbols from /boot/kernel/wlan_xauth.ko...Reading symbols from /bootdir/boot/kernel/wlan_xauth.ko.symbols...done. done. Loaded symbols for /boot/kernel/wlan_xauth.ko Reading symbols from /boot/kernel/fdescfs.ko...Reading symbols from /bootdir/boot/kernel/fdescfs.ko.symbols...done. done. Loaded symbols for /boot/kernel/fdescfs.ko Reading symbols from /boot/kernel/if_bridge.ko...Reading symbols from /bootdir/boot/kernel/if_bridge.ko.symbols...done. done. Loaded symbols for /boot/kernel/if_bridge.ko Reading symbols from /boot/kernel/bridgestp.ko...Reading symbols from /bootdir/boot/kernel/bridgestp.ko.symbols...done. done. Loaded symbols for /boot/kernel/bridgestp.ko Reading symbols from /boot/kernel/pflog.ko...Reading symbols from /bootdir/boot/kernel/pflog.ko.symbols...done. done. Loaded symbols for /boot/kernel/pflog.ko Reading symbols from /boot/kernel/pf.ko...Reading symbols from /bootdir/boot/kernel/pf.ko.symbols...done. done. Loaded symbols for /boot/kernel/pf.ko Unread portion of the kernel message buffer: panic: mutex sleepq chain (0xffffffff8102c460) not owned at ../../../kern/subr_sleepqueue.c:641 cpuid = 0 KDB: enter: panic exclusive sx so_rcv_sx r = 0 (0xffffff001b9db3b8) locked @ kern/uipc_sockbuf.c:148 exclusive sx so_rcv_sx r = 0 (0xffffff0002b1b670) locked @ kern/uipc_sockbuf.c:148 panic: from debugger cpuid = 0 Uptime: 2h44m32s Physical memory: 2027 MB Dumping 1070 MB: 1055 1039 1023 1007 991 975 959 943 927 911 895 879 863 847 831 815 799 783 767 751 735 719 703 687 671 655 639 623 607 591 575 559 543 527 511 495 479 463 447 431 415 399 383 367 351 335 319 303 287 271 255 239 223 207 191 175 159 143 127 111 95 79 63 47 31 15 #0 doadump () at pcpu.h:194 194 pcpu.h: No such file or directory. in pcpu.h (kgdb) quit    bt #0 doadump () at pcpu.h:194 #1 0xffffffff80491bb3 in boot (howto=260) at ../../../kern/kern_shutdown.c:418 #2 0xffffffff80492037 in panic (fmt=Variable "fmt" is not available. ) at ../../../kern/kern_shutdown.c:572 #3 0xffffffff801c6717 in db_panic (addr=Variable "addr" is not available. ) at ../../../ddb/db_command.c:446 #4 0xffffffff801c700f in db_command (last_cmdp=0xffffffff80a64608, cmd_table=0x0, dopager=1) at ../../../ddb/db_command.c:413 #5 0xffffffff801c7220 in db_command_loop () at ../../../ddb/db_command.c:466 #6 0xffffffff801c8b59 in db_trap (type=Variable "type" is not available. ) at ../../../ddb/db_main.c:228 #7 0xffffffff804bdab5 in kdb_trap (type=3, code=0, tf=0xffffffffbfcb5890) at ../../../kern/subr_kdb.c:524 #8 0xffffffff80752fa5 in trap (frame=0xffffffffbfcb5890) at ../../../amd64/amd64/trap.c:499 #9 0xffffffff8073872e in calltrap () at ../../../amd64/amd64/exception.S:169 #10 0xffffffff804bdc8d in kdb_enter_why (why=0xffffffff808164ee "panic", msg=0xa
) at cpufunc.h:63 #11 0xffffffff80492046 in panic (fmt=Variable "fmt" is not available. ) at ../../../kern/kern_shutdown.c:556 #12 0xffffffff804849f2 in _mtx_assert (m=Variable "m" is not available. ) at ../../../kern/kern_mutex.c:644 #13 0xffffffff804c5453 in sleepq_resume_thread (sq=0xffffff0002160460, td=0xffffff000274e000, pri=-1) at ../../../kern/subr_sleepqueue.c:641 #14 0xffffffff804c5b74 in sleepq_broadcast (wchan=0xffffff00026ea4d0, flags=Variable "flags" is not available. ) at ../../../kern/subr_sleepqueue.c:777 #15 0xffffffff80499d39 in unlock_sx (lock=Variable "lock" is not available. ) at ../../../kern/kern_sx.c:149 #16 0xffffffff8045de27 in _cv_timedwait (cvp=0xffffff00026ea5a0, ---Type to continue, or q to quit--- lock=0xffffff00026ea4d0, timo=5000) at ../../../kern/kern_condvar.c:302 #17 0xffffffff8126dcf7 in txg_thread_wait (tx=0xffffff00026ea4c8, cpr=0xffffffffbfcb5bb0, cv=0xffffff00026ea5a0, secmax=5) at /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:156 #18 0xffffffff8126de33 in txg_timelimit_thread (arg=Variable "arg" is not available. ) at /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:441 #19 0xffffffff804712ca in fork_exit ( callout=0xffffffff8126ddd0 , arg=0xffffff00026ea400, frame=0xffffffffbfcb5c80) at ../../../kern/kern_fork.c:783 #20 0xffffffff80738afe in fork_trampoline () at ../../../amd64/amd64/exception.S:415 #21 0x0000000000000000 in ?? () #22 0x0000000000000000 in ?? () #23 0x0000000000000001 in ?? () #24 0x0000000000000000 in ?? () #25 0x0000000000000000 in ?? () #26 0x0000000000000000 in ?? () #27 0x0000000000000000 in ?? () #28 0x0000000000000000 in ?? () #29 0x0000000000000000 in ?? () #30 0x0000000000000000 in ?? () ---Type to continue, or q to quit--- #31 0x0000000000000000 in ?? () #32 0x0000000000000000 in ?? () #33 0x0000000000000000 in ?? () #34 0x0000000000000000 in ?? () #35 0x0000000000000000 in ?? () #36 0x0000000000000000 in ?? () #37 0x0000000000000000 in ?? () #38 0x0000000000000000 in ?? () #39 0x0000000000000000 in ?? () #40 0x0000000000000000 in ?? () #41 0x0000000000000000 in ?? () #42 0x0000000000000000 in ?? () #43 0x0000000000000000 in ?? () #44 0x0000000000000000 in ?? () #45 0x000000000142b000 in ?? () #46 0xffffffff8101da80 in tdg_maxid () #47 0xffffffff8102a280 in tdq_cpu () #48 0xffffff000274c9b0 in ?? () #49 0xffffff000274c6a0 in ?? () #50 0xffffffffbfcb5a60 in ?? () #51 0xffffffffbfcb5a08 in ?? () #52 0x0000000000000000 in ?? () #53 0xffffffff804b2931 in sched_switch (td=0xffffffff8126ddd0, newtd=Cannot access memory at address 0xffffffffffffffb0 ) at ../../../kern/sched_ule.c:1897 Previous frame inner to this frame (corrupt stack?) (kgdb) l *txg_thread_wait+0x5f 0xffffffff8126dd0f is in txg_thread_wait (/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:158). 153 CALLB_CPR_SAFE_BEGIN(cpr); 154 155 if (secmax) 156 (void) cv_timedwait(cv, &tx->tx_sync_lock, secmax * hz); 157 else 158 cv_wait(cv, &tx->tx_sync_lock); 159 160 CALLB_CPR_SAFE_END(cpr, &tx->tx_sync_lock); 161 } 162 (kgdb) l *txg_thread_quiesce_thread+0x9c 0xffffffff8126df5c is in txg_quiesce_thread (/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:360). 355 * However, we can only have one txg in "quiescing" or 356 * "quiesced, waiting to sync" state. So we wait until 357 * the "quiesced, waiting to sync" txg has been consumed 358 * by the sync thread. 359 */ 360 while (!tx->tx_exiting && 361 (tx->tx_open_txg >= tx->tx_quiesce_txg_waiting || 362 tx->tx_quiesced_txg != 0)) 363 txg_thread_wait(tx, &cpr, &tx->tx_quiesce_more_cv, 0); 364 (kgdb)