Dump header from device /dev/ad10s1e Architecture: amd64 Architecture Version: 2 Dump Length: 305152000B (291 MB) Blocksize: 512 Dumptime: Wed Jun 11 08:33:00 2008 Hostname: pegasus.clearchain.com Magic: FreeBSD Kernel Dump Version String: FreeBSD 7.0-STABLE #15: Wed Jun 11 08:24:40 CST 2008 root@pegasus.clearchain.com:/data/usr/src/sys/amd64/compile/PEGASUS Panic String: mutex sleepq chain (0xffffffff8102bf08) not owned at ../../../kern/subr_sleepqueue.c:778 Dump Parity: 755976200 Bounds: 2 Dump Status: good db> show lock 0xffffffff8102bf08 class: spin mutex name: sleepq chain flags: {SPIN, RECURSE} state: {UNOWNED} db> bt Tracing pid 135 tid 100054 td 0xffffff00026469f0 kdb_enter_why() at kdb_enter_why+0x3d panic() at panic+0x176 _mtx_assert() at _mtx_assert+0xe2 sleepq_broadcast() at sleepq_broadcast+0xaa taskq_thread() at taskq_thread+0x1f9 fork_exit() at fork_exit+0x12a fork_trampoline() at fork_trampoline+0xe --- trap 0, rip = 0, rsp = 0xffffffffbfbecd30, rbp = 0 --- db> show thread 100054 Thread 100054 at 0xffffff00026469f0: proc (pid 135): 0xffffff0002626000 name: spa_zio_intr_2 flags: 0x4 pflags: 0 state: RUNNING (CPU 0) priority: 128 container lock: sched lock 0 (0xffffffff8102a380) db> show lock 0xffffffff8102a380 class: spin mutex name: sched lock 0 type: sched lock flags: {SPIN, RECURSE} state: {UNOWNED} db> show allpcpu Current CPU: 0 cpuid = 0 curthread = 0xffffff00026469f0: pid 135 "spa_zio_intr_2" curpcb = 0xffffffffbfbecd40 fpcurthread = none idlethread = 0xffffff0002171350: pid 12 "idle: cpu0" spin locks held: cpuid = 1 curthread = 0xffffff00021716a0: pid 11 "idle: cpu1" curpcb = 0xffffffffbd63bd40 fpcurthread = none idlethread = 0xffffff00021716a0: pid 11 "idle: cpu1" spin locks held: db> show witness Sleep locks: 0 md bio queue -- last acquired @ dev/md/md.c:700 0 ls_lock -- last acquired @ kern/kern_lockf.c:617 2 owner graph lock -- last acquired @ kern/kern_lockf.c:374 14 UMA zone -- last acquired @ vm/uma_core.c:2257 1 lock owners lock -- last acquired @ kern/kern_lockf.c:364 14 UMA zone -- (already displayed) 2 owner graph lock -- (already displayed) 14 UMA zone -- (already displayed) 0 so_rcv_sx -- last acquired @ kern/uipc_sockbuf.c:148 10 so_rcv -- last acquired @ kern/uipc_socket.c:2477 12 sellck -- last acquired @ kern/sys_generic.c:759 11 radix node head -- last acquired @ net/route.c:147 12 rtentry -- last acquired @ netinet/ip_output.c:596 13 ifaddr -- last acquired @ net/route.c:813 14 UMA zone -- (already displayed) 13 sctp-addr -- last acquired @ netinet/sctp_pcb.c:649 13 vm page queue free mutex -- last acquired @ vm/vm_page.c:1316 13 sctp_it_wq -- last acquired @ netinet/sctputil.c:1345 13 rts_inq -- last acquired @ net/netisr.c:140 12 ifnet -- last acquired @ net/if.c:1473 14 UMA zone -- (already displayed) 13 eventhandler -- last acquired @ kern/subr_eventhandler.c:212 14 eventhandler list -- last acquired @ kern/kern_exit.c:233 13 if_addr_mtx -- last acquired @ netinet6/ip6_output.c:717 14 UMA zone -- (already displayed) 13 vm page queue free mutex -- (already displayed) 14 UMA zone -- (already displayed) 13 vm page queue free mutex -- (already displayed) 11 process lock -- last acquired @ kern/subr_sleepqueue.c:390 12 session -- last acquired @ kern/kern_proc.c:588 13 uidinfo hash -- last acquired @ kern/kern_resource.c:1213 14 uidinfo struct -- last acquired @ order list:0 14 sleep mtxpool -- last acquired @ kern/kern_event.c:993 15 kqueue -- last acquired @ kern/kern_event.c:1182 16 struct mount mtx -- last acquired @ kern/vfs_mount.c:2078 17 vnode interlock -- last acquired @ kern/vfs_default.c:577 18 cdev -- last acquired @ kern/kern_conf.c:71 18 vnode_free_list -- last acquired @ kern/vfs_subr.c:3012 18 Syncer mtx -- last acquired @ kern/vfs_subr.c:1573 13 tty -- last acquired @ kern/kern_event.c:1664 17 vnode interlock -- (already displayed) 12 sigacts -- last acquired @ kern/subr_sleepqueue.c:392 12 ktrace -- last acquired @ kern/kern_fork.c:600 14 sleep mtxpool -- (already displayed) 12 fdesc -- last acquired @ kern/kern_descrip.c:1539 14 sleep mtxpool -- (already displayed) 14 UMA zone -- (already displayed) 15 kqueue -- (already displayed) 14 UMA zone -- (already displayed) 2 unp_mtx -- last acquired @ kern/uipc_usrreq.c:557 10 so_rcv -- (already displayed) 8 accept -- last acquired @ kern/uipc_syscalls.c:371 9 so_snd -- last acquired @ kern/uipc_socket.c:2476 10 so_rcv -- (already displayed) 14 sleep mtxpool -- (already displayed) 11 process lock -- (already displayed) 11 radix node head -- (already displayed) 12 rtentry -- (already displayed) 10 tcp_hc_entry -- last acquired @ netinet/tcp_hostcache.c:291 14 UMA zone -- (already displayed) 13 vm page queue free mutex -- (already displayed) 12 sellck -- (already displayed) 14 UMA zone -- (already displayed) 15 kqueue -- (already displayed) 13 vm page queue free mutex -- (already displayed) 10 so_rcv -- (already displayed) 9 so_snd -- (already displayed) 14 UMA zone -- (already displayed) 11 process lock -- (already displayed) 4 user map -- last acquired @ vm/vm_fault.c:786 14 UMA zone -- (already displayed) 13 vm page queue free mutex -- (already displayed) 8 vm object_list -- last acquired @ vm/vm_object.c:225 7 standard object -- last acquired @ vm/vnode_pager.c:361 13 vm page queue free mutex -- (already displayed) 17 vnode interlock -- (already displayed) 8 vm page queue mutex -- last acquired @ vm/vm_object.c:1816 17 vnode interlock -- (already displayed) 9 pmap -- last acquired @ amd64/amd64/pmap.c:1961 13 vm page queue free mutex -- (already displayed) 13 vm page queue free mutex -- (already displayed) 8 vm object_list -- (already displayed) 14 UMA zone -- (already displayed) 8 swap_pager swhash -- last acquired @ vm/swap_pager.c:1888 8 vm page queue mutex -- (already displayed) 9 pmap -- (already displayed) 17 vnode interlock -- (already displayed) 6 buf queue lock -- last acquired @ kern/vfs_bio.c:1468 17 vnode interlock -- (already displayed) 6 needsbuffer lock -- last acquired @ kern/vfs_bio.c:321 6 system map -- last acquired @ vm/vm_map.c:2421 8 vm page queue mutex -- (already displayed) 7 kmem object -- last acquired @ vm/vm_object.c:441 13 vm page queue free mutex -- (already displayed) 8 vm page queue mutex -- (already displayed) 7 KMAP ENTRY -- last acquired @ vm/uma_core.c:2257 9 pmap -- (already displayed) 13 vm page queue free mutex -- (already displayed) 7 kernel object -- last acquired @ kern/vfs_bio.c:3680 13 vm page queue free mutex -- (already displayed) 8 vm page queue mutex -- (already displayed) 14 UMA zone -- (already displayed) 7 standard object -- (already displayed) 6 bio queue -- last acquired @ geom/geom_io.c:68 6 bdone lock -- last acquired @ kern/vfs_bio.c:3002 16 struct mount mtx -- (already displayed) 6 buffer daemon lock -- last acquired @ kern/vfs_bio.c:2107 8 swap_pager swhash -- (already displayed) 2 tcpinp -- last acquired @ netinet/tcp_usrreq.c:699 9 so_snd -- (already displayed) 14 UMA zone -- (already displayed) 13 vm page queue free mutex -- (already displayed) 10 so_rcv -- (already displayed) 12 ifnet -- (already displayed) 10 arc4_mtx -- last acquired @ libkern/arc4random.c:137 11 radix node head -- (already displayed) 12 rtentry -- (already displayed) 6 isn_mtx -- last acquired @ netinet/tcp_subr.c:1426 7 random reseed -- last acquired @ dev/random/yarrow.c:278 10 arc4_mtx -- (already displayed) 8 PFil hook read/write mutex -- last acquired @ net/pfil.c:73 9 pfil_head_list lock -- last acquired @ net/pfil.c:159 9 pf task mtx -- last acquired @ /data/usr/src/sys/modules/pf/../../contrib/pf/net/pf.c:6729 14 UMA zone -- (already displayed) 13 vm page queue free mutex -- (already displayed) 10 arc4_mtx -- (already displayed) 10 if send queue -- last acquired @ dev/re/if_re.c:2351 9 network driver -- last acquired @ dev/re/if_re.c:2085 14 UMA zone -- (already displayed) 13 vm page queue free mutex -- (already displayed) 10 UMA boot pages -- last acquired @ vm/uma_core.c:916 13 if_addr_mtx -- (already displayed) 10 taskqueue -- last acquired @ kern/subr_taskqueue.c:73 10 if send queue -- (already displayed) 10 bpf interface lock -- last acquired @ net/bpf.c:1358 11 bpf cdev lock -- last acquired @ net/bpf.c:1362 12 sellck -- (already displayed) 10 tcp_hc_entry -- (already displayed) 8 accept -- (already displayed) 3 tcp_sc_head -- last acquired @ netinet/tcp_syncache.c:477 14 UMA zone -- (already displayed) 13 vm page queue free mutex -- (already displayed) 10 arc4_mtx -- (already displayed) 11 radix node head -- (already displayed) 12 rtentry -- (already displayed) 10 tcp_hc_entry -- (already displayed) 8 PFil hook read/write mutex -- (already displayed) 10 if send queue -- (already displayed) 9 network driver -- (already displayed) 3 so_glabel -- last acquired @ kern/uipc_socket.c:280 8 ip_inq -- last acquired @ net/netisr.c:140 0 so_snd_sx -- last acquired @ kern/uipc_sockbuf.c:148 9 so_snd -- (already displayed) 14 UMA zone -- (already displayed) 1 unp_global_rwlock -- last acquired @ kern/uipc_usrreq.c:556 2 unp_mtx -- (already displayed) 8 accept -- (already displayed) 14 UMA zone -- (already displayed) 3 so_glabel -- (already displayed) 9 so_snd -- (already displayed) 13 vm page queue free mutex -- (already displayed) 2 tcpinp -- (already displayed) 6 udp -- last acquired @ netinet/udp_usrreq.c:385 7 udpinp -- last acquired @ netinet/udp_usrreq.c:549 8 in_multi_mtx -- last acquired @ netinet/in_mcast.c:320 9 igmp_mtx -- last acquired @ netinet/igmp.c:446 13 if_addr_mtx -- (already displayed) 13 if_addr_mtx -- (already displayed) 14 UMA zone -- (already displayed) 9 network driver -- (already displayed) 9 so_snd -- (already displayed) 12 ifnet -- (already displayed) 14 UMA zone -- (already displayed) 10 arc4_mtx -- (already displayed) 11 radix node head -- (already displayed) 12 rtentry -- (already displayed) 8 PFil hook read/write mutex -- (already displayed) 10 if send queue -- (already displayed) 9 network driver -- (already displayed) 10 so_rcv -- (already displayed) 8 accept -- (already displayed) 8 ip_inq -- (already displayed) 10 bpf interface lock -- (already displayed) 8 if_afdata -- last acquired @ netinet6/scope6.c:406 9 scope6_lock -- last acquired @ netinet6/scope6.c:435 8 addrsel_lock -- last acquired @ netinet6/in6_src.c:858 10 tcp_hc_entry -- (already displayed) 8 ip6_inq -- last acquired @ net/netisr.c:140 14 UMA zone -- (already displayed) 13 vm page queue free mutex -- (already displayed) 13 vm page queue free mutex -- (already displayed) 11 process lock -- (already displayed) 4 user map -- (already displayed) 7 standard object -- (already displayed) 6 system map -- (already displayed) 7 random reseed -- (already displayed) 6 bio queue -- (already displayed) 17 vnode interlock -- (already displayed) 0 pf_statetbl_lock -- last acquired @ /data/usr/src/sys/modules/pf/../../contrib/pf/net/pf.c:979 9 pf task mtx -- (already displayed) 14 UMA zone -- (already displayed) 9 pfil_head_list lock -- (already displayed) 8 PFil hook read/write mutex -- (already displayed) 0 if_bridge -- last acquired @ /data/usr/src/sys/modules/if_bridge/../../net/if_bridge.c:1175 1 bstp -- last acquired @ /data/usr/src/sys/modules/bridgestp/../../net/bridgestp.c:2163 12 ifnet -- (already displayed) 12 ifnet -- (already displayed) 14 UMA zone -- (already displayed) 0 if_bridge list -- last acquired @ /data/usr/src/sys/modules/if_bridge/../../net/if_bridge.c:610 0 bridgestp list -- last acquired @ /data/usr/src/sys/modules/bridgestp/../../net/bridgestp.c:2143 0 zvol -- last acquired @ /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zvol.c:348 0 zfs:vdev:geom:queue -- last acquired @ /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_geom.c:203 0 devfsmount -- last acquired @ fs/devfs/devfs_vnops.c:201 6 devfs interlock -- last acquired @ fs/devfs/devfs_vnops.c:194 17 vnode interlock -- (already displayed) 18 cdev -- (already displayed) 18 vnode_free_list -- (already displayed) 14 UMA zone -- (already displayed) 17 vnode interlock -- (already displayed) 16 struct mount mtx -- (already displayed) 18 cdev -- (already displayed) 13 vm page queue free mutex -- (already displayed) 1 DEVFS ruleset lock -- last acquired @ fs/devfs/devfs_rule.c:177 14 UMA zone -- (already displayed) 11 process lock -- (already displayed) 4 user map -- (already displayed) 0 gmirror:lock -- last acquired @ /data/usr/src/sys/modules/geom/geom_mirror/../../../geom/mirror/g_mirror.c:3121 1 gmirror:events -- last acquired @ /data/usr/src/sys/modules/geom/geom_mirror/../../../geom/mirror/g_mirror.c:218 1 gmirror:queue -- last acquired @ /data/usr/src/sys/modules/geom/geom_mirror/../../../geom/mirror/g_mirror.c:1833 14 UMA zone -- (already displayed) 1 GEOM topology -- last acquired @ geom/geom_event.c:233 6 GEOM orphanage -- last acquired @ geom/geom_event.c:201 14 UMA zone -- (already displayed) 13 eventhandler -- (already displayed) 14 eventhandler list -- (already displayed) 6 devstat -- last acquired @ kern/subr_devstat.c:83 13 vm page queue free mutex -- (already displayed) 6 unit# allocation -- last acquired @ kern/subr_unit.c:623 18 cdev -- (already displayed) 6 bio queue -- (already displayed) 6 bdone lock -- (already displayed) 10 UMA boot pages -- (already displayed) 6 TID lock -- last acquired @ kern/subr_unit.c:752 8 vm object_list -- (already displayed) 6 system map -- (already displayed) 7 standard object -- (already displayed) 2 proctree -- last acquired @ kern/tty.c:2080 3 allproc -- last acquired @ kern/kern_exit.c:801 4 allprison -- last acquired @ kern/kern_jail.c:951 14 sleep mtxpool -- (already displayed) 11 process lock -- (already displayed) 12 fdesc -- (already displayed) 4 filedesc structure -- last acquired @ kern/kern_descrip.c:2054 17 vnode interlock -- (already displayed) 11 process lock -- (already displayed) 14 sleep mtxpool -- (already displayed) 6 Name Cache -- last acquired @ kern/vfs_cache.c:327 17 vnode interlock -- (already displayed) 14 UMA zone -- (already displayed) 10 UMA boot pages -- (already displayed) 12 fdesc -- (already displayed) 18 cdev -- (already displayed) 5 Giant -- last acquired @ kern/kern_timeout.c:241 6 pipe mutex -- last acquired @ kern/sys_pipe.c:1331 7 sigio lock -- last acquired @ kern/kern_descrip.c:1052 8 process group -- last acquired @ kern/kern_proc.c:439 11 process lock -- (already displayed) 12 session -- (already displayed) 14 UMA zone -- (already displayed) 12 sellck -- (already displayed) 15 kqueue -- (already displayed) 14 UMA zone -- (already displayed) 6 system map -- (already displayed) 10 UMA boot pages -- (already displayed) 13 vm page queue free mutex -- (already displayed) 6 UMA lock -- last acquired @ vm/uma_core.c:1492 14 UMA zone -- (already displayed) 7 KMAP ENTRY -- (already displayed) 13 eventhandler -- (already displayed) 14 eventhandler list -- (already displayed) 6 kobj -- last acquired @ kern/subr_kobj.c:258 7 kernel environment -- last acquired @ kern/kern_environment.c:301 6 malloc -- last acquired @ kern/kern_malloc.c:741 7 kernel object -- (already displayed) 8 vm object_list -- (already displayed) 7 KMAP ENTRY -- (already displayed) 13 uidinfo hash -- (already displayed) 11 process lock -- (already displayed) 14 sleep mtxpool -- (already displayed) 6 evclass_mtx -- last acquired @ security/audit/audit_bsm_klib.c:113 6 TID lock -- (already displayed) 7 standard object -- (already displayed) 6 intr event -- last acquired @ kern/kern_intr.c:454 18 cdev -- (already displayed) 6 GEOM orphanage -- (already displayed) 6 ttylist -- last acquired @ kern/tty.c:2855 13 tty -- (already displayed) 6 intr config -- last acquired @ kern/subr_autoconf.c:72 6 XPT lock -- last acquired @ cam/cam_xpt.c:2668 14 UMA zone -- (already displayed) 7 XPT topology lock -- last acquired @ cam/cam_xpt.c:2673 7 kernel environment -- (already displayed) 10 taskqueue -- (already displayed) 6 taskqueue list -- last acquired @ kern/subr_taskqueue.c:125 6 rman head -- last acquired @ kern/subr_rman.c:152 6 rman -- last acquired @ kern/subr_rman.c:539 14 UMA zone -- (already displayed) 13 vm page queue free mutex -- (already displayed) 6 devd -- last acquired @ kern/subr_bus.c:489 12 sellck -- (already displayed) 12 sellck -- (already displayed) 6 ACPI semaphore -- last acquired @ dev/acpica/Osd/OsdSynch.c:303 6 acpi subsystem HW lock -- last acquired @ dev/acpica/Osd/OsdSynch.c:377 6 acpi subsystem GPE lock -- last acquired @ dev/acpica/Osd/OsdSynch.c:377 10 taskqueue -- (already displayed) 9 network driver -- (already displayed) 6 bounce pages lock -- last acquired @ amd64/amd64/busdma_machdep.c:1070 12 ifnet -- (already displayed) 6 bpf global lock -- last acquired @ net/bpf.c:394 10 bpf interface lock -- (already displayed) 6 unit# allocation -- (already displayed) 18 vnode_free_list -- (already displayed) 6 pfs_node -- last acquired @ fs/pseudofs/pseudofs_internal.h:103 6 pfs_fileno -- last acquired @ kern/subr_unit.c:623 7 random reseed -- (already displayed) 10 arc4_mtx -- (already displayed) 7 mountlist -- last acquired @ kern/vfs_subr.c:3197 16 struct mount mtx -- (already displayed) 6 nfsd_mtx -- last acquired @ nfsserver/nfs_srvsock.c:799 9 so_snd -- (already displayed) 14 sleep mtxpool -- (already displayed) 10 so_rcv -- (already displayed) 6 if_clone lock -- last acquired @ net/if_clone.c:164 6 if_cloners lock -- last acquired @ net/if_clone.c:129 6 domain list -- last acquired @ kern/uipc_domain.c:228 9 pfil_head_list lock -- (already displayed) 8 PFil hook read/write mutex -- (already displayed) 6 isn_mtx -- (already displayed) 11 radix node head -- (already displayed) 6 lo_mtx -- last acquired @ net/if_loop.c:157 7 XPT topology lock -- (already displayed) 6 ATA queue lock -- last acquired @ dev/ata/ata-queue.c:177 7 ATA state lock -- last acquired @ dev/ata/ata-queue.c:194 6 devstat -- (already displayed) 6 NFS iod lock -- last acquired @ nfsclient/nfs_nfsiod.c:196 16 struct mount mtx -- (already displayed) 6 mntid -- last acquired @ kern/vfs_subr.c:460 7 mountlist -- (already displayed) 17 vnode interlock -- (already displayed) 6 buf queue lock -- (already displayed) 6 bio queue -- (already displayed) 6 bdone lock -- (already displayed) 6 needsbuffer lock -- (already displayed) 6 FFS Lock -- last acquired @ ufs/ffs/ffs_vfsops.c:1150 10 arc4_mtx -- (already displayed) 6 Name Cache -- (already displayed) 6 knlist lock for lockless objects -- last acquired @ kern/kern_event.c:1664 6 vfs hash -- last acquired @ kern/vfs_hash.c:114 17 vnode interlock -- (already displayed) 6 devfs interlock -- (already displayed) 6 dirhash list -- last acquired @ ufs/ufs/ufs_dirhash.c:349 7 dirhash -- last acquired @ ufs/ufs/ufs_dirhash.c:368 7 dirhash -- (already displayed) 6 pbuf mutex -- last acquired @ vm/vm_pager.c:413 8 vm page queue mutex -- (already displayed) 8 process group -- (already displayed) 13 tty -- (already displayed) 12 session -- (already displayed) 7 sigio lock -- (already displayed) 6 buffer daemon lock -- (already displayed) 8 if_afdata -- (already displayed) 13 if_addr_mtx -- (already displayed) 8 ip6_inq -- (already displayed) 6 udp -- (already displayed) 9 so_snd -- (already displayed) 6 pipe mutex -- (already displayed) 15 kqueue -- (already displayed) 14 UMA zone -- (already displayed) 11 bpf cdev lock -- (already displayed) 6 system map -- (already displayed) 17 vnode interlock -- (already displayed) 4 user map -- (already displayed) 8 process group -- (already displayed) 5 Giant -- (already displayed) 11 process lock -- (already displayed) 12 session -- (already displayed) 7 sigio lock -- (already displayed) 3 clone events drain lock -- last acquired @ fs/devfs/devfs_vnops.c:626 13 eventhandler -- (already displayed) 14 eventhandler list -- (already displayed) 14 UMA zone -- (already displayed) 18 cdev -- (already displayed) 4 filedesc structure -- (already displayed) 11 process lock -- (already displayed) 7 mountlist -- (already displayed) 17 vnode interlock -- (already displayed) 2 swapdev -- last acquired @ vm/swap_pager.c:2029 2 unique lock -- last acquired @ /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/unique.c:100 14 UMA zone -- (already displayed) 7 random reseed -- (already displayed) 6 bio queue -- (already displayed) 6 bdone lock -- (already displayed) 7 mountlist -- (already displayed) 0 MD config lock -- last acquired @ dev/md/md.c:1161 14 UMA zone -- (already displayed) 13 eventhandler -- (already displayed) 2 proctree -- (already displayed) 4 filedesc structure -- (already displayed) 11 process lock -- (already displayed) 8 vm object_list -- (already displayed) 7 standard object -- (already displayed) 1 GEOM topology -- (already displayed) 0 g_disk_done -- last acquired @ geom/geom_disk.c:198 14 UMA zone -- (already displayed) 6 bio queue -- (already displayed) 0 iterator -- last acquired @ netinet/sctputil.c:1209 0 ipqlock -- last acquired @ netinet/ip_input.c:1086 0 ip6qlock -- last acquired @ netinet6/frag6.c:688 0 msq -- last acquired @ kern/sysv_msg.c:1027 0 semid -- last acquired @ kern/sysv_sem.c:658 1 sem -- last acquired @ kern/sysv_sem.c:1288 0 intr sources -- last acquired @ amd64/amd64/intr_machdep.c:593 0 audit_mtx -- last acquired @ security/audit/audit_worker.c:393 0 uma object -- last acquired @ vm/vm_meter.c:115 0 p_peers -- last acquired @ kern/kern_exit.c:284 0 runningbufspace lock -- last acquired @ kern/vfs_bio.c:340 0 umtxql -- last acquired @ kern/kern_umtx.c:300 0 Softdep Lock -- last acquired @ ufs/ffs/ffs_softdep.c:6255 0 protect sysfilt_ops -- last acquired @ kern/kern_event.c:774 0 ACPI root bus -- last acquired @ dev/acpica/acpi.c:1022 6 rman -- (already displayed) 6 system map -- (already displayed) 14 UMA zone -- (already displayed) 6 ACPI semaphore -- (already displayed) 0 accept_filter_mtx -- last acquired @ kern/uipc_accf.c:116 0 ACPI PCI bus methods -- last acquired @ dev/acpica/acpi_pcib.c:221 0 rtsock route_cb lock -- last acquired @ net/rtsock.c:236 0 rawcb -- last acquired @ net/raw_cb.c:104 10 so_rcv -- (already displayed) 14 UMA zone -- (already displayed) 0 vm daemon -- last acquired @ vm/vm_pageout.c:1531 0 ACPI PCI link -- last acquired @ dev/acpica/acpi_pci_link.c:432 14 UMA zone -- (already displayed) 6 ACPI semaphore -- (already displayed) 0 lock states lock -- last acquired @ kern/kern_lockf.c:700 0 sysctl lock -- last acquired @ kern/kern_sysctl.c:1396 10 arc4_mtx -- (already displayed) 14 UMA zone -- (already displayed) 3 allproc -- (already displayed) 11 process lock -- (already displayed) 4 user map -- (already displayed) 18 cdev -- (already displayed) 4 filedesc structure -- (already displayed) 12 fdesc -- (already displayed) 1 kernel linker -- last acquired @ kern/kern_linker.c:1170 14 UMA zone -- (already displayed) 2 module subsystem sx lock -- last acquired @ kern/kern_module.c:407 14 UMA zone -- (already displayed) 13 vm page queue free mutex -- (already displayed) 4 filedesc structure -- (already displayed) 17 vnode interlock -- (already displayed) 6 Name Cache -- (already displayed) 6 buf queue lock -- (already displayed) 6 needsbuffer lock -- (already displayed) 6 vfs hash -- (already displayed) 18 vnode_free_list -- (already displayed) 16 struct mount mtx -- (already displayed) 6 system map -- (already displayed) 7 standard object -- (already displayed) 6 bio queue -- (already displayed) 6 bdone lock -- (already displayed) 6 dirhash list -- (already displayed) 13 vm page queue free mutex -- (already displayed) 7 kernel object -- (already displayed) 7 dirhash -- (already displayed) 18 cdev -- (already displayed) 8 vm object_list -- (already displayed) 6 pbuf mutex -- (already displayed) 6 buffer daemon lock -- (already displayed) 6 kobj -- (already displayed) 1 filelist lock -- last acquired @ kern/kern_descrip.c:2246 4 filedesc structure -- (already displayed) 6 GEOM orphanage -- (already displayed) 5 Giant -- (already displayed) 12 ktrace -- (already displayed) 6 malloc -- (already displayed) 6 system map -- (already displayed) 6 devstat -- (already displayed) 6 ttylist -- (already displayed) 8 vm object_list -- (already displayed) 6 UMA lock -- (already displayed) 18 Syncer mtx -- (already displayed) 1 unp_global_rwlock -- (already displayed) 1 tcp -- last acquired @ netinet/tcp_timer.c:128 2 tcpinp -- (already displayed) 14 UMA zone -- (already displayed) 13 vm page queue free mutex -- (already displayed) 8 accept -- (already displayed) 14 sleep mtxpool -- (already displayed) 3 so_glabel -- (already displayed) 3 tcp_sc_head -- (already displayed) 6 udp -- (already displayed) 1 sctp-info -- last acquired @ netinet/sctp_pcb.c:6138 1 rip -- last acquired @ netinet/raw_ip.c:613 14 UMA zone -- (already displayed) 13 vm page queue free mutex -- (already displayed) 2 rawinp -- last acquired @ netinet/in_pcb.c:221 1 addrsel_sxlock -- last acquired @ netinet6/in6_src.c:1023 8 addrsel_lock -- (already displayed) 1 db_capture_sx -- last acquired @ ddb/db_capture.c:181 1 db_script_mtx -- last acquired @ ddb/db_script.c:481 6 ACPI semaphore -- (already displayed) 4 allprison -- (already displayed) 1 cpufreq lock -- last acquired @ kern/kern_cpu.c:413 14 UMA zone -- (already displayed) 13 vm page queue free mutex -- (already displayed) 12 ifnet -- (already displayed) 1 ACPI CPU -- last acquired @ dev/acpica/acpi_cpu.c:1187 13 eventhandler -- (already displayed) 14 eventhandler list -- (already displayed) 13 vm page queue free mutex -- (already displayed) 0 ng_node -- last acquired @ order list:0 1 ng_worklist -- last acquired @ order list:0 0 802.11 com lock -- last acquired @ order list:0 0 ddp_list_mtx -- last acquired @ order list:0 1 ddp_mtx -- last acquired @ order list:0 0 slip_mtx -- last acquired @ order list:0 1 slip sc_mtx -- last acquired @ order list:0 0 unp -- last acquired @ order list:0 9 so_snd -- (already displayed) Spin locks: Locks which were never acquired: SCSI CD Changer List arp_inq pfs_vncache gif_mtx ppp_softc_list_mtx tunmtx nfs4dev state nfs4dev waitq nfs4dev newq ntfs nthash NFS xid lock NFS reqq lock ACPI global lock CAM SIMQ lock hptlock LED sx LED mtx audit_pipe_mtx nfslock msi audit_worker_sx audit_trigger_mtx ktrace_sx bpin lock zfs_debug fifo mutex pmc shared lock kqueue order securelevel mutex lock firmware table ACPI power resources encapmtx ACPI cmbat net80211 instances UUID generator mutex lock ACPI generic battery acct_sx ACPI PCI power methods ACPI lid ACPI HPET support ACPI embedded controller ACPI AC adapter ACPI thermal zone MSDOSFS fileno ACPI Smart Battery phys_pager list dev_pager list swap_pager list vm map sleep mutex db> show alllocks db> bt Tracing pid 135 tid 100054 td 0xffffff00026469f0 kdb_enter_why() at kdb_enter_why+0x3d panic() at panic+0x176 _mtx_assert() at _mtx_assert+0xe2 sleepq_broadcast() at sleepq_broadcast+0xaa taskq_thread() at taskq_thread+0x1f9 fork_exit() at fork_exit+0x12a fork_trampoline() at fork_trampoline+0xe --- trap 0, rip = 0, rsp = 0xffffffffbfbecd30, rbp = 0 --- db> panic Script started on Wed Jun 11 08:37:26 2008 You have mail. pegasus# kgsbdb /boot/kernel/kernel /var/carsrsash/ bounds info.0 info.1 info.2 vmcore.0 vmcore.1 vmcore.2 pegasus# kgdb /boot/kernel/kernel /var/crash/vm vmcore.0 vmcore.1 vmcore.2 pegasus# kgdb /boot/kernel/kernel /var/crash/vmcore..2 [GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so: Undefined symbol "ps_pglobal_lookup"] GNU gdb 6.1.1 [FreeBSD] Copyright 2004 Free Software Foundation, Inc. GDB is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain conditions. Type "show copying" to see the conditions. There is absolutely no warranty for GDB. Type "show warranty" for details. This GDB was configured as "amd64-marcel-freebsd". There is no member named pathname. Reading symbols from /boot/kernel/zfs.ko...Reading symbols from /boot/kernel/zfs.ko.symbols...done. done. Loaded symbols for /boot/kernel/zfs.ko Reading symbols from /boot/kernel/geom_mirror.ko...Reading symbols from /boot/kernel/geom_mirror.ko.symbols...done. done. Loaded symbols for /boot/kernel/geom_mirror.ko Reading symbols from /boot/kernel/speaker.ko...Reading symbols from /boot/kernel/speaker.ko.symbols...done. done. Loaded symbols for /boot/kernel/speaker.ko Reading symbols from /boot/kernel/wlan_xauth.ko...Reading symbols from /boot/kernel/wlan_xauth.ko.symbols...done. done. Loaded symbols for /boot/kernel/wlan_xauth.ko Reading symbols from /boot/kernel/if_bridge.ko...Reading symbols from /boot/kernel/if_bridge.ko.symbols...done. done. Loaded symbols for /boot/kernel/if_bridge.ko Reading symbols from /boot/kernel/bridgestp.ko...Reading symbols from /boot/kernel/bridgestp.ko.symbols...done. done. Loaded symbols for /boot/kernel/bridgestp.ko Reading symbols from /boot/kernel/pf.ko...Reading symbols from /boot/kernel/pf.ko.symbols...done. done. Loaded symbols for /boot/kernel/pf.ko Unread portion of the kernel message buffer: panic: mutex sleepq chain (0xffffffff8102bf08) not owned at ../../../kern/subr_sleepqueue.c:778 cpuid = 0 KDB: enter: panic panic: from debugger cpuid = 0 Uptime: 1m13s Physical memory: 2027 MB Dumping 291 MB: 276 260 244 228 212 196 180 164 148 132 116 100 84 68 52 36 20 4 #0 doadump () at pcpu.h:194 194 pcpu.h: No such file or directory. in pcpu.h (kgdb) bt #0 doadump () at pcpu.h:194 #1 0xffffffff80491bd3 in boot (howto=260) at ../../../kern/kern_shutdown.c:418 #2 0xffffffff80492057 in panic (fmt=Variable "fmt" is not available. ) at ../../../kern/kern_shutdown.c:572 #3 0xffffffff801c6717 in db_panic (addr=Variable "addr" is not available. ) at ../../../ddb/db_command.c:446 #4 0xffffffff801c700f in db_command (last_cmdp=0xffffffff80a64708, cmd_table=0x0, dopager=1) at ../../../ddb/db_command.c:413 #5 0xffffffff801c7220 in db_command_loop () at ../../../ddb/db_command.c:466 #6 0xffffffff801c8b59 in db_trap (type=Variable "type" is not available. ) at ../../../ddb/db_main.c:228 #7 0xffffffff804bdad5 in kdb_trap (type=3, code=0, tf=0xffffffffbfbec920) at ../../../kern/subr_kdb.c:524 #8 0xffffffff80753055 in trap (frame=0xffffffffbfbec920) at ../../../amd64/amd64/trap.c:499 #9 0xffffffff807387de in calltrap () at ../../../amd64/amd64/exception.S:169 #10 0xffffffff804bdcad in kdb_enter_why (why=0xffffffff808165ce "panic", msg=0xa
) at cpufunc.h:63 #11 0xffffffff80492066 in panic (fmt=Variable "fmt" is not available. ) at ../../../kern/kern_shutdown.c:556 #12 0xffffffff80484a12 in _mtx_assert (m=Variable "m" is not available. ) at ../../../kern/kern_mutex.c:644 #13 0xffffffff804c5bba in sleepq_broadcast (wchan=0xffffff00022987a0, flags=Variable "flags" is not available. ) at ../../../kern/subr_sleepqueue.c:778 #14 0xffffffff812392a9 in taskq_thread (arg=Variable "arg" is not available. ) at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:827 #15 0xffffffff804712ea in fork_exit ( ---Type to continue, or q to quit---q callout=0xffffffff812390b0 ,Quit ) at ../../../kern/kern_fork.c:783 #16 0xffffffff80738bae in fork_trampoline () at ../../../amd64/amd64/exception.S:415 #17 0x0000000000000000 in ?? () #18 0x0000000000000000 in ?? () #19 0x0000000000000001 in ?? () #20 0x0000000000000000 in ?? () #21 0x0000000000000000 in ?? () #22 0x0000000000000000 in ?? () #23 0x0000000000000000 in ?? () #24 0x0000000000000000 in ?? () #25 0x0000000000000000 in ?? () #26 0x0000000000000000 in ?? () #27 0x0000000000000000 in ?? () #28 0x0000000000000000 in ?? () #29 0x0000000000000000 in ?? () #30 0x0000000000000000 in ?? () #31 0x0000000000000000 in ?? () #32 0x0000000000000000 in ?? () #33 0x0000000000000000 in ?? () #34 0x0000000000000000 in ?? () #35 0x0000000000000000 in ?? () #36 0x0000000000000000 in ?? () #37 0x0000000000000000 in ?? () ---Type to continue, or q to quit---q Quit (kgdb) up 14 #14 0xffffffff812392a9 in taskq_thread (arg=Variable "arg" is not available. ) at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:827 827 mutex_exit(&tq->tq_lock); (kgdb) bt  fra   l 822 tq->tq_active++; 823 continue; 824 } 825 tqe->tqent_prev->tqent_next = tqe->tqent_next; 826 tqe->tqent_next->tqent_prev = tqe->tqent_prev; 827 mutex_exit(&tq->tq_lock); 828 829 rw_enter(&tq->tq_threadlock, RW_READER); 830 start = gethrtime(); 831 DTRACE_PROBE2(taskq__exec__start, taskq_t *, tq, (kgdb) quit pegasus# exit exit Script done on Wed Jun 11 08:38:15 2008