**** Please note I had to disable the atomic_cmp_set_rel_int barrier in CSR to get the race to occur. KTR may be slightly out of order due to it **** Dump header from device /dev/ad6s1d Architecture: amd64 Architecture Version: 2 Dump Length: 312967168B (298 MB) Blocksize: 512 Dumptime: Thu Jun 12 22:59:32 2008 Hostname: pegasus.clearchain.com Magic: FreeBSD Kernel Dump Version String: FreeBSD 7.0-STABLE #21: Thu Jun 12 22:37:14 CST 2008 root@pegasus.clearchain.com:/data/usr/src/sys/amd64/compile/PEGASUS Panic String: mutex sleepq chain (0xffffffff81025618) not owned at ../../../kern/subr_sleepqueue.c:778 Dump Parity: 3835743747 Bounds: 3 Dump Status: good db> show lock 0xffffffff81025618 class: spin mutex name: sleepq chain flags: {SPIN, RECURSE} state: {UNOWNED} db> bt Tracing pid 134 tid 100064 td 0xffffff0002627350 kdb_enter_why() at kdb_enter_why+0x3d panic() at panic+0x176 _mtx_assert() at _mtx_assert+0xe2 sleepq_broadcast() at sleepq_broadcast+0xaa taskq_thread() at taskq_thread+0x1f9 fork_exit() at fork_exit+0x12a fork_trampoline() at fork_trampoline+0xe --- trap 0, rip = 0, rsp = 0xffffffffbfa59d30, rbp = 0 --- db> show thread 100064 Thread 100064 at 0xffffff0002627350: proc (pid 134): 0xffffff0002623468 name: spa_zio_intr_2 flags: 0x4 pflags: 0 state: RUNNING (CPU 0) priority: 96 container lock: sched lock 0 (0xffffffff810235c0) db> show lock 0xffffffff810235c0 class: spin mutex name: sched lock 0 type: sched lock flags: {SPIN, RECURSE} state: {UNOWNED} db> show allpcpu Current CPU: 0 cpuid = 0 curthread = 0xffffff0002627350: pid 134 "spa_zio_intr_2" curpcb = 0xffffffffbfa59d40 fpcurthread = none idlethread = 0xffffff0002170350: pid 12 "idle: cpu0" cpuid = 1 curthread = 0xffffff00021706a0: pid 11 "idle: cpu1" curpcb = 0xffffffffbd476d40 fpcurthread = none idlethread = 0xffffff00021706a0: pid 11 "idle: cpu1" db> show ktr 20612 (0xffffff00024fc350:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff81025618 held @ ../../../kern/subr_sleepqueue.c:264 20611 (0xffffff00024fc350:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff81025618 held @ ../../../kern/subr_sleepqueue.c:232 20610 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&tq->tq_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:839 20609 (0xffffff00024fc350:cpu1): _sx_xlock_hard: 0xffffff0002299de0 resuming from sleep queue 20608 (0xffffff0002627350:cpu0): _mtx_unlock_spin_flags: mutex 0xffffffff810255e0 held @ ../../../kern/kern_mutex.c:568 20607 (0xffffff0002627350:cpu0): _thread_lock_flags: mutex 0xffffffff810255e0 held @ ../../../kern/subr_sleepqueue.c:779 20606 (0xffffff0002627350:cpu0): _sx_xunlock_hard: 0xffffff0002299de0 waking up all threads on exclusive queue 20605 (0xffffff0002627350:cpu0): _mtx_lock_spin_flags: mutex 0xffffffff810255e0 held @ ../../../kern/subr_sleepqueue.c:232 20604 (0xffffff0002627350:cpu0): _sx_xunlock_hard: 0xffffff0002299de0 contested 20603 (0xffffff0002627350:cpu0): XUNLOCK (sx) zfs:&tq->tq_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:827 20602 (0xffffff00024fc350:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff810255e0 held @ ../../../kern/sched_ule.c:1771 20601 (0xffffff00024fc350:cpu1): _sx_xlock_hard: 0xffffff0002299de0 blocking on sleep queue 20600 (0xffffff00024fc350:cpu1): _sx_xlock_hard: 0xffffff0002299de0 set excl waiters flag 20599 (0xffffff00024fc350:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff810255e0 held @ ../../../kern/subr_sleepqueue.c:232 20598 (0xffffff00024fc350:cpu1): _sx_xlock_hard: zfs:&tq->tq_lock contested (lock=0xffffff0002627350) at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:839 20597 (0xffffff00024fc350:cpu1): SUNLOCK (sx) zfs:&tq->tq_threadlock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:837 20596 (0xffffff0002627350:cpu0): XLOCK (sx) zfs:&tq->tq_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:839 20595 (0xffffff00024fc350:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff81025b90 held @ ../../../kern/subr_sleepqueue.c:264 20594 (0xffffff00024fc350:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff81025b90 held @ ../../../kern/subr_sleepqueue.c:232 20593 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&tq->tq_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:733 20592 (0xffffff00024fc350:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff81025618 held @ ../../../kern/subr_sleepqueue.c:264 20591 (0xffffff00024fc350:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff81025618 held @ ../../../kern/subr_sleepqueue.c:232 20590 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&tq->tq_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:724 20589 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:827 20588 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:822 20587 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:960 20586 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:951 20585 (0xffffff0002627350:cpu0): SUNLOCK (sx) zfs:&tq->tq_threadlock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:837 20584 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&vd->vdev_stat_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c:1637 20583 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&vd->vdev_stat_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c:1634 20582 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:809 20581 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:806 20580 (0xffffff00024fc350:cpu1): SLOCK (sx) zfs:&tq->tq_threadlock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:829 20579 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&tq->tq_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:827 20578 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&tq->tq_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:839 20577 (0xffffff00024fc350:cpu1): SUNLOCK (sx) zfs:&tq->tq_threadlock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:837 20576 (0xffffff00024fc350:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff81025b90 held @ ../../../kern/subr_sleepqueue.c:264 20575 (0xffffff00024fc350:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff81025b90 held @ ../../../kern/subr_sleepqueue.c:232 20574 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:830 20573 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:822 20572 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:960 20571 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:951 20570 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&vd->vdev_stat_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c:1637 20569 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&vd->vdev_stat_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c:1634 20568 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:809 20567 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:806 20566 (0xffffff00024fc350:cpu1): SLOCK (sx) zfs:&tq->tq_threadlock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:829 20565 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&tq->tq_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:827 20564 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&tq->tq_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:839 20563 (0xffffff00024fc350:cpu1): SUNLOCK (sx) zfs:&tq->tq_threadlock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:837 20562 (0xffffff00024fc350:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff81025c38 held @ ../../../kern/subr_sleepqueue.c:264 20561 (0xffffff00024fc350:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff81025c38 held @ ../../../kern/subr_sleepqueue.c:232 20560 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:830 20559 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:822 20558 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:960 20557 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:951 20556 (0xffffff0002627350:cpu0): _mtx_unlock_spin_flags: mutex 0xffffffff810257a0 held @ ../../../kern/subr_sleepqueue.c:264 20555 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&vd->vdev_stat_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c:1637 20554 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&vd->vdev_stat_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c:1634 20553 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:809 20552 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:806 20551 (0xffffff00024fc350:cpu1): SLOCK (sx) zfs:&tq->tq_threadlock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:829 20550 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&tq->tq_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:827 20549 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&tq->tq_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:839 20548 (0xffffff00024fc350:cpu1): SUNLOCK (sx) zfs:&tq->tq_threadlock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:837 20547 (0xffffff0002627350:cpu0): _mtx_lock_spin_flags: mutex 0xffffffff810257a0 held @ ../../../kern/subr_sleepqueue.c:232 20546 (0xffffff00024fc350:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff81025ce0 held @ ../../../kern/subr_sleepqueue.c:264 20545 (0xffffff00024fc350:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff81025ce0 held @ ../../../kern/subr_sleepqueue.c:232 20544 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:830 20543 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:822 20542 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:960 20541 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:951 20540 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&vd->vdev_stat_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c:1637 20539 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&vd->vdev_stat_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c:1634 20538 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:809 20537 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:806 20536 (0xffffff0002627350:cpu0): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:830 20535 (0xffffff0002627350:cpu0): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:822 20534 (0xffffff00024fc350:cpu1): SLOCK (sx) zfs:&tq->tq_threadlock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:829 20533 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&tq->tq_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:827 20532 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&tq->tq_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:839 20531 (0xffffff00024fc350:cpu1): SUNLOCK (sx) zfs:&tq->tq_threadlock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:837 20530 (0xffffff00024fc350:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff81026060 held @ ../../../kern/subr_sleepqueue.c:264 20529 (0xffffff00024fc350:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff81026060 held @ ../../../kern/subr_sleepqueue.c:232 20528 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:830 20527 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:822 20526 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:960 20525 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:951 20524 (0xffffff0002627350:cpu0): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:960 20523 (0xffffff0002627350:cpu0): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:951 20522 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&vd->vdev_stat_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c:1637 20521 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&vd->vdev_stat_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c:1634 20520 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:809 20519 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:806 20518 (0xffffff00024fc350:cpu1): SLOCK (sx) zfs:&tq->tq_threadlock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:829 20517 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&tq->tq_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:827 20516 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&tq->tq_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:839 20515 (0xffffff00024fc350:cpu1): SUNLOCK (sx) zfs:&tq->tq_threadlock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:837 20514 (0xffffff00024fc350:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff810266b8 held @ ../../../kern/subr_sleepqueue.c:264 20513 (0xffffff00024fc350:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff810266b8 held @ ../../../kern/subr_sleepqueue.c:232 20512 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:830 20511 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:822 20510 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:960 20509 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:951 20508 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&vd->vdev_stat_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c:1637 20507 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&vd->vdev_stat_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c:1634 20506 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:809 20505 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:806 20504 (0xffffff00024fc350:cpu1): SLOCK (sx) zfs:&tq->tq_threadlock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:829 20503 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&tq->tq_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:827 20502 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&tq->tq_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:839 20501 (0xffffff00024fc350:cpu1): SUNLOCK (sx) zfs:&tq->tq_threadlock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:837 20500 (0xffffff00024fc350:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff810251b8 held @ ../../../kern/subr_sleepqueue.c:264 20499 (0xffffff00024fc350:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff810251b8 held @ ../../../kern/subr_sleepqueue.c:232 20498 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:830 20497 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:822 20496 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:960 20495 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&zio->io_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c:951 20494 (0xffffff0002627350:cpu0): _mtx_unlock_spin_flags: mutex 0xffffffff81025960 held @ ../../../kern/subr_sleepqueue.c:264 20493 (0xffffff00024fc350:cpu1): XUNLOCK (sx) zfs:&vd->vdev_stat_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c:1637 20492 (0xffffff00024fc350:cpu1): XLOCK (sx) zfs:&vd->vdev_stat_lock r = 0 at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c:1634 20491 (0xffffff00024fc350:cpu1): _sx_xlock_hard: 0xffffff00024d2d80 resuming from sleep queue 20490 (0xffffff0002627350:cpu0): _mtx_unlock_spin_flags: mutex 0xffffffff81025960 held @ ../../../kern/kern_mutex.c:568 20489 (0xffffff0002627350:cpu0): _thread_lock_flags: mutex 0xffffffff81025960 held @ ../../../kern/subr_sleepqueue.c:779 20488 (0xffffff0002627350:cpu0): _sx_xunlock_hard: 0xffffff00024d2d80 waking up all threads on exclusive queue 20487 (0xffffff0002627350:cpu0): _mtx_lock_spin_flags: mutex 0xffffffff81025960 held @ ../../../kern/subr_sleepqueue.c:232 20486 (0xffffff0002627350:cpu0): _sx_xunlock_hard: 0xffffff00024d2d80 contested 20485 (0xffffff00024fc350:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff81025960 held @ ../../../kern/sched_ule.c:1771 20484 (0xffffff00024fc350:cpu1): _sx_xlock_hard: 0xffffff00024d2d80 blocking on sleep queue 20483 (0xffffff00024fc350:cpu1): _sx_xlock_hard: 0xffffff00024d2d80 set excl waiters flag db> panic Script started on Thu Jun 12 23:06:29 2008 You have mail. pegasus# gkgdb /boot/kernel/kernel /var/crash/ bounds info.1 info.3 vmcore.1 vmcore.3 info.0 info.2 vmcore.0 vmcore.2 pegasus# kgdb /boot/kernel/kernel /var/crash/vm vmcore.0 vmcore.1 vmcore.2 vmcore.3 pegasus# kgdb /boot/kernel/kernel /var/crash/vmcore.3 [GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so: Undefined symbol "ps_pglobal_lookup"] GNU gdb 6.1.1 [FreeBSD] Copyright 2004 Free Software Foundation, Inc. GDB is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain conditions. Type "show copying" to see the conditions. There is absolutely no warranty for GDB. Type "show warranty" for details. This GDB was configured as "amd64-marcel-freebsd". There is no member named pathname. Reading symbols from /boot/kernel/zfs.ko...Reading symbols from /boot/kernel/zfs.ko.symbols...done. done. Loaded symbols for /boot/kernel/zfs.ko Reading symbols from /boot/kernel/geom_mirror.ko...Reading symbols from /boot/kernel/geom_mirror.ko.symbols...done. done. Loaded symbols for /boot/kernel/geom_mirror.ko Reading symbols from /boot/kernel/speaker.ko...Reading symbols from /boot/kernel/speaker.ko.symbols...done. done. Loaded symbols for /boot/kernel/speaker.ko Reading symbols from /boot/kernel/wlan_xauth.ko...Reading symbols from /boot/kernel/wlan_xauth.ko.symbols...done. done. Loaded symbols for /boot/kernel/wlan_xauth.ko Reading symbols from /boot/kernel/if_bridge.ko...Reading symbols from /boot/kernel/if_bridge.ko.symbols...done. done. Loaded symbols for /boot/kernel/if_bridge.ko Reading symbols from /boot/kernel/bridgestp.ko...Reading symbols from /boot/kernel/bridgestp.ko.symbols...done. done. Loaded symbols for /boot/kernel/bridgestp.ko Reading symbols from /boot/kernel/pf.ko...Reading symbols from /boot/kernel/pf.ko.symbols...done. done. Loaded symbols for /boot/kernel/pf.ko Unread portion of the kernel message buffer: panic: mutex sleepq chain (0xffffffff81025618) not owned at ../../../kern/subr_sleepqueue.c:778 cpuid = 0 KDB: enter: panic panic: from debugger cpuid = 0 Uptime: 42s Physical memory: 2027 MB Dumping 298 MB: 283 267 251 235 219 203 187 171 155 139 123 107 91 75 59 43 27 11 #0 doadump () at pcpu.h:194 194 pcpu.h: No such file or directory. in pcpu.h (kgdb) bt #0 doadump () at pcpu.h:194 #1 0xffffffff80490013 in boot (howto=260) at ../../../kern/kern_shutdown.c:418 #2 0xffffffff80490467 in panic (fmt=Variable "fmt" is not available. ) at ../../../kern/kern_shutdown.c:572 #3 0xffffffff801c6377 in db_panic (addr=Variable "addr" is not available. ) at ../../../ddb/db_command.c:446 #4 0xffffffff801c6c6f in db_command (last_cmdp=0xffffffff80a5d948, cmd_table=0x0, dopager=1) at ../../../ddb/db_command.c:413 #5 0xffffffff801c6e80 in db_command_loop () at ../../../ddb/db_command.c:466 #6 0xffffffff801c87b9 in db_trap (type=Variable "type" is not available. ) at ../../../ddb/db_main.c:228 #7 0xffffffff804bbbc5 in kdb_trap (type=3, code=0, tf=0xffffffffbfa59920) at ../../../kern/subr_kdb.c:524 #8 0xffffffff8074ded3 in trap (frame=0xffffffffbfa59920) at ../../../amd64/amd64/trap.c:499 #9 0xffffffff8073369e in calltrap () at ../../../amd64/amd64/exception.S:169 #10 0xffffffff804bbd9d in kdb_enter_why (why=0xffffffff80810eff "panic", msg=0xa
) at cpufunc.h:63 #11 0xffffffff80490476 in panic (fmt=Variable "fmt" is not available. ) at ../../../kern/kern_shutdown.c:556 #12 0xffffffff80483fd2 in _mtx_assert (m=Variable "m" is not available. ) at ../../../kern/kern_mutex.c:653 #13 0xffffffff804c3b9a in sleepq_broadcast (wchan=0xffffff0002299de0, flags=Variable "flags" is not available. ) at ../../../kern/subr_sleepqueue.c:778 #14 0xffffffff811b42a9 in taskq_thread (arg=Variable "arg" is not available. ) at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:827 #15 0xffffffff80470aaa in fork_exit ( ---Type to continue, or q to quit---q callout=0xffffffff811b40b0 ,Quit ) at ../../../kern/kern_fork.c:783 #16 0xffffffff80733a6e in fork_trampoline () at ../../../amd64/amd64/exception.S:415 #17 0x0000000000000000 in ?? () #18 0x0000000000000000 in ?? () #19 0x0000000000000001 in ?? () #20 0x0000000000000000 in ?? () #21 0x0000000000000000 in ?? () #22 0x0000000000000000 in ?? () #23 0x0000000000000000 in ?? () #24 0x0000000000000000 in ?? () #25 0x0000000000000000 in ?? () #26 0x0000000000000000 in ?? () #27 0x0000000000000000 in ?? () #28 0x0000000000000000 in ?? () #29 0x0000000000000000 in ?? () #30 0x0000000000000000 in ?? () #31 0x0000000000000000 in ?? () #32 0x0000000000000000 in ?? () #33 0x0000000000000000 in ?? () #34 0x0000000000000000 in ?? () #35 0x0000000000000000 in ?? () #36 0x0000000000000000 in ?? () #37 0x0000000000000000 in ?? () ---Type to continue, or q to quit---q Quit (kgdb) up 134  #13 0xffffffff804c3b9a in sleepq_broadcast (wchan=0xffffff0002299de0, flags=Variable "flags" is not available. ) at ../../../kern/subr_sleepqueue.c:778 778 mtx_assert(&sc->sc_lock, MA_OWNED); (kgdb) bt #0 doadump () at pcpu.h:194 #1 0xffffffff80490013 in boot (howto=260) at ../../../kern/kern_shutdown.c:418 #2 0xffffffff80490467 in panic (fmt=Variable "fmt" is not available. ) at ../../../kern/kern_shutdown.c:572 #3 0xffffffff801c6377 in db_panic (addr=Variable "addr" is not available. ) at ../../../ddb/db_command.c:446 #4 0xffffffff801c6c6f in db_command (last_cmdp=0xffffffff80a5d948, cmd_table=0x0, dopager=1) at ../../../ddb/db_command.c:413 #5 0xffffffff801c6e80 in db_command_loop () at ../../../ddb/db_command.c:466 #6 0xffffffff801c87b9 in db_trap (type=Variable "type" is not available. ) at ../../../ddb/db_main.c:228 #7 0xffffffff804bbbc5 in kdb_trap (type=3, code=0, tf=0xffffffffbfa59920) at ../../../kern/subr_kdb.c:524 #8 0xffffffff8074ded3 in trap (frame=0xffffffffbfa59920) at ../../../amd64/amd64/trap.c:499 #9 0xffffffff8073369e in calltrap () at ../../../amd64/amd64/exception.S:169 #10 0xffffffff804bbd9d in kdb_enter_why (why=0xffffffff80810eff "panic", msg=0xa
) at cpufunc.h:63 #11 0xffffffff80490476 in panic (fmt=Variable "fmt" is not available. ) at ../../../kern/kern_shutdown.c:556 #12 0xffffffff80483fd2 in _mtx_assert (m=Variable "m" is not available. ) at ../../../kern/kern_mutex.c:653 #13 0xffffffff804c3b9a in sleepq_broadcast (wchan=0xffffff0002299de0, flags=Variable "flags" is not available. ) at ../../../kern/subr_sleepqueue.c:778 #14 0xffffffff811b42a9 in taskq_thread (arg=Variable "arg" is not available. ) at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:827 #15 0xffffffff80470aaa in fork_exit ( ---Type to continue, or q to quit---q callout=0xffffffff811b40b0 ,Quit ) at ../../../kern/kern_fork.c:783 #16 0xffffffff80733a6e in fork_trampoline () at ../../../amd64/amd64/exception.S:415 #17 0x0000000000000000 in ?? () #18 0x0000000000000000 in ?? () #19 0x0000000000000001 in ?? () #20 0x0000000000000000 in ?? () #21 0x0000000000000000 in ?? () #22 0x0000000000000000 in ?? () #23 0x0000000000000000 in ?? () #24 0x0000000000000000 in ?? () #25 0x0000000000000000 in ?? () #26 0x0000000000000000 in ?? () #27 0x0000000000000000 in ?? () #28 0x0000000000000000 in ?? () #29 0x0000000000000000 in ?? () #30 0x0000000000000000 in ?? () #31 0x0000000000000000 in ?? () #32 0x0000000000000000 in ?? () #33 0x0000000000000000 in ?? () #34 0x0000000000000000 in ?? () #35 0x0000000000000000 in ?? () #36 0x0000000000000000 in ?? () #37 0x0000000000000000 in ?? () ---Type to continue, or q to quit---q Quit (kgdb) up 1 #14 0xffffffff811b42a9 in taskq_thread (arg=Variable "arg" is not available. ) at /data/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/os/taskq.c:827 827 mutex_exit(&tq->tq_lock); (kgdb) l 822 tq->tq_active++; 823 continue; 824 } 825 tqe->tqent_prev->tqent_next = tqe->tqent_next; 826 tqe->tqent_next->tqent_prev = tqe->tqent_prev; 827 mutex_exit(&tq->tq_lock); 828 829 rw_enter(&tq->tq_threadlock, RW_READER); 830 start = gethrtime(); 831 DTRACE_PROBE2(taskq__exec__start, taskq_t *, tq, (kgdb) quit pegasus# exit exit Script done on Thu Jun 12 23:07:25 2008