Dump header from device /dev/ad6s1d Architecture: amd64 Architecture Version: 2 Dump Length: 316932096B (302 MB) Blocksize: 512 Dumptime: Thu Jun 26 21:38:11 2008 Hostname: pegasus.clearchain.com Magic: FreeBSD Kernel Dump Version String: FreeBSD 7.0-STABLE #30: Thu Jun 26 20:49:50 CST 2008 root@pegasus.clearchain.com:/data/usr/src/sys/amd64/compile/PEGASUS Panic String: mutex sleepq chain (0xffffffff810255a8) not owned at ../../../kern/subr_sleepqueue.c:778 Dump Parity: 4269915731 Bounds: 5 Dump Status: good db> bt Tracing pid 129 tid 100056 td 0xffffff0002647350 kdb_enter_why() at kdb_enter_why+0x3d panic() at panic+0x176 _mtx_assert() at _mtx_assert+0xe2 sleepq_broadcast() at sleepq_broadcast+0xaa spa_scrub_io_done() at spa_scrub_io_done+0x109 zio_done() at zio_done+0x1eb zio_next_stage() at zio_next_stage+0x9c zio_read_decompress() at zio_read_decompress+0x6b zio_next_stage() at zio_next_stage+0x9c zio_wait_for_children() at zio_wait_for_children+0x66 zio_wait_children_done() at zio_wait_children_done+0x15 zio_next_stage() at zio_next_stage+0x9c zio_vdev_io_assess() at zio_vdev_io_assess+0x17f zio_next_stage() at zio_next_stage+0x9c vdev_mirror_io_done() at vdev_mirror_io_done+0x2bd zio_vdev_io_done() at zio_vdev_io_done+0x14 taskq_thread() at taskq_thread+0x23c fork_exit() at fork_exit+0x12a fork_trampoline() at fork_trampoline+0xe --- trap 0, rip = 0, rsp = 0xffffffffbfa31d30, rbp = 0 --- db> show lock 0xffffffff810255a8 class: spin mutex name: sleepq chain flags: {SPIN, RECURSE} state: {UNOWNED} db> show allpcpu Current CPU: 0 cpuid = 0 curthread = 0xffffff0002647350: pid 129 "spa_zio_intr_1" curpcb = 0xffffffffbfa31d40 fpcurthread = none idlethread = 0xffffff0002170350: pid 12 "idle: cpu0" cpuid = 1 curthread = 0xffffff00021706a0: pid 11 "idle: cpu1" curpcb = 0xffffffffbd476d40 fpcurthread = none idlethread = 0xffffff00021706a0: pid 11 "idle: cpu1" db> show thread 0xffffff0002647350 Thread 100056 at 0xffffff0002647350: proc (pid 129): 0xffffff0002643468 name: spa_zio_intr_1 flags: 0x4 pflags: 0 state: RUNNING (CPU 0) priority: 96 container lock: sched lock 0 (0xffffffff810235c0) db> show lock 0xffffffff810235c0 class: spin mutex name: sched lock 0 type: sched lock flags: {SPIN, RECURSE} state: {UNOWNED} db> show ktr 14716 (0xffffff0002844350:cpu1): UNLOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/subr_sleepqueue.c:341 14715 (0xffffff0002844350:cpu1): LOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/subr_sleepqueue.c:331 14714 (0xffffff0002844350:cpu1): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14713 (0xffffff0002844350:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff810255a8 held @ ../../../kern/subr_sleepqueue.c:232 14712 (0xffffff0002844350:cpu1): UNLOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/subr_sleepqueue.c:565 14711 (0xffffff00021706a0:cpu1): LOCK (spin mutex) sched lock 1 r = 0 at ../../../amd64/amd64/mp_machdep.c:966 14710 (0xffffff0002647350:cpu0): UNLOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/subr_sleepqueue.c:783 14709 (0xffffff0002647350:cpu0): LOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/sched_ule.c:978 14708 (0xffffff0002647350:cpu0): UNLOCK (spin mutex) sleepq chain r = 1 at ../../../kern/kern_mutex.c:568 14707 (0xffffff0002647350:cpu0): _mtx_unlock_spin_flags: mutex 0xffffffff81025570 held @ ../../../kern/kern_mutex.c:568 14706 (0xffffff0002647350:cpu0): LOCK (spin mutex) sleepq chain r = 1 at ../../../kern/subr_sleepqueue.c:779 14705 (0xffffff0002647350:cpu0): _thread_lock_flags: mutex 0xffffffff81025570 held @ ../../../kern/subr_sleepqueue.c:779 14704 (0xffffff0002647350:cpu0): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14703 (0xffffff0002647350:cpu0): _mtx_lock_spin_flags: mutex 0xffffffff81025570 held @ ../../../kern/subr_sleepqueue.c:232 14702 (0xffffff00021706a0:cpu1): UNLOCK (spin mutex) sched lock 1 r = 0 at ../../../amd64/amd64/mp_machdep.c:971 14701 (0xffffff0002844350:cpu1): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/sched_ule.c:1771 14700 (0xffffff0002844350:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff81025570 held @ ../../../kern/sched_ule.c:1771 14699 (0xffffff0002844350:cpu1): LOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/sched_ule.c:1871 14698 (0xffffff0002844350:cpu1): UNLOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/kern_mutex.c:591 14697 (0xffffff0002844350:cpu1): LOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/subr_sleepqueue.c:563 14696 (0xffffff0002844350:cpu1): UNLOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/subr_sleepqueue.c:341 14695 (0xffffff0002844350:cpu1): LOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/subr_sleepqueue.c:331 14694 (0xffffff0002844350:cpu1): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14693 (0xffffff0002844350:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff81025570 held @ ../../../kern/subr_sleepqueue.c:232 14692 (0xffffff0002844350:cpu1): UNLOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/subr_sleepqueue.c:565 14691 (0xffffff0002647350:cpu0): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:264 14690 (0xffffff00021706a0:cpu1): LOCK (spin mutex) sched lock 1 r = 0 at ../../../amd64/amd64/mp_machdep.c:966 14689 (0xffffff0002647350:cpu0): _mtx_unlock_spin_flags: mutex 0xffffffff810255a8 held @ ../../../kern/subr_sleepqueue.c:264 14688 (0xffffff0002647350:cpu0): UNLOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/subr_sleepqueue.c:783 14687 (0xffffff0002647350:cpu0): LOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/sched_ule.c:978 14686 (0xffffff0002647350:cpu0): UNLOCK (spin mutex) sleepq chain r = 1 at ../../../kern/kern_mutex.c:568 14685 (0xffffff0002647350:cpu0): _mtx_unlock_spin_flags: mutex 0xffffffff810255a8 held @ ../../../kern/kern_mutex.c:568 14684 (0xffffff0002647350:cpu0): LOCK (spin mutex) sleepq chain r = 1 at ../../../kern/subr_sleepqueue.c:779 14683 (0xffffff0002647350:cpu0): _thread_lock_flags: mutex 0xffffffff810255a8 held @ ../../../kern/subr_sleepqueue.c:779 14682 (0xffffff0002647350:cpu0): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14681 (0xffffff0002647350:cpu0): _mtx_lock_spin_flags: mutex 0xffffffff810255a8 held @ ../../../kern/subr_sleepqueue.c:232 14680 (0xffffff0002647350:cpu0): UNLOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2262 14679 (0xffffff0002647350:cpu0): LOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2257 14678 (0xffffff0002647350:cpu0): UNLOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2262 14677 (0xffffff0002647350:cpu0): LOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2257 14676 (0xffffff0002647350:cpu0): UNLOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2262 14675 (0xffffff0002647350:cpu0): LOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2257 14674 (0xffffff00021706a0:cpu1): UNLOCK (spin mutex) sched lock 1 r = 0 at ../../../amd64/amd64/mp_machdep.c:971 14673 (0xffffff0002844350:cpu1): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/sched_ule.c:1771 14672 (0xffffff0002844350:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff810255a8 held @ ../../../kern/sched_ule.c:1771 14671 (0xffffff0002844350:cpu1): LOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/sched_ule.c:1871 14670 (0xffffff0002844350:cpu1): UNLOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/kern_mutex.c:591 14669 (0xffffff0002844350:cpu1): LOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/subr_sleepqueue.c:563 14668 (0xffffff0002844350:cpu1): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14667 (0xffffff0002844350:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff810255a8 held @ ../../../kern/subr_sleepqueue.c:232 14666 (0xffffff0002844350:cpu1): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:264 14665 (0xffffff0002844350:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff810255a8 held @ ../../../kern/subr_sleepqueue.c:264 14664 (0xffffff0002844350:cpu1): UNLOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/subr_sleepqueue.c:341 14663 (0xffffff0002844350:cpu1): LOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/subr_sleepqueue.c:331 14662 (0xffffff0002844350:cpu1): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14661 (0xffffff0002844350:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff810255a8 held @ ../../../kern/subr_sleepqueue.c:232 14660 (0xffffff0002844350:cpu1): UNLOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/subr_sleepqueue.c:565 14659 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/sched_ule.c:1771 14658 (0xffffff00026476a0:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff810257a0 held @ ../../../kern/sched_ule.c:1771 14657 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/sched_ule.c:1871 14656 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/kern_mutex.c:591 14655 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/subr_sleepqueue.c:563 14654 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14653 (0xffffff00026476a0:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff810257a0 held @ ../../../kern/subr_sleepqueue.c:232 14652 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:264 14651 (0xffffff00026476a0:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff810257a0 held @ ../../../kern/subr_sleepqueue.c:264 14650 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/subr_sleepqueue.c:341 14649 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/subr_sleepqueue.c:331 14648 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14647 (0xffffff00026476a0:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff810257a0 held @ ../../../kern/subr_sleepqueue.c:232 14646 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) zio_cache r = 0 at ../../../vm/uma_core.c:2262 14645 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) zio_cache r = 0 at ../../../vm/uma_core.c:2257 14644 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:264 14643 (0xffffff00026476a0:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff81025ea0 held @ ../../../kern/subr_sleepqueue.c:264 14642 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14641 (0xffffff00026476a0:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff81025ea0 held @ ../../../kern/subr_sleepqueue.c:232 14640 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:264 14639 (0xffffff00026476a0:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff810255a8 held @ ../../../kern/subr_sleepqueue.c:264 14638 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14637 (0xffffff00026476a0:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff810255a8 held @ ../../../kern/subr_sleepqueue.c:232 14636 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2262 14635 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2257 14634 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2262 14633 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2257 14632 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2262 14631 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2257 14630 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2262 14629 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2257 14628 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 128 r = 0 at ../../../vm/uma_core.c:2262 14627 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 128 r = 0 at ../../../vm/uma_core.c:2257 14626 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) zio_cache r = 0 at ../../../vm/uma_core.c:2262 14625 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) zio_cache r = 0 at ../../../vm/uma_core.c:2257 14624 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:264 14623 (0xffffff00026476a0:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff810265a0 held @ ../../../kern/subr_sleepqueue.c:264 14622 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14621 (0xffffff00026476a0:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff810265a0 held @ ../../../kern/subr_sleepqueue.c:232 14620 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:264 14619 (0xffffff00026476a0:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff810257a0 held @ ../../../kern/subr_sleepqueue.c:264 14618 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14617 (0xffffff00026476a0:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff810257a0 held @ ../../../kern/subr_sleepqueue.c:232 14616 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2262 14615 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2257 14614 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2262 14613 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2257 14612 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 64 r = 0 at ../../../vm/uma_core.c:2262 14611 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 64 r = 0 at ../../../vm/uma_core.c:2257 14610 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) zio_cache r = 0 at ../../../vm/uma_core.c:2262 14609 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) zio_cache r = 0 at ../../../vm/uma_core.c:2257 14608 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:264 14607 (0xffffff00026476a0:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff81026178 held @ ../../../kern/subr_sleepqueue.c:264 14606 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14605 (0xffffff00026476a0:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff81026178 held @ ../../../kern/subr_sleepqueue.c:232 14604 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:264 14603 (0xffffff00026476a0:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff810255a8 held @ ../../../kern/subr_sleepqueue.c:264 14602 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14601 (0xffffff00026476a0:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff810255a8 held @ ../../../kern/subr_sleepqueue.c:232 14600 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2262 14599 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2257 14598 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2262 14597 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2257 14596 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2262 14595 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2257 14594 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2262 14593 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2257 14592 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 128 r = 0 at ../../../vm/uma_core.c:2262 14591 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 128 r = 0 at ../../../vm/uma_core.c:2257 14590 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) zio_cache r = 0 at ../../../vm/uma_core.c:2262 14589 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) zio_cache r = 0 at ../../../vm/uma_core.c:2257 14588 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:264 14587 (0xffffff00026476a0:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff81026610 held @ ../../../kern/subr_sleepqueue.c:264 14586 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14585 (0xffffff00026476a0:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff81026610 held @ ../../../kern/subr_sleepqueue.c:232 14584 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:264 14583 (0xffffff00026476a0:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff810255a8 held @ ../../../kern/subr_sleepqueue.c:264 14582 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/subr_sleepqueue.c:783 14581 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/sched_ule.c:978 14580 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sleepq chain r = 1 at ../../../kern/kern_mutex.c:568 14579 (0xffffff00026476a0:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff810255a8 held @ ../../../kern/kern_mutex.c:568 14578 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sleepq chain r = 1 at ../../../kern/subr_sleepqueue.c:779 14577 (0xffffff00026476a0:cpu1): _thread_lock_flags: mutex 0xffffffff810255a8 held @ ../../../kern/subr_sleepqueue.c:779 14576 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14575 (0xffffff00026476a0:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff810255a8 held @ ../../../kern/subr_sleepqueue.c:232 14574 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2262 14573 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2257 14572 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2262 14571 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2257 14570 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2262 14569 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2257 14568 (0xffffff0002647350:cpu0): UNLOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2262 14567 (0xffffff0002647350:cpu0): LOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2257 14566 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2262 14565 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2257 14564 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 128 r = 0 at ../../../vm/uma_core.c:2262 14563 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 128 r = 0 at ../../../vm/uma_core.c:2257 14562 (0xffffff0002647350:cpu0): UNLOCK (sleep mutex) 128 r = 0 at ../../../vm/uma_core.c:2262 14561 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) zio_cache r = 0 at ../../../vm/uma_core.c:2262 14560 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) zio_cache r = 0 at ../../../vm/uma_core.c:2257 14559 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:264 14558 (0xffffff00026476a0:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff810258b8 held @ ../../../kern/subr_sleepqueue.c:264 14557 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14556 (0xffffff00026476a0:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff810258b8 held @ ../../../kern/subr_sleepqueue.c:232 14555 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:264 14554 (0xffffff00026476a0:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff810257a0 held @ ../../../kern/subr_sleepqueue.c:264 14553 (0xffffff0002647350:cpu0): LOCK (sleep mutex) 128 r = 0 at ../../../vm/uma_core.c:2257 14552 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14551 (0xffffff00026476a0:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff810257a0 held @ ../../../kern/subr_sleepqueue.c:232 14550 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2262 14549 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2257 14548 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2262 14547 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2257 14546 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) zio_cache r = 0 at ../../../vm/uma_core.c:2262 14545 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) zio_cache r = 0 at ../../../vm/uma_core.c:2257 14544 (0xffffff0002647350:cpu0): UNLOCK (sleep mutex) zio_cache r = 0 at ../../../vm/uma_core.c:2262 14543 (0xffffff00026476a0:cpu1): _mtx_lock_sleep: spinning on 0xffffff007feb77f0 held by 0xffffff0002647350 14542 (0xffffff00026476a0:cpu1): _mtx_lock_sleep: zio_cache contested (lock=0xffffff0002647350) at ../../../vm/uma_core.c:2257 14541 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:264 14540 (0xffffff00026476a0:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff81025538 held @ ../../../kern/subr_sleepqueue.c:264 14539 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14538 (0xffffff00026476a0:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff81025538 held @ ../../../kern/subr_sleepqueue.c:232 14537 (0xffffff0002647350:cpu0): LOCK (sleep mutex) zio_cache r = 0 at ../../../vm/uma_core.c:2257 14536 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:264 14535 (0xffffff00026476a0:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff810257a0 held @ ../../../kern/subr_sleepqueue.c:264 14534 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14533 (0xffffff00026476a0:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff810257a0 held @ ../../../kern/subr_sleepqueue.c:232 14532 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2262 14531 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2257 14530 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2262 14529 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2257 14528 (0xffffff0002647350:cpu0): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:264 14527 (0xffffff0002647350:cpu0): _mtx_unlock_spin_flags: mutex 0xffffffff81025b90 held @ ../../../kern/subr_sleepqueue.c:264 14526 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 64 r = 0 at ../../../vm/uma_core.c:2262 14525 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 64 r = 0 at ../../../vm/uma_core.c:2257 14524 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) zio_cache r = 0 at ../../../vm/uma_core.c:2262 14523 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) zio_cache r = 0 at ../../../vm/uma_core.c:2257 14522 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:264 14521 (0xffffff00026476a0:cpu1): _mtx_unlock_spin_flags: mutex 0xffffffff81026878 held @ ../../../kern/subr_sleepqueue.c:264 14520 (0xffffff0002647350:cpu0): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14519 (0xffffff00026476a0:cpu1): LOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:232 14518 (0xffffff0002647350:cpu0): _mtx_lock_spin_flags: mutex 0xffffffff81025b90 held @ ../../../kern/subr_sleepqueue.c:232 14517 (0xffffff00026476a0:cpu1): _mtx_lock_spin_flags: mutex 0xffffffff81026878 held @ ../../../kern/subr_sleepqueue.c:232 14516 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2262 14515 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 4096 r = 0 at ../../../vm/uma_core.c:2257 14514 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2262 14513 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 32 r = 0 at ../../../vm/uma_core.c:2257 14512 (0xffffff00026476a0:cpu1): UNLOCK (sleep mutex) 64 r = 0 at ../../../vm/uma_core.c:2262 14511 (0xffffff0002647350:cpu0): UNLOCK (spin mutex) sleepq chain r = 0 at ../../../kern/subr_sleepqueue.c:264 14510 (0xffffff00026476a0:cpu1): LOCK (sleep mutex) 64 r = 0 at ../../../vm/uma_core.c:2257 14509 (0xffffff0002647350:cpu0): _mtx_unlock_spin_flags: mutex 0xffffffff810257a0 held @ ../../../kern/subr_sleepqueue.c:264 14508 (0xffffff00026476a0:cpu1): UNLOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/subr_sleepqueue.c:565 14507 (0xffffff00021706a0:cpu1): LOCK (spin mutex) sched lock 1 r = 0 at ../../../amd64/amd64/mp_machdep.c:966 14506 (0xffffff0002647350:cpu0): UNLOCK (spin mutex) sched lock 1 r = 0 at ../../../kern/subr_sleepqueue.c:783 db> panic