path: root/tests/bcachefs/replication.ktest

#!/usr/bin/env bash

. $(dirname $(readlink -e "${BASH_SOURCE[0]}"))/bcachefs-test-libs.sh

require-kernel-config MD
require-kernel-config BLK_DEV_DM
require-kernel-config DM_FLAKEY

config-scratch-devs 4G
config-scratch-devs 4G
config-scratch-devs 4G
config-scratch-devs 4G
config-scratch-devs 4G

config-mem 4G

test_recovery()
{
    set_watchdog 180

    run_quiet "" bcachefs format -f		\
	--discard				\
	--bucket=64k				\
	--block_size=4k				\
	${ktest_scratch_dev[0]}			\
	${ktest_scratch_dev[1]}			\
	${ktest_scratch_dev[2]}

    for i in `seq 0 20`; do
	mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]}:${ktest_scratch_dev[2]} /mnt

	#run_fio_randrw
	fio --eta=always			\
	    --exitall_on_error=1		\
	    --randrepeat=0			\
	    --ioengine=libaio			\
	    --iodepth=64			\
	    --iodepth_batch=16			\
	    --direct=1				\
	    --numjobs=1				\
	    --filename=/mnt/fiotest		\
	    --filesize=3500M			\
	    --name=randwrite			\
	    --rw=randwrite			\
	    --bsrange=4k-8k			\
	    --runtime=2

	$ktest_dir/tests/xfstests/src/godown -f /mnt

	umount /mnt
    done

    bcachefs fsck -ny ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]} ${ktest_scratch_dev[2]}
    check_counters ${ktest_scratch_dev[0]}
}

test_twodevices()
{
    run_basic_fio_test				\
	${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]} ${ktest_scratch_dev[2]}
}

test_largebuckets()
{
    set_watchdog 30
    run_quiet "" bcachefs format -f --bucket=8M	\
	${ktest_scratch_dev[0]}			\
	${ktest_scratch_dev[1]}			\
	${ktest_scratch_dev[2]}

    mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]}:${ktest_scratch_dev[2]} /mnt
    dd if=/dev/zero of=/mnt/foo bs=8M count=1024 oflag=direct
    umount /mnt

    bcachefs fsck -ny ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]} ${ktest_scratch_dev[2]}
    check_counters ${ktest_scratch_dev[0]}
}

test_replicas()
{
    run_basic_fio_test				\
	--replicas=2				\
	${ktest_scratch_dev[0]}			\
	${ktest_scratch_dev[1]}			\
	${ktest_scratch_dev[2]}
}

test_replicas_variable_buckets()
{
    set_watchdog 1200
    run_quiet "" bcachefs format -f			\
	--replicas=2					\
	--bucket=32k	${ktest_scratch_dev[0]}		\
	--bucket=64k	${ktest_scratch_dev[1]}		\
	--bucket=128k	${ktest_scratch_dev[2]}		\
	--bucket=128k	${ktest_scratch_dev[3]}		\
	--bucket=128k	${ktest_scratch_dev[4]}

    mount -t bcachefs "$(join_by : "${ktest_scratch_dev[@]}")" /mnt
    fs_mark -v -n 20000 -s 4096 -k -S 0 -D 1000 -N 100000 -d /mnt -t 60
    df -h /mnt
    umount /mnt

    bcachefs fsck -ny ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]} ${ktest_scratch_dev[2]} ${ktest_scratch_dev[3]} ${ktest_scratch_dev[4]}
    check_counters ${ktest_scratch_dev[0]}
}

d_test_copygc_torture()
{
    loops=$((($ktest_priority + 1) * 2))

    bcachefs_antagonist
    set_watchdog 2400

    run_quiet "" bcachefs format -f "$@"	\
	--encrypted --no_passphrase		\
	${ktest_scratch_dev[0]}			\
	${ktest_scratch_dev[1]}			\
	${ktest_scratch_dev[2]}

    mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]}:${ktest_scratch_dev[2]} /mnt
    #enable_memory_faults

    fio						\
	--filename=/mnt/fiotest			\
	--ioengine=sync				\
	--name=write				\
	--rw=write				\
	--bs=16M				\
	--fill_fs=1
    echo 3 > /proc/sys/vm/drop_caches

    for i in $(seq 1 $loops); do
	run_fio_base				\
	    --verify=meta			\
	    --buffer_compress_percentage=50	\
						\
	    --name=randwrite			\
	    --stonewall				\
	    --rw=randwrite			\
	    --loops=$loops			\
	    --bsrange=4k-4k

	cur_size=$(stat -c '%s' /mnt/fiotest)
	new_size=$(($cur_size - 1048576))
	echo "truncating to $new_size"

	truncate --size=$new_size /mnt/fiotest
    done

    rm /mnt/fiotest

    expect_sysfs cache dirty_buckets	0
    expect_sysfs cache dirty_data	0
    expect_sysfs cache cached_buckets	0
    expect_sysfs cache cached_data	0

    #disable_memory_faults
    umount /mnt

    bcachefs fsck -ny ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]} ${ktest_scratch_dev[2]}
    check_counters ${ktest_scratch_dev[0]}
}

do_remove_test()
{
    set_watchdog 300
    offline=$1
    evacuate=$2

    setup_tracing 'bcachefs:*'

    run_quiet "" bcachefs format -f		\
	--replicas=2				\
	${ktest_scratch_dev[0]}			\
	${ktest_scratch_dev[1]}			\
	${ktest_scratch_dev[2]}

    mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]}:${ktest_scratch_dev[2]} /mnt

    local fioout="$ktest_out/fio-out"
    run_fio_randrw >"$fioout" 2>&1 &
    local fiopid=$!

    sleep 1

    if [[ $offline = 1 ]]; then
	echo -n "offlining ${ktest_scratch_dev[0]}... "
	bcachefs device offline --force	${ktest_scratch_dev[0]}
	echo "done"
    fi

    if [[ $evacuate = 1 ]]; then
	echo -n "evacuating ${ktest_scratch_dev[0]}... "
	bcachefs device evacuate	${ktest_scratch_dev[0]}
	echo "done"

	echo -n "evacuating ${ktest_scratch_dev[0]}... "
	bcachefs device evacuate	${ktest_scratch_dev[0]}
	echo "done"
    fi

    bcachefs fs usage -h /mnt

    echo -n "removing ${ktest_scratch_dev[0]}... "
    if [[ $evacuate = 1 ]]; then
	#bcachefs device remove		${ktest_scratch_dev[0]}
	bcachefs device remove		0 /mnt
    else
	#bcachefs device remove --force	${ktest_scratch_dev[0]}
	bcachefs device remove --force	0 /mnt
    fi
    echo "done"

    if ! wait $fiopid; then
	cat "$fioout"
	return 1
    fi

    bcachefs fs usage -h /mnt
    umount /mnt

    mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]}:${ktest_scratch_dev[2]} /mnt
    umount /mnt

    bcachefs fsck -ny ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]} ${ktest_scratch_dev[2]}
    check_counters ${ktest_scratch_dev[0]}
}

test_device_remove_offline()
{
    do_remove_test 1 0
}

test_device_remove_online()
{
    do_remove_test 0 0
}

test_device_evacuate_offline()
{
    do_remove_test 1 1
}

test_device_evacuate_online()
{
    do_remove_test 0 1
}

test_device_add()
{
    set_watchdog 240
    #setup_tracing 'bcachefs:*'

    run_quiet "" bcachefs format -f		\
	--block_size=4k				\
	--no_initialize				\
	${ktest_scratch_dev[0]}

    mount -t bcachefs ${ktest_scratch_dev[0]} /mnt

    run_fio_randrw >"$ktest_out/fio-out" 2>&1 &
    local fiopid=$!

    sleep 1

    echo -n "adding ${ktest_scratch_dev[1]}... "
    bcachefs device add -f /mnt ${ktest_scratch_dev[1]}
    echo "done"

    if ! wait $fiopid; then
	cat "$fioout"
	return 1
    fi

    umount /mnt
    bcachefs fsck -n ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]}

    mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]} /mnt
    umount /mnt

    bcachefs fsck -ny ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]}
    check_counters ${ktest_scratch_dev[0]}
}

test_device_add_label()
{
    # Test setting device label when adding
    # Caught bugs:
    #   - label was not set correctly; set to dev-0 label instead of new one.

    bcachefs format -f -L test_add_label --label=foo ${ktest_scratch_dev[0]}
    mount -t bcachefs ${ktest_scratch_dev[0]} /mnt
    bcachefs device add -f --label=bar /mnt ${ktest_scratch_dev[1]}
    expect_sysfs dev-1 label bar
    umount /mnt

    bcachefs fsck -ny ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]}
    check_counters ${ktest_scratch_dev[0]}
}

test_device_add_label_compound()
{
    # Test adding a device with a compond label, a '.' separated "path"
    # (e.g. ssd.sata)
    # Caught bugs:
    #   - filesystem crashed with message "Invalid superblock section disk_groups".

    set_watchdog 30

    bcachefs format -f -L test_add_compound_label --label=hdd ${ktest_scratch_dev[0]}
    mount -t bcachefs ${ktest_scratch_dev[0]} /mnt
    bcachefs device add -f --label=foo.bar /mnt ${ktest_scratch_dev[1]}
    expect_sysfs dev-1 label foo.bar
    umount /mnt

    bcachefs fsck -ny ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]}
    check_counters ${ktest_scratch_dev[0]}
}

test_device_set_state()
{
    set_watchdog 120

    run_quiet "" bcachefs format -f		\
	--replicas=2				\
	${ktest_scratch_dev[0]}			\
	${ktest_scratch_dev[1]}			\
	${ktest_scratch_dev[2]}

    mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]}:${ktest_scratch_dev[2]} /mnt

    run_fio_randrw >"$ktest_out/fio-out" 2>&1 &
    local fiopid=$!

    sleep 1

    for state in ro failed spare; do
	echo "setting state $state"
	bcachefs device set-state --force $state    ${ktest_scratch_dev[1]}

	echo "setting state rw"
	#bcachefs device set-state	  rw ${ktest_scratch_dev[1]}
	bcachefs device set-state	  rw 1 /mnt
    done

    if ! wait $fiopid; then
	cat "$fioout"
	return 1
    fi

    umount /mnt

    bcachefs fsck -ny ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]} ${ktest_scratch_dev[2]}
    check_counters ${ktest_scratch_dev[0]}
}

test_device_ro()
{
    set_watchdog 120

    run_quiet "" bcachefs format -f		\
	--btree_node_size=128k			\
	${ktest_scratch_dev[0]}			\
	${ktest_scratch_dev[1]}			\
	${ktest_scratch_dev[2]}

    mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]}:${ktest_scratch_dev[2]} /mnt

    local fioout="$ktest_out/fio-out"
    run_fio_randrw >"$fioout" 2>&1 &
    local fiopid=$!

    sleep 4

    bcachefs device set-state ro ${ktest_scratch_dev[1]}

    if ! wait $fiopid; then
	cat "$fioout"
	return 1
    fi

    umount /mnt

    mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]}:${ktest_scratch_dev[2]} /mnt
    umount /mnt

    bcachefs fsck -ny ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]} ${ktest_scratch_dev[2]}
    check_counters ${ktest_scratch_dev[0]}
}

test_device_set_state_offline()
{
    set_watchdog 120
    run_quiet "" bcachefs format -f		\
	--replicas=2				\
	${ktest_scratch_dev[0]}			\
	${ktest_scratch_dev[1]}			\
	${ktest_scratch_dev[2]}

    mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]}:${ktest_scratch_dev[2]} /mnt

    local fioout="$ktest_out/fio-out"
    run_fio_randrw >"$fioout" 2>&1 &
    local fiopid=$!

    sleep 1

    bcachefs device set-state --force	    ro ${ktest_scratch_dev[1]}

    if ! wait $fiopid; then
	cat "$fioout"
	return 1
    fi

    umount /mnt

    bcachefs device set-state --offline	    rw ${ktest_scratch_dev[1]}

    mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]}:${ktest_scratch_dev[2]} /mnt
    cat /sys/block/[sv]dc/bcachefs/state
    umount /mnt

    bcachefs fsck -ny ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]} ${ktest_scratch_dev[2]}
    check_counters ${ktest_scratch_dev[0]}
}

test_device_readd()
{
    set_watchdog 120
    run_quiet "" bcachefs format -f		\
	--replicas=2				\
	${ktest_scratch_dev[0]}			\
	${ktest_scratch_dev[1]}	

    mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]} /mnt

    local fioout="$ktest_out/fio-out"
    run_fio_randrw >"$fioout" 2>&1 &
    local fiopid=$!

    sleep 1

    echo -n "offlining ${ktest_scratch_dev[0]}... "
    bcachefs device offline --force	${ktest_scratch_dev[0]}
    echo "done"

    echo -n "onlining ${ktest_scratch_dev[0]}... "
    bcachefs device online ${ktest_scratch_dev[0]}
    echo "done"

    if ! wait $fiopid; then
	cat "$fioout"
	return 1
    fi

    umount /mnt

    mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]} /mnt
    umount /mnt

    bcachefs fsck -ny ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]}
    check_counters ${ktest_scratch_dev[0]}
}

test_device_repeated_add_remove()
{
    ## Test rapidly adding and removing the same device.
    # Caught bugs:
    #   - found a race condition that leads to a kernel BUG in btree_key_cache.

    set_watchdog 120

    bcachefs format -f -L test_device_repeated_add_remove --label=hdd ${ktest_scratch_dev[0]}
    mount -t bcachefs ${ktest_scratch_dev[0]} /mnt

    for ii in {1..10}; do
        echo "add-remove run #$ii ----------------------------------------------------"
        echo "bcachefs device add"
        bcachefs device add -f /mnt ${ktest_scratch_dev[1]}
        echo "bcachefs device evacuate"
        bcachefs device evacuate ${ktest_scratch_dev[1]}
        echo "bcachefs device remove"
        bcachefs device remove ${ktest_scratch_dev[1]}
    done

    umount /mnt

    bcachefs fsck -ny ${ktest_scratch_dev[0]}
    check_counters ${ktest_scratch_dev[0]}
}

do_replicas_errors_test()
{
    set_watchdog 180
    error_type=$1

    local p=/sys/module/bcachefs/parameters/verify_all_btree_replicas
    [[ -f $p ]] && echo 1 > $p

    local BLK_DEV_SIZE=`blockdev --getsz ${ktest_scratch_dev[1]}`
    local FLAKEY_TABLE="0 $BLK_DEV_SIZE flakey ${ktest_scratch_dev[1]} 0 180 0"
    local FLAKEY_TABLE_ERROR="0 $BLK_DEV_SIZE flakey ${ktest_scratch_dev[1]} 0 0 180 1 $error_type"

    dmsetup create flakey --table "$FLAKEY_TABLE"

    run_quiet "" bcachefs format -f		\
	--replicas=2				\
	${ktest_scratch_dev[0]} /dev/mapper/flakey

    mount -t bcachefs -o degraded ${ktest_scratch_dev[0]}:/dev/mapper/flakey /mnt

    echo $FLAKEY_TABLE_ERROR|dmsetup load flakey

    run_fio_randrw

    # Check the read retry path for indirect extents:
    cp --reflink /mnt/fiotest /mnt/fiotest.reflinked
    dd if=/mnt/fiotest of=/dev/null bs=4k iflag=direct

    umount /mnt
    dmsetup remove flakey

    bcachefs fsck -ny ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]}
    check_counters ${ktest_scratch_dev[0]}

    [[ -f $p ]] && echo 0 > $p
    true
}

test_replicas_write_errors()
{
    do_replicas_errors_test error_writes
}

test_replicas_read_errors()
{
    do_replicas_errors_test error_reads
}

test_cmd_fs_usage()
{
    set_watchdog 240

    run_quiet "" bcachefs format -f		\
	--replicas=2				\
	--bucket=128k				\
	${ktest_scratch_dev[0]}			\
	${ktest_scratch_dev[1]}			\
	${ktest_scratch_dev[2]}

    echo -n "mounting... "
    mount -t bcachefs -o verbose ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]}:${ktest_scratch_dev[2]} /mnt
    echo done

    echo "Before writing:"
    bcachefs fs usage  /mnt
    echo
    df -h /mnt
    echo

    run_fio_randrw --size=500M > /dev/null

    echo
    echo "After writing:"
    bcachefs fs usage -h /mnt
    echo
    df -h /mnt

    umount /mnt

    bcachefs fsck -ny ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]} ${ktest_scratch_dev[2]}
    check_counters ${ktest_scratch_dev[0]}
}

test_rereplicate()
{
    set_watchdog 60
    run_quiet "" bcachefs format		\
	${ktest_scratch_dev[0]}			\
	${ktest_scratch_dev[1]}			\
	${ktest_scratch_dev[2]}
    mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]}:${ktest_scratch_dev[2]} /mnt

    dd if=/dev/zero of=/mnt/test bs=1M count=1024 oflag=direct

    umount /mnt
    mount -t bcachefs -o metadata_replicas=2,data_replicas=2 ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]}:${ktest_scratch_dev[2]} /mnt

    #echo 2 > /sys/fs/bcachefs/*/options/data_replicas
    #echo 2 > /sys/fs/bcachefs/*/options/metadata_replicas

    bcachefs data rereplicate /mnt

    bcachefs fs usage -h /mnt
    umount /mnt

    mount -t bcachefs -o degraded ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]} /mnt
    umount /mnt

    bcachefs fsck -ny ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]} ${ktest_scratch_dev[2]}
    check_counters ${ktest_scratch_dev[0]}
}

disabled_test_device_add_faults()
{
    setup_tracing 'bcachefs:*'

    run_quiet "" bcachefs format -f		\
	--errors=remount-ro			\
	${ktest_scratch_dev[0]}
    mount ${ktest_scratch_dev[0]} /mnt

    device_add_fault_fn()
    {
	set_faults "$1 enable"
	! bcachefs device add -f /mnt ${ktest_scratch_dev[1]}
    }

    run_fault_injection_test bcachefs:add			device_add_fault_fn
    run_fault_injection_test bcachefs:bch_fs_init:dev_alloc	device_add_fault_fn
    run_fault_injection_test bcachefs:bch_fs_init:alloc_start	device_add_fault_fn
    run_fault_injection_test bcachefs:bch_fs_init:read_super	device_add_fault_fn

    check_counters ${ktest_scratch_dev[0]}
}

test_striping()
{
    set_watchdog 60
    setup_tracing 'bcachefs:*'

    if false; then
	run_quiet "" bcachefs format -f		\
	    --replicas=2			\
	    ${ktest_scratch_dev[0]}		\
	    ${ktest_scratch_dev[1]}

	mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]} /mnt

	dd if=/dev/zero of=/mnt/test0 bs=1M count=1024 oflag=direct
	bcachefs fs usage -h /mnt

	bcachefs device add -f /mnt ${ktest_scratch_dev[2]}

	dd if=/dev/zero of=/mnt/test1 bs=1M count=1024 oflag=direct
    else
	run_quiet "" bcachefs format		\
	    --data_checksum=none		\
	    --replicas=2			\
	    --label cache			\
	    --durability=1			\
	    --bucket=8M				\
	    ${ktest_scratch_dev[0]}		\
	    --label hdd				\
	    --durability=2			\
	    ${ktest_scratch_dev[1]}		\
	    ${ktest_scratch_dev[2]}		\
	    --foreground_target=cache		\
	    --background_target=hdd		\
	    --promote_target=cache

	mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]}:${ktest_scratch_dev[2]} /mnt

	dd if=/dev/zero of=/mnt/test0 bs=1M count=1024 oflag=direct
    fi

    bcachefs fs usage -h /mnt

    umount /mnt

    check_counters ${ktest_scratch_dev[0]}
}

disabled_change_replicas()
{
    run_quiet "" bcachefs format -f		\
	--replicas=2				\
	${ktest_scratch_dev[0]}			\
	${ktest_scratch_dev[1]}

    mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]} /mnt

    (
	cd /sys/fs/bcachefs/*
	while true; do
	    echo 1 > options/data_replicas
	    sleep 0.1
	    echo 2 > options/data_replicas
	    sleep 0.1
	done
    ) &

    run_fio_randrw

    while true; do
	dd if=/dev/zero of=/mnt/foo bs=1M count=1024 oflag=direct
	sync
	#rm /mnt/foo
    done

    kill %1
    umount /mnt

    check_counters ${ktest_scratch_dev[0]}
}

disabled_change_replicas2()
{
    run_quiet "" bcachefs format -f		\
	--replicas=2				\
	${ktest_scratch_dev[0]}			\
	${ktest_scratch_dev[1]}

    mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]} /mnt

    local opt=/sys/fs/bcachefs/*/options/data_replicas
    while true; do
	echo 1 > $opt
	dd if=/dev/zero of=/mnt/foo bs=1M count=1024
	echo 2 > $opt
	sync
	#rm /mnt/foo
    done

    umount /mnt

    bcachefs fsck -ny ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]}
    check_counters ${ktest_scratch_dev[0]}
}

test_replicas_gc()
{
    local p=/sys/module/bcachefs/parameters/force_reconstruct_read

    set_watchdog 60

    [[ -f $p ]] && echo 1 > $p

    run_quiet "" bcachefs format -f		\
	--replicas=2				\
	${ktest_scratch_dev[@]}	
    devs="$(join_by : "${ktest_scratch_dev[@]}")"

    mount -t bcachefs $devs /mnt

    antagonist_trigger_gc &
    run_fio_randrw

    echo 1 > /sys/fs/bcachefs/*/internal/trigger_gc
    umount /mnt

    # test remount:
    mount -t bcachefs -o fix_errors,erasure_code $devs /mnt
    umount /mnt

    bcachefs fsck -ny ${ktest_scratch_dev[@]}
    check_counters ${ktest_scratch_dev[0]}

    [[ -f $p ]] && echo 0 > $p
    true
}

test_replace_replica()
{
    set_watchdog 60

    bcachefs format --replicas=2 ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]}
    mount -t bcachefs -o degraded ${ktest_scratch_dev[0]} /mnt

    bcachefs device add -f /mnt ${ktest_scratch_dev[2]}

    bcachefs device set-state --force failed ${ktest_scratch_dev[1]}

    bcachefs data rereplicate /mnt

    umount /mnt

    mount -t bcachefs -o degraded ${ktest_scratch_dev[2]} /mnt
    umount /mnt

    # We expect more btree_path_upgrade_fail events than usual in this test,
    # due to the way the btree node rewrite path works and the fact that we're
    # rewriting every btree node
    check_counters ${ktest_scratch_dev[0]} 4
}

test_key_type_error()
{
    set_watchdog 20

    bcachefs format --replicas=2 ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]}

    mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]} /mnt
    bcachefs setattr --data_replicas=1 /mnt
    bcachefs setattr --foreground_target=$(basename ${ktest_scratch_dev[1]}) /mnt

    dd if=/dev/zero of=/mnt/foo bs=64k count=8 oflag=direct

    umount /mnt
    mount -t bcachefs -o very_degraded ${ktest_scratch_dev[0]} /mnt

    bcachefs device remove --force ${ktest_scratch_dev[1]}

    # should fail:
    ! dd if=/mnt/foo of=/dev/null bs=64k count=1 iflag=direct

    umount /mnt

    bcachefs fsck -ny ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]}
    check_counters ${ktest_scratch_dev[0]}
}

test_force_remove()
{
    set_watchdog 20

    bcachefs format -f ${ktest_scratch_dev[0]} ${ktest_scratch_dev[1]}

    mount -t bcachefs ${ktest_scratch_dev[0]}:${ktest_scratch_dev[1]} /mnt

    fio --group_reporting		\
	--time_based			\
	--runtime=60s			\
	--ioengine=io_uring		\
	--iodepth=256			\
	--verify_async=8		\
	--bs=4k-64k			\
	--directory=/mnt		\
	--size=16m			\
	--norandommap			\
	--random_distribution=zipf:0.5	\
	--numjobs=16			\
	--rw=randrw			\
	--name=A			\
	--direct=1			\
	--name=B			\
	--direct=0 >/dev/null &
    bcachefs device evacuate ${ktest_scratch_dev[0]} || true
    bcachefs device remove --force --force-metadata ${ktest_scratch_dev[1]} || true
    bcachefs device remove --force --force-metadata ${ktest_scratch_dev[1]} || true
    kill %1
    wait
    umount /mnt
}

main "$@"