summaryrefslogtreecommitdiff
path: root/drivers/md
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@ppc970.osdl.org>2005-04-16 15:20:36 -0700
committerLinus Torvalds <torvalds@ppc970.osdl.org>2005-04-16 15:20:36 -0700
commit1da177e4c3f41524e886b7f1b8a0c1fc7321cac2 (patch)
tree0bba044c4ce775e45a88a51686b5d9f90697ea9d /drivers/md
Linux-2.6.12-rc2v2.6.12-rc2
Initial git repository build. I'm not bothering with the full history, even though we have it. We can create a separate "historical" git archive of that later if we want to, and in the meantime it's about 3.2GB when imported into git - space that would just make the early git days unnecessarily complicated, when we don't have a lot of good infrastructure for it. Let it rip!
Diffstat (limited to 'drivers/md')
-rw-r--r--drivers/md/Kconfig240
-rw-r--r--drivers/md/Makefile107
-rw-r--r--drivers/md/dm-bio-list.h68
-rw-r--r--drivers/md/dm-bio-record.h45
-rw-r--r--drivers/md/dm-crypt.c977
-rw-r--r--drivers/md/dm-emc.c359
-rw-r--r--drivers/md/dm-exception-store.c648
-rw-r--r--drivers/md/dm-hw-handler.c216
-rw-r--r--drivers/md/dm-hw-handler.h61
-rw-r--r--drivers/md/dm-io.c426
-rw-r--r--drivers/md/dm-io.h77
-rw-r--r--drivers/md/dm-ioctl.c1416
-rw-r--r--drivers/md/dm-linear.c123
-rw-r--r--drivers/md/dm-log.c711
-rw-r--r--drivers/md/dm-log.h130
-rw-r--r--drivers/md/dm-mpath.c1302
-rw-r--r--drivers/md/dm-mpath.h25
-rw-r--r--drivers/md/dm-path-selector.c156
-rw-r--r--drivers/md/dm-path-selector.h93
-rw-r--r--drivers/md/dm-raid1.c1269
-rw-r--r--drivers/md/dm-round-robin.c214
-rw-r--r--drivers/md/dm-snap.c1208
-rw-r--r--drivers/md/dm-snap.h161
-rw-r--r--drivers/md/dm-stripe.c234
-rw-r--r--drivers/md/dm-table.c950
-rw-r--r--drivers/md/dm-target.c196
-rw-r--r--drivers/md/dm-zero.c81
-rw-r--r--drivers/md/dm.c1194
-rw-r--r--drivers/md/dm.h195
-rw-r--r--drivers/md/faulty.c343
-rw-r--r--drivers/md/kcopyd.c687
-rw-r--r--drivers/md/kcopyd.h42
-rw-r--r--drivers/md/linear.c343
-rw-r--r--drivers/md/md.c3766
-rw-r--r--drivers/md/mktables.c125
-rw-r--r--drivers/md/multipath.c584
-rw-r--r--drivers/md/raid0.c539
-rw-r--r--drivers/md/raid1.c1449
-rw-r--r--drivers/md/raid10.c1787
-rw-r--r--drivers/md/raid5.c1965
-rw-r--r--drivers/md/raid6.h135
-rw-r--r--drivers/md/raid6algos.c153
-rw-r--r--drivers/md/raid6altivec.uc122
-rw-r--r--drivers/md/raid6int.uc117
-rw-r--r--drivers/md/raid6main.c2136
-rw-r--r--drivers/md/raid6mmx.c150
-rw-r--r--drivers/md/raid6recov.c133
-rw-r--r--drivers/md/raid6sse1.c171
-rw-r--r--drivers/md/raid6sse2.c270
-rw-r--r--drivers/md/raid6test/Makefile58
-rw-r--r--drivers/md/raid6test/test.c103
-rw-r--r--drivers/md/raid6x86.h245
-rw-r--r--drivers/md/unroll.pl24
-rw-r--r--drivers/md/xor.c154
54 files changed, 28483 insertions, 0 deletions
diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig
new file mode 100644
index 000000000000..ac43f98062fd
--- /dev/null
+++ b/drivers/md/Kconfig
@@ -0,0 +1,240 @@
+#
+# Block device driver configuration
+#
+
+menu "Multi-device support (RAID and LVM)"
+
+config MD
+ bool "Multiple devices driver support (RAID and LVM)"
+ help
+ Support multiple physical spindles through a single logical device.
+ Required for RAID and logical volume management.
+
+config BLK_DEV_MD
+ tristate "RAID support"
+ depends on MD
+ ---help---
+ This driver lets you combine several hard disk partitions into one
+ logical block device. This can be used to simply append one
+ partition to another one or to combine several redundant hard disks
+ into a RAID1/4/5 device so as to provide protection against hard
+ disk failures. This is called "Software RAID" since the combining of
+ the partitions is done by the kernel. "Hardware RAID" means that the
+ combining is done by a dedicated controller; if you have such a
+ controller, you do not need to say Y here.
+
+ More information about Software RAID on Linux is contained in the
+ Software RAID mini-HOWTO, available from
+ <http://www.tldp.org/docs.html#howto>. There you will also learn
+ where to get the supporting user space utilities raidtools.
+
+ If unsure, say N.
+
+config MD_LINEAR
+ tristate "Linear (append) mode"
+ depends on BLK_DEV_MD
+ ---help---
+ If you say Y here, then your multiple devices driver will be able to
+ use the so-called linear mode, i.e. it will combine the hard disk
+ partitions by simply appending one to the other.
+
+ To compile this as a module, choose M here: the module
+ will be called linear.
+
+ If unsure, say Y.
+
+config MD_RAID0
+ tristate "RAID-0 (striping) mode"
+ depends on BLK_DEV_MD
+ ---help---
+ If you say Y here, then your multiple devices driver will be able to
+ use the so-called raid0 mode, i.e. it will combine the hard disk
+ partitions into one logical device in such a fashion as to fill them
+ up evenly, one chunk here and one chunk there. This will increase
+ the throughput rate if the partitions reside on distinct disks.
+
+ Information about Software RAID on Linux is contained in the
+ Software-RAID mini-HOWTO, available from
+ <http://www.tldp.org/docs.html#howto>. There you will also
+ learn where to get the supporting user space utilities raidtools.
+
+ To compile this as a module, choose M here: the module
+ will be called raid0.
+
+ If unsure, say Y.
+
+config MD_RAID1
+ tristate "RAID-1 (mirroring) mode"
+ depends on BLK_DEV_MD
+ ---help---
+ A RAID-1 set consists of several disk drives which are exact copies
+ of each other. In the event of a mirror failure, the RAID driver
+ will continue to use the operational mirrors in the set, providing
+ an error free MD (multiple device) to the higher levels of the
+ kernel. In a set with N drives, the available space is the capacity
+ of a single drive, and the set protects against a failure of (N - 1)
+ drives.
+
+ Information about Software RAID on Linux is contained in the
+ Software-RAID mini-HOWTO, available from
+ <http://www.tldp.org/docs.html#howto>. There you will also
+ learn where to get the supporting user space utilities raidtools.
+
+ If you want to use such a RAID-1 set, say Y. To compile this code
+ as a module, choose M here: the module will be called raid1.
+
+ If unsure, say Y.
+
+config MD_RAID10
+ tristate "RAID-10 (mirrored striping) mode (EXPERIMENTAL)"
+ depends on BLK_DEV_MD && EXPERIMENTAL
+ ---help---
+ RAID-10 provides a combination of striping (RAID-0) and
+ mirroring (RAID-1) with easier configuration and more flexable
+ layout.
+ Unlike RAID-0, but like RAID-1, RAID-10 requires all devices to
+ be the same size (or at least, only as much as the smallest device
+ will be used).
+ RAID-10 provides a variety of layouts that provide different levels
+ of redundancy and performance.
+
+ RAID-10 requires mdadm-1.7.0 or later, available at:
+
+ ftp://ftp.kernel.org/pub/linux/utils/raid/mdadm/
+
+ If unsure, say Y.
+
+config MD_RAID5
+ tristate "RAID-4/RAID-5 mode"
+ depends on BLK_DEV_MD
+ ---help---
+ A RAID-5 set of N drives with a capacity of C MB per drive provides
+ the capacity of C * (N - 1) MB, and protects against a failure
+ of a single drive. For a given sector (row) number, (N - 1) drives
+ contain data sectors, and one drive contains the parity protection.
+ For a RAID-4 set, the parity blocks are present on a single drive,
+ while a RAID-5 set distributes the parity across the drives in one
+ of the available parity distribution methods.
+
+ Information about Software RAID on Linux is contained in the
+ Software-RAID mini-HOWTO, available from
+ <http://www.tldp.org/docs.html#howto>. There you will also
+ learn where to get the supporting user space utilities raidtools.
+
+ If you want to use such a RAID-4/RAID-5 set, say Y. To
+ compile this code as a module, choose M here: the module
+ will be called raid5.
+
+ If unsure, say Y.
+
+config MD_RAID6
+ tristate "RAID-6 mode"
+ depends on BLK_DEV_MD
+ ---help---
+ A RAID-6 set of N drives with a capacity of C MB per drive
+ provides the capacity of C * (N - 2) MB, and protects
+ against a failure of any two drives. For a given sector
+ (row) number, (N - 2) drives contain data sectors, and two
+ drives contains two independent redundancy syndromes. Like
+ RAID-5, RAID-6 distributes the syndromes across the drives
+ in one of the available parity distribution methods.
+
+ RAID-6 requires mdadm-1.5.0 or later, available at:
+
+ ftp://ftp.kernel.org/pub/linux/utils/raid/mdadm/
+
+ If you want to use such a RAID-6 set, say Y. To compile
+ this code as a module, choose M here: the module will be
+ called raid6.
+
+ If unsure, say Y.
+
+config MD_MULTIPATH
+ tristate "Multipath I/O support"
+ depends on BLK_DEV_MD
+ help
+ Multipath-IO is the ability of certain devices to address the same
+ physical disk over multiple 'IO paths'. The code ensures that such
+ paths can be defined and handled at runtime, and ensures that a
+ transparent failover to the backup path(s) happens if a IO errors
+ arrives on the primary path.
+
+ If unsure, say N.
+
+config MD_FAULTY
+ tristate "Faulty test module for MD"
+ depends on BLK_DEV_MD
+ help
+ The "faulty" module allows for a block device that occasionally returns
+ read or write errors. It is useful for testing.
+
+ In unsure, say N.
+
+config BLK_DEV_DM
+ tristate "Device mapper support"
+ depends on MD
+ ---help---
+ Device-mapper is a low level volume manager. It works by allowing
+ people to specify mappings for ranges of logical sectors. Various
+ mapping types are available, in addition people may write their own
+ modules containing custom mappings if they wish.
+
+ Higher level volume managers such as LVM2 use this driver.
+
+ To compile this as a module, choose M here: the module will be
+ called dm-mod.
+
+ If unsure, say N.
+
+config DM_CRYPT
+ tristate "Crypt target support"
+ depends on BLK_DEV_DM && EXPERIMENTAL
+ select CRYPTO
+ ---help---
+ This device-mapper target allows you to create a device that
+ transparently encrypts the data on it. You'll need to activate
+ the ciphers you're going to use in the cryptoapi configuration.
+
+ Information on how to use dm-crypt can be found on
+
+ <http://www.saout.de/misc/dm-crypt/>
+
+ To compile this code as a module, choose M here: the module will
+ be called dm-crypt.
+
+ If unsure, say N.
+
+config DM_SNAPSHOT
+ tristate "Snapshot target (EXPERIMENTAL)"
+ depends on BLK_DEV_DM && EXPERIMENTAL
+ ---help---
+ Allow volume managers to take writeable snapshots of a device.
+
+config DM_MIRROR
+ tristate "Mirror target (EXPERIMENTAL)"
+ depends on BLK_DEV_DM && EXPERIMENTAL
+ ---help---
+ Allow volume managers to mirror logical volumes, also
+ needed for live data migration tools such as 'pvmove'.
+
+config DM_ZERO
+ tristate "Zero target (EXPERIMENTAL)"
+ depends on BLK_DEV_DM && EXPERIMENTAL
+ ---help---
+ A target that discards writes, and returns all zeroes for
+ reads. Useful in some recovery situations.
+
+config DM_MULTIPATH
+ tristate "Multipath target (EXPERIMENTAL)"
+ depends on BLK_DEV_DM && EXPERIMENTAL
+ ---help---
+ Allow volume managers to support multipath hardware.
+
+config DM_MULTIPATH_EMC
+ tristate "EMC CX/AX multipath support (EXPERIMENTAL)"
+ depends on DM_MULTIPATH && BLK_DEV_DM && EXPERIMENTAL
+ ---help---
+ Multipath support for EMC CX/AX series hardware.
+
+endmenu
+
diff --git a/drivers/md/Makefile b/drivers/md/Makefile
new file mode 100644
index 000000000000..90de9c146a5f
--- /dev/null
+++ b/drivers/md/Makefile
@@ -0,0 +1,107 @@
+#
+# Makefile for the kernel software RAID and LVM drivers.
+#
+
+dm-mod-objs := dm.o dm-table.o dm-target.o dm-linear.o dm-stripe.o \
+ dm-ioctl.o dm-io.o kcopyd.o
+dm-multipath-objs := dm-hw-handler.o dm-path-selector.o dm-mpath.o
+dm-snapshot-objs := dm-snap.o dm-exception-store.o
+dm-mirror-objs := dm-log.o dm-raid1.o
+raid6-objs := raid6main.o raid6algos.o raid6recov.o raid6tables.o \
+ raid6int1.o raid6int2.o raid6int4.o \
+ raid6int8.o raid6int16.o raid6int32.o \
+ raid6altivec1.o raid6altivec2.o raid6altivec4.o \
+ raid6altivec8.o \
+ raid6mmx.o raid6sse1.o raid6sse2.o
+hostprogs-y := mktables
+
+# Note: link order is important. All raid personalities
+# and xor.o must come before md.o, as they each initialise
+# themselves, and md.o may use the personalities when it
+# auto-initialised.
+
+obj-$(CONFIG_MD_LINEAR) += linear.o
+obj-$(CONFIG_MD_RAID0) += raid0.o
+obj-$(CONFIG_MD_RAID1) += raid1.o
+obj-$(CONFIG_MD_RAID10) += raid10.o
+obj-$(CONFIG_MD_RAID5) += raid5.o xor.o
+obj-$(CONFIG_MD_RAID6) += raid6.o xor.o
+obj-$(CONFIG_MD_MULTIPATH) += multipath.o
+obj-$(CONFIG_MD_FAULTY) += faulty.o
+obj-$(CONFIG_BLK_DEV_MD) += md.o
+obj-$(CONFIG_BLK_DEV_DM) += dm-mod.o
+obj-$(CONFIG_DM_CRYPT) += dm-crypt.o
+obj-$(CONFIG_DM_MULTIPATH) += dm-multipath.o dm-round-robin.o
+obj-$(CONFIG_DM_MULTIPATH_EMC) += dm-emc.o
+obj-$(CONFIG_DM_SNAPSHOT) += dm-snapshot.o
+obj-$(CONFIG_DM_MIRROR) += dm-mirror.o
+obj-$(CONFIG_DM_ZERO) += dm-zero.o
+
+quiet_cmd_unroll = UNROLL $@
+ cmd_unroll = $(PERL) $(srctree)/$(src)/unroll.pl $(UNROLL) \
+ < $< > $@ || ( rm -f $@ && exit 1 )
+
+ifeq ($(CONFIG_ALTIVEC),y)
+altivec_flags := -maltivec -mabi=altivec
+endif
+
+targets += raid6int1.c
+$(obj)/raid6int1.c: UNROLL := 1
+$(obj)/raid6int1.c: $(src)/raid6int.uc $(src)/unroll.pl FORCE
+ $(call if_changed,unroll)
+
+targets += raid6int2.c
+$(obj)/raid6int2.c: UNROLL := 2
+$(obj)/raid6int2.c: $(src)/raid6int.uc $(src)/unroll.pl FORCE
+ $(call if_changed,unroll)
+
+targets += raid6int4.c
+$(obj)/raid6int4.c: UNROLL := 4
+$(obj)/raid6int4.c: $(src)/raid6int.uc $(src)/unroll.pl FORCE
+ $(call if_changed,unroll)
+
+targets += raid6int8.c
+$(obj)/raid6int8.c: UNROLL := 8
+$(obj)/raid6int8.c: $(src)/raid6int.uc $(src)/unroll.pl FORCE
+ $(call if_changed,unroll)
+
+targets += raid6int16.c
+$(obj)/raid6int16.c: UNROLL := 16
+$(obj)/raid6int16.c: $(src)/raid6int.uc $(src)/unroll.pl FORCE
+ $(call if_changed,unroll)
+
+targets += raid6int32.c
+$(obj)/raid6int32.c: UNROLL := 32
+$(obj)/raid6int32.c: $(src)/raid6int.uc $(src)/unroll.pl FORCE
+ $(call if_changed,unroll)
+
+CFLAGS_raid6altivec1.o += $(altivec_flags)
+targets += raid6altivec1.c
+$(obj)/raid6altivec1.c: UNROLL := 1
+$(obj)/raid6altivec1.c: $(src)/raid6altivec.uc $(src)/unroll.pl FORCE
+ $(call if_changed,unroll)
+
+CFLAGS_raid6altivec2.o += $(altivec_flags)
+targets += raid6altivec2.c
+$(obj)/raid6altivec2.c: UNROLL := 2
+$(obj)/raid6altivec2.c: $(src)/raid6altivec.uc $(src)/unroll.pl FORCE
+ $(call if_changed,unroll)
+
+CFLAGS_raid6altivec4.o += $(altivec_flags)
+targets += raid6altivec4.c
+$(obj)/raid6altivec4.c: UNROLL := 4
+$(obj)/raid6altivec4.c: $(src)/raid6altivec.uc $(src)/unroll.pl FORCE
+ $(call if_changed,unroll)
+
+CFLAGS_raid6altivec8.o += $(altivec_flags)
+targets += raid6altivec8.c
+$(obj)/raid6altivec8.c: UNROLL := 8
+$(obj)/raid6altivec8.c: $(src)/raid6altivec.uc $(src)/unroll.pl FORCE
+ $(call if_changed,unroll)
+
+quiet_cmd_mktable = TABLE $@
+ cmd_mktable = $(obj)/mktables > $@ || ( rm -f $@ && exit 1 )
+
+targets += raid6tables.c
+$(obj)/raid6tables.c: $(obj)/mktables FORCE
+ $(call if_changed,mktable)
diff --git a/drivers/md/dm-bio-list.h b/drivers/md/dm-bio-list.h
new file mode 100644
index 000000000000..bc021e1fd4d1
--- /dev/null
+++ b/drivers/md/dm-bio-list.h
@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2004 Red Hat UK Ltd.
+ *
+ * This file is released under the GPL.
+ */
+
+#ifndef DM_BIO_LIST_H
+#define DM_BIO_LIST_H
+
+#include <linux/bio.h>
+
+struct bio_list {
+ struct bio *head;
+ struct bio *tail;
+};
+
+static inline void bio_list_init(struct bio_list *bl)
+{
+ bl->head = bl->tail = NULL;
+}
+
+static inline void bio_list_add(struct bio_list *bl, struct bio *bio)
+{
+ bio->bi_next = NULL;
+
+ if (bl->tail)
+ bl->tail->bi_next = bio;
+ else
+ bl->head = bio;
+
+ bl->tail = bio;
+}
+
+static inline void bio_list_merge(struct bio_list *bl, struct bio_list *bl2)
+{
+ if (bl->tail)
+ bl->tail->bi_next = bl2->head;
+ else
+ bl->head = bl2->head;
+
+ bl->tail = bl2->tail;
+}
+
+static inline struct bio *bio_list_pop(struct bio_list *bl)
+{
+ struct bio *bio = bl->head;
+
+ if (bio) {
+ bl->head = bl->head->bi_next;
+ if (!bl->head)
+ bl->tail = NULL;
+
+ bio->bi_next = NULL;
+ }
+
+ return bio;
+}
+
+static inline struct bio *bio_list_get(struct bio_list *bl)
+{
+ struct bio *bio = bl->head;
+
+ bl->head = bl->tail = NULL;
+
+ return bio;
+}
+
+#endif
diff --git a/drivers/md/dm-bio-record.h b/drivers/md/dm-bio-record.h
new file mode 100644
index 000000000000..d3ec217847d6
--- /dev/null
+++ b/drivers/md/dm-bio-record.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the GPL.
+ */
+
+#ifndef DM_BIO_RECORD_H
+#define DM_BIO_RECORD_H
+
+#include <linux/bio.h>
+
+/*
+ * There are lots of mutable fields in the bio struct that get
+ * changed by the lower levels of the block layer. Some targets,
+ * such as multipath, may wish to resubmit a bio on error. The
+ * functions in this file help the target record and restore the
+ * original bio state.
+ */
+struct dm_bio_details {
+ sector_t bi_sector;
+ struct block_device *bi_bdev;
+ unsigned int bi_size;
+ unsigned short bi_idx;
+ unsigned long bi_flags;
+};
+
+static inline void dm_bio_record(struct dm_bio_details *bd, struct bio *bio)
+{
+ bd->bi_sector = bio->bi_sector;
+ bd->bi_bdev = bio->bi_bdev;
+ bd->bi_size = bio->bi_size;
+ bd->bi_idx = bio->bi_idx;
+ bd->bi_flags = bio->bi_flags;
+}
+
+static inline void dm_bio_restore(struct dm_bio_details *bd, struct bio *bio)
+{
+ bio->bi_sector = bd->bi_sector;
+ bio->bi_bdev = bd->bi_bdev;
+ bio->bi_size = bd->bi_size;
+ bio->bi_idx = bd->bi_idx;
+ bio->bi_flags = bd->bi_flags;
+}
+
+#endif
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
new file mode 100644
index 000000000000..77619a56e2bf
--- /dev/null
+++ b/drivers/md/dm-crypt.c
@@ -0,0 +1,977 @@
+/*
+ * Copyright (C) 2003 Christophe Saout <christophe@saout.de>
+ * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
+ *
+ * This file is released under the GPL.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/bio.h>
+#include <linux/blkdev.h>
+#include <linux/mempool.h>
+#include <linux/slab.h>
+#include <linux/crypto.h>
+#include <linux/workqueue.h>
+#include <asm/atomic.h>
+#include <asm/scatterlist.h>
+#include <asm/page.h>
+
+#include "dm.h"
+
+#define PFX "crypt: "
+
+/*
+ * per bio private data
+ */
+struct crypt_io {
+ struct dm_target *target;
+ struct bio *bio;
+ struct bio *first_clone;
+ struct work_struct work;
+ atomic_t pending;
+ int error;
+};
+
+/*
+ * context holding the current state of a multi-part conversion
+ */
+struct convert_context {
+ struct bio *bio_in;
+ struct bio *bio_out;
+ unsigned int offset_in;
+ unsigned int offset_out;
+ unsigned int idx_in;
+ unsigned int idx_out;
+ sector_t sector;
+ int write;
+};
+
+struct crypt_config;
+
+struct crypt_iv_operations {
+ int (*ctr)(struct crypt_config *cc, struct dm_target *ti,
+ const char *opts);
+ void (*dtr)(struct crypt_config *cc);
+ const char *(*status)(struct crypt_config *cc);
+ int (*generator)(struct crypt_config *cc, u8 *iv, sector_t sector);
+};
+
+/*
+ * Crypt: maps a linear range of a block device
+ * and encrypts / decrypts at the same time.
+ */
+struct crypt_config {
+ struct dm_dev *dev;
+ sector_t start;
+
+ /*
+ * pool for per bio private data and
+ * for encryption buffer pages
+ */
+ mempool_t *io_pool;
+ mempool_t *page_pool;
+
+ /*
+ * crypto related data
+ */
+ struct crypt_iv_operations *iv_gen_ops;
+ char *iv_mode;
+ void *iv_gen_private;
+ sector_t iv_offset;
+ unsigned int iv_size;
+
+ struct crypto_tfm *tfm;
+ unsigned int key_size;
+ u8 key[0];
+};
+
+#define MIN_IOS 256
+#define MIN_POOL_PAGES 32
+#define MIN_BIO_PAGES 8
+
+static kmem_cache_t *_crypt_io_pool;
+
+/*
+ * Mempool alloc and free functions for the page
+ */
+static void *mempool_alloc_page(unsigned int __nocast gfp_mask, void *data)
+{
+ return alloc_page(gfp_mask);
+}
+
+static void mempool_free_page(void *page, void *data)
+{
+ __free_page(page);
+}
+
+
+/*
+ * Different IV generation algorithms:
+ *
+ * plain: the initial vector is the 32-bit low-endian version of the sector
+ * number, padded with zeros if neccessary.
+ *
+ * ess_iv: "encrypted sector|salt initial vector", the sector number is
+ * encrypted with the bulk cipher using a salt as key. The salt
+ * should be derived from the bulk cipher's key via hashing.
+ *
+ * plumb: unimplemented, see:
+ * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454
+ */
+
+static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
+{
+ memset(iv, 0, cc->iv_size);
+ *(u32 *)iv = cpu_to_le32(sector & 0xffffffff);
+
+ return 0;
+}
+
+static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
+ const char *opts)
+{
+ struct crypto_tfm *essiv_tfm;
+ struct crypto_tfm *hash_tfm;
+ struct scatterlist sg;
+ unsigned int saltsize;
+ u8 *salt;
+
+ if (opts == NULL) {
+ ti->error = PFX "Digest algorithm missing for ESSIV mode";
+ return -EINVAL;
+ }
+
+ /* Hash the cipher key with the given hash algorithm */
+ hash_tfm = crypto_alloc_tfm(opts, 0);
+ if (hash_tfm == NULL) {
+ ti->error = PFX "Error initializing ESSIV hash";
+ return -EINVAL;
+ }
+
+ if (crypto_tfm_alg_type(hash_tfm) != CRYPTO_ALG_TYPE_DIGEST) {
+ ti->error = PFX "Expected digest algorithm for ESSIV hash";
+ crypto_free_tfm(hash_tfm);
+ return -EINVAL;
+ }
+
+ saltsize = crypto_tfm_alg_digestsize(hash_tfm);
+ salt = kmalloc(saltsize, GFP_KERNEL);
+ if (salt == NULL) {
+ ti->error = PFX "Error kmallocing salt storage in ESSIV";
+ crypto_free_tfm(hash_tfm);
+ return -ENOMEM;
+ }
+
+ sg.page = virt_to_page(cc->key);
+ sg.offset = offset_in_page(cc->key);
+ sg.length = cc->key_size;
+ crypto_digest_digest(hash_tfm, &sg, 1, salt);
+ crypto_free_tfm(hash_tfm);
+
+ /* Setup the essiv_tfm with the given salt */
+ essiv_tfm = crypto_alloc_tfm(crypto_tfm_alg_name(cc->tfm),
+ CRYPTO_TFM_MODE_ECB);
+ if (essiv_tfm == NULL) {
+ ti->error = PFX "Error allocating crypto tfm for ESSIV";
+ kfree(salt);
+ return -EINVAL;
+ }
+ if (crypto_tfm_alg_blocksize(essiv_tfm)
+ != crypto_tfm_alg_ivsize(cc->tfm)) {
+ ti->error = PFX "Block size of ESSIV cipher does "
+ "not match IV size of block cipher";
+ crypto_free_tfm(essiv_tfm);
+ kfree(salt);
+ return -EINVAL;
+ }
+ if (crypto_cipher_setkey(essiv_tfm, salt, saltsize) < 0) {
+ ti->error = PFX "Failed to set key for ESSIV cipher";
+ crypto_free_tfm(essiv_tfm);
+ kfree(salt);
+ return -EINVAL;
+ }
+ kfree(salt);
+
+ cc->iv_gen_private = (void *)essiv_tfm;
+ return 0;
+}
+
+static void crypt_iv_essiv_dtr(struct crypt_config *cc)
+{
+ crypto_free_tfm((struct crypto_tfm *)cc->iv_gen_private);
+ cc->iv_gen_private = NULL;
+}
+
+static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
+{
+ struct scatterlist sg = { NULL, };
+
+ memset(iv, 0, cc->iv_size);
+ *(u64 *)iv = cpu_to_le64(sector);
+
+ sg.page = virt_to_page(iv);
+ sg.offset = offset_in_page(iv);
+ sg.length = cc->iv_size;
+ crypto_cipher_encrypt((struct crypto_tfm *)cc->iv_gen_private,
+ &sg, &sg, cc->iv_size);
+
+ return 0;
+}
+
+static struct crypt_iv_operations crypt_iv_plain_ops = {
+ .generator = crypt_iv_plain_gen
+};
+
+static struct crypt_iv_operations crypt_iv_essiv_ops = {
+ .ctr = crypt_iv_essiv_ctr,
+ .dtr = crypt_iv_essiv_dtr,
+ .generator = crypt_iv_essiv_gen
+};
+
+
+static inline int
+crypt_convert_scatterlist(struct crypt_config *cc, struct scatterlist *out,
+ struct scatterlist *in, unsigned int length,
+ int write, sector_t sector)
+{
+ u8 iv[cc->iv_size];
+ int r;
+
+ if (cc->iv_gen_ops) {
+ r = cc->iv_gen_ops->generator(cc, iv, sector);
+ if (r < 0)
+ return r;
+
+ if (write)
+ r = crypto_cipher_encrypt_iv(cc->tfm, out, in, length, iv);
+ else
+ r = crypto_cipher_decrypt_iv(cc->tfm, out, in, length, iv);
+ } else {
+ if (write)
+ r = crypto_cipher_encrypt(cc->tfm, out, in, length);
+ else
+ r = crypto_cipher_decrypt(cc->tfm, out, in, length);
+ }
+
+ return r;
+}
+
+static void
+crypt_convert_init(struct crypt_config *cc, struct convert_context *ctx,
+ struct bio *bio_out, struct bio *bio_in,
+ sector_t sector, int write)
+{
+ ctx->bio_in = bio_in;
+ ctx->bio_out = bio_out;
+ ctx->offset_in = 0;
+ ctx->offset_out = 0;
+ ctx->idx_in = bio_in ? bio_in->bi_idx : 0;
+ ctx->idx_out = bio_out ? bio_out->bi_idx : 0;
+ ctx->sector = sector + cc->iv_offset;
+ ctx->write = write;
+}
+
+/*
+ * Encrypt / decrypt data from one bio to another one (can be the same one)
+ */
+static int crypt_convert(struct crypt_config *cc,
+ struct convert_context *ctx)
+{
+ int r = 0;
+
+ while(ctx->idx_in < ctx->bio_in->bi_vcnt &&
+ ctx->idx_out < ctx->bio_out->bi_vcnt) {
+ struct bio_vec *bv_in = bio_iovec_idx(ctx->bio_in, ctx->idx_in);
+ struct bio_vec *bv_out = bio_iovec_idx(ctx->bio_out, ctx->idx_out);
+ struct scatterlist sg_in = {
+ .page = bv_in->bv_page,
+ .offset = bv_in->bv_offset + ctx->offset_in,
+ .length = 1 << SECTOR_SHIFT
+ };
+ struct scatterlist sg_out = {
+ .page = bv_out->bv_page,
+ .offset = bv_out->bv_offset + ctx->offset_out,
+ .length = 1 << SECTOR_SHIFT
+ };
+
+ ctx->offset_in += sg_in.length;
+ if (ctx->offset_in >= bv_in->bv_len) {
+ ctx->offset_in = 0;
+ ctx->idx_in++;
+ }
+
+ ctx->offset_out += sg_out.length;
+ if (ctx->offset_out >= bv_out->bv_len) {
+ ctx->offset_out = 0;
+ ctx->idx_out++;
+ }
+
+ r = crypt_convert_scatterlist(cc, &sg_out, &sg_in, sg_in.length,
+ ctx->write, ctx->sector);
+ if (r < 0)
+ break;
+
+ ctx->sector++;
+ }
+
+ return r;
+}
+
+/*
+ * Generate a new unfragmented bio with the given size
+ * This should never violate the device limitations
+ * May return a smaller bio when running out of pages
+ */
+static struct bio *
+crypt_alloc_buffer(struct crypt_config *cc, unsigned int size,
+ struct bio *base_bio, unsigned int *bio_vec_idx)
+{
+ struct bio *bio;
+ unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
+ int gfp_mask = GFP_NOIO | __GFP_HIGHMEM;
+ unsigned long flags = current->flags;
+ unsigned int i;
+
+ /*
+ * Tell VM to act less aggressively and fail earlier.
+ * This is not necessary but increases throughput.
+ * FIXME: Is this really intelligent?
+ */
+ current->flags &= ~PF_MEMALLOC;
+
+ if (base_bio)
+ bio = bio_clone(base_bio, GFP_NOIO);
+ else
+ bio = bio_alloc(GFP_NOIO, nr_iovecs);
+ if (!bio) {
+ if (flags & PF_MEMALLOC)
+ current->flags |= PF_MEMALLOC;
+ return NULL;
+ }
+
+ /* if the last bio was not complete, continue where that one ended */
+ bio->bi_idx = *bio_vec_idx;
+ bio->bi_vcnt = *bio_vec_idx;
+ bio->bi_size = 0;
+ bio->bi_flags &= ~(1 << BIO_SEG_VALID);
+
+ /* bio->bi_idx pages have already been allocated */
+ size -= bio->bi_idx * PAGE_SIZE;
+
+ for(i = bio->bi_idx; i < nr_iovecs; i++) {
+ struct bio_vec *bv = bio_iovec_idx(bio, i);
+
+ bv->bv_page = mempool_alloc(cc->page_pool, gfp_mask);
+ if (!bv->bv_page)
+ break;
+
+ /*
+ * if additional pages cannot be allocated without waiting,
+ * return a partially allocated bio, the caller will then try
+ * to allocate additional bios while submitting this partial bio
+ */
+ if ((i - bio->bi_idx) == (MIN_BIO_PAGES - 1))
+ gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT;
+
+ bv->bv_offset = 0;
+ if (size > PAGE_SIZE)
+ bv->bv_len = PAGE_SIZE;
+ else
+ bv->bv_len = size;
+
+ bio->bi_size += bv->bv_len;
+ bio->bi_vcnt++;
+ size -= bv->bv_len;
+ }
+
+ if (flags & PF_MEMALLOC)
+ current->flags |= PF_MEMALLOC;
+
+ if (!bio->bi_size) {
+ bio_put(bio);
+ return NULL;
+ }
+
+ /*
+ * Remember the last bio_vec allocated to be able
+ * to correctly continue after the splitting.
+ */
+ *bio_vec_idx = bio->bi_vcnt;
+
+ return bio;
+}
+
+static void crypt_free_buffer_pages(struct crypt_config *cc,
+ struct bio *bio, unsigned int bytes)
+{
+ unsigned int i, start, end;
+ struct bio_vec *bv;
+
+ /*
+ * This is ugly, but Jens Axboe thinks that using bi_idx in the
+ * endio function is too dangerous at the moment, so I calculate the
+ * correct position using bi_vcnt and bi_size.
+ * The bv_offset and bv_len fields might already be modified but we
+ * know that we always allocated whole pages.
+ * A fix to the bi_idx issue in the kernel is in the works, so
+ * we will hopefully be able to revert to the cleaner solution soon.
+ */
+ i = bio->bi_vcnt - 1;
+ bv = bio_iovec_idx(bio, i);
+ end = (i << PAGE_SHIFT) + (bv->bv_offset + bv->bv_len) - bio->bi_size;
+ start = end - bytes;
+
+ start >>= PAGE_SHIFT;
+ if (!bio->bi_size)
+ end = bio->bi_vcnt;
+ else
+ end >>= PAGE_SHIFT;
+
+ for(i = start; i < end; i++) {
+ bv = bio_iovec_idx(bio, i);
+ BUG_ON(!bv->bv_page);
+ mempool_free(bv->bv_page, cc->page_pool);
+ bv->bv_page = NULL;
+ }
+}
+
+/*
+ * One of the bios was finished. Check for completion of
+ * the whole request and correctly clean up the buffer.
+ */
+static void dec_pending(struct crypt_io *io, int error)
+{
+ struct crypt_config *cc = (struct crypt_config *) io->target->private;
+
+ if (error < 0)
+ io->error = error;
+
+ if (!atomic_dec_and_test(&io->pending))
+ return;
+
+ if (io->first_clone)
+ bio_put(io->first_clone);
+
+ bio_endio(io->bio, io->bio->bi_size, io->error);
+
+ mempool_free(io, cc->io_pool);
+}
+
+/*
+ * kcryptd:
+ *
+ * Needed because it would be very unwise to do decryption in an
+ * interrupt context, so bios returning from read requests get
+ * queued here.
+ */
+static struct workqueue_struct *_kcryptd_workqueue;
+
+static void kcryptd_do_work(void *data)
+{
+ struct crypt_io *io = (struct crypt_io *) data;
+ struct crypt_config *cc = (struct crypt_config *) io->target->private;
+ struct convert_context ctx;
+ int r;
+
+ crypt_convert_init(cc, &ctx, io->bio, io->bio,
+ io->bio->bi_sector - io->target->begin, 0);
+ r = crypt_convert(cc, &ctx);
+
+ dec_pending(io, r);
+}
+
+static void kcryptd_queue_io(struct crypt_io *io)
+{
+ INIT_WORK(&io->work, kcryptd_do_work, io);
+ queue_work(_kcryptd_workqueue, &io->work);
+}
+
+/*
+ * Decode key from its hex representation
+ */
+static int crypt_decode_key(u8 *key, char *hex, unsigned int size)
+{
+ char buffer[3];
+ char *endp;
+ unsigned int i;
+
+ buffer[2] = '\0';
+
+ for(i = 0; i < size; i++) {
+ buffer[0] = *hex++;
+ buffer[1] = *hex++;
+
+ key[i] = (u8)simple_strtoul(buffer, &endp, 16);
+
+ if (endp != &buffer[2])
+ return -EINVAL;
+ }
+
+ if (*hex != '\0')
+ return -EINVAL;
+
+ return 0;
+}
+
+/*
+ * Encode key into its hex representation
+ */
+static void crypt_encode_key(char *hex, u8 *key, unsigned int size)
+{
+ unsigned int i;
+
+ for(i = 0; i < size; i++) {
+ sprintf(hex, "%02x", *key);
+ hex += 2;
+ key++;
+ }
+}
+
+/*
+ * Construct an encryption mapping:
+ * <cipher> <key> <iv_offset> <dev_path> <start>
+ */
+static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+ struct crypt_config *cc;
+ struct crypto_tfm *tfm;
+ char *tmp;
+ char *cipher;
+ char *chainmode;
+ char *ivmode;
+ char *ivopts;
+ unsigned int crypto_flags;
+ unsigned int key_size;
+
+ if (argc != 5) {
+ ti->error = PFX "Not enough arguments";
+ return -EINVAL;
+ }
+
+ tmp = argv[0];
+ cipher = strsep(&tmp, "-");
+ chainmode = strsep(&tmp, "-");
+ ivopts = strsep(&tmp, "-");
+ ivmode = strsep(&ivopts, ":");
+
+ if (tmp)
+ DMWARN(PFX "Unexpected additional cipher options");
+
+ key_size = strlen(argv[1]) >> 1;
+
+ cc = kmalloc(sizeof(*cc) + key_size * sizeof(u8), GFP_KERNEL);
+ if (cc == NULL) {
+ ti->error =
+ PFX "Cannot allocate transparent encryption context";
+ return -ENOMEM;
+ }
+
+ cc->key_size = key_size;
+ if ((!key_size && strcmp(argv[1], "-") != 0) ||
+ (key_size && crypt_decode_key(cc->key, argv[1], key_size) < 0)) {
+ ti->error = PFX "Error decoding key";
+ goto bad1;
+ }
+
+ /* Compatiblity mode for old dm-crypt cipher strings */
+ if (!chainmode || (strcmp(chainmode, "plain") == 0 && !ivmode)) {
+ chainmode = "cbc";
+ ivmode = "plain";
+ }
+
+ /* Choose crypto_flags according to chainmode */
+ if (strcmp(chainmode, "cbc") == 0)
+ crypto_flags = CRYPTO_TFM_MODE_CBC;
+ else if (strcmp(chainmode, "ecb") == 0)
+ crypto_flags = CRYPTO_TFM_MODE_ECB;
+ else {
+ ti->error = PFX "Unknown chaining mode";
+ goto bad1;
+ }
+
+ if (crypto_flags != CRYPTO_TFM_MODE_ECB && !ivmode) {
+ ti->error = PFX "This chaining mode requires an IV mechanism";
+ goto bad1;
+ }
+
+ tfm = crypto_alloc_tfm(cipher, crypto_flags);
+ if (!tfm) {
+ ti->error = PFX "Error allocating crypto tfm";
+ goto bad1;
+ }
+ if (crypto_tfm_alg_type(tfm) != CRYPTO_ALG_TYPE_CIPHER) {
+ ti->error = PFX "Expected cipher algorithm";
+ goto bad2;
+ }
+
+ cc->tfm = tfm;
+
+ /*
+ * Choose ivmode. Valid modes: "plain", "essiv:<esshash>".
+ * See comments at iv code
+ */
+
+ if (ivmode == NULL)
+ cc->iv_gen_ops = NULL;
+ else if (strcmp(ivmode, "plain") == 0)
+ cc->iv_gen_ops = &crypt_iv_plain_ops;
+ else if (strcmp(ivmode, "essiv") == 0)
+ cc->iv_gen_ops = &crypt_iv_essiv_ops;
+ else {
+ ti->error = PFX "Invalid IV mode";
+ goto bad2;
+ }
+
+ if (cc->iv_gen_ops && cc->iv_gen_ops->ctr &&
+ cc->iv_gen_ops->ctr(cc, ti, ivopts) < 0)
+ goto bad2;
+
+ if (tfm->crt_cipher.cit_decrypt_iv && tfm->crt_cipher.cit_encrypt_iv)
+ /* at least a 64 bit sector number should fit in our buffer */
+ cc->iv_size = max(crypto_tfm_alg_ivsize(tfm),
+ (unsigned int)(sizeof(u64) / sizeof(u8)));
+ else {
+ cc->iv_size = 0;
+ if (cc->iv_gen_ops) {
+ DMWARN(PFX "Selected cipher does not support IVs");
+ if (cc->iv_gen_ops->dtr)
+ cc->iv_gen_ops->dtr(cc);
+ cc->iv_gen_ops = NULL;
+ }
+ }
+
+ cc->io_pool = mempool_create(MIN_IOS, mempool_alloc_slab,
+ mempool_free_slab, _crypt_io_pool);
+ if (!cc->io_pool) {
+ ti->error = PFX "Cannot allocate crypt io mempool";
+ goto bad3;
+ }
+
+ cc->page_pool = mempool_create(MIN_POOL_PAGES, mempool_alloc_page,
+ mempool_free_page, NULL);
+ if (!cc->page_pool) {
+ ti->error = PFX "Cannot allocate page mempool";
+ goto bad4;
+ }
+
+ if (tfm->crt_cipher.cit_setkey(tfm, cc->key, key_size) < 0) {
+ ti->error = PFX "Error setting key";
+ goto bad5;
+ }
+
+ if (sscanf(argv[2], SECTOR_FORMAT, &cc->iv_offset) != 1) {
+ ti->error = PFX "Invalid iv_offset sector";
+ goto bad5;
+ }
+
+ if (sscanf(argv[4], SECTOR_FORMAT, &cc->start) != 1) {
+ ti->error = PFX "Invalid device sector";
+ goto bad5;
+ }
+
+ if (dm_get_device(ti, argv[3], cc->start, ti->len,
+ dm_table_get_mode(ti->table), &cc->dev)) {
+ ti->error = PFX "Device lookup failed";
+ goto bad5;
+ }
+
+ if (ivmode && cc->iv_gen_ops) {
+ if (ivopts)
+ *(ivopts - 1) = ':';
+ cc->iv_mode = kmalloc(strlen(ivmode) + 1, GFP_KERNEL);
+ if (!cc->iv_mode) {
+ ti->error = PFX "Error kmallocing iv_mode string";
+ goto bad5;
+ }
+ strcpy(cc->iv_mode, ivmode);
+ } else
+ cc->iv_mode = NULL;
+
+ ti->private = cc;
+ return 0;
+
+bad5:
+ mempool_destroy(cc->page_pool);
+bad4:
+ mempool_destroy(cc->io_pool);
+bad3:
+ if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
+ cc->iv_gen_ops->dtr(cc);
+bad2:
+ crypto_free_tfm(tfm);
+bad1:
+ kfree(cc);
+ return -EINVAL;
+}
+
+static void crypt_dtr(struct dm_target *ti)
+{
+ struct crypt_config *cc = (struct crypt_config *) ti->private;
+
+ mempool_destroy(cc->page_pool);
+ mempool_destroy(cc->io_pool);
+
+ if (cc->iv_mode)
+ kfree(cc->iv_mode);
+ if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
+ cc->iv_gen_ops->dtr(cc);
+ crypto_free_tfm(cc->tfm);
+ dm_put_device(ti, cc->dev);
+ kfree(cc);
+}
+
+static int crypt_endio(struct bio *bio, unsigned int done, int error)
+{
+ struct crypt_io *io = (struct crypt_io *) bio->bi_private;
+ struct crypt_config *cc = (struct crypt_config *) io->target->private;
+
+ if (bio_data_dir(bio) == WRITE) {
+ /*
+ * free the processed pages, even if
+ * it's only a partially completed write
+ */
+ crypt_free_buffer_pages(cc, bio, done);
+ }
+
+ if (bio->bi_size)
+ return 1;
+
+ bio_put(bio);
+
+ /*
+ * successful reads are decrypted by the worker thread
+ */
+ if ((bio_data_dir(bio) == READ)
+ && bio_flagged(bio, BIO_UPTODATE)) {
+ kcryptd_queue_io(io);
+ return 0;
+ }
+
+ dec_pending(io, error);
+ return error;
+}
+
+static inline struct bio *
+crypt_clone(struct crypt_config *cc, struct crypt_io *io, struct bio *bio,
+ sector_t sector, unsigned int *bvec_idx,
+ struct convert_context *ctx)
+{
+ struct bio *clone;
+
+ if (bio_data_dir(bio) == WRITE) {
+ clone = crypt_alloc_buffer(cc, bio->bi_size,
+ io->first_clone, bvec_idx);
+ if (clone) {
+ ctx->bio_out = clone;
+ if (crypt_convert(cc, ctx) < 0) {
+ crypt_free_buffer_pages(cc, clone,
+ clone->bi_size);
+ bio_put(clone);
+ return NULL;
+ }
+ }
+ } else {
+ /*
+ * The block layer might modify the bvec array, so always
+ * copy the required bvecs because we need the original
+ * one in order to decrypt the whole bio data *afterwards*.
+ */
+ clone = bio_alloc(GFP_NOIO, bio_segments(bio));
+ if (clone) {
+ clone->bi_idx = 0;
+ clone->bi_vcnt = bio_segments(bio);
+ clone->bi_size = bio->bi_size;
+ memcpy(clone->bi_io_vec, bio_iovec(bio),
+ sizeof(struct bio_vec) * clone->bi_vcnt);
+ }
+ }
+
+ if (!clone)
+ return NULL;
+
+ clone->bi_private = io;
+ clone->bi_end_io = crypt_endio;
+ clone->bi_bdev = cc->dev->bdev;
+ clone->bi_sector = cc->start + sector;
+ clone->bi_rw = bio->bi_rw;
+
+ return clone;
+}
+
+static int crypt_map(struct dm_target *ti, struct bio *bio,
+ union map_info *map_context)
+{
+ struct crypt_config *cc = (struct crypt_config *) ti->private;
+ struct crypt_io *io = mempool_alloc(cc->io_pool, GFP_NOIO);
+ struct convert_context ctx;
+ struct bio *clone;
+ unsigned int remaining = bio->bi_size;
+ sector_t sector = bio->bi_sector - ti->begin;
+ unsigned int bvec_idx = 0;
+
+ io->target = ti;
+ io->bio = bio;
+ io->first_clone = NULL;
+ io->error = 0;
+ atomic_set(&io->pending, 1); /* hold a reference */
+
+ if (bio_data_dir(bio) == WRITE)
+ crypt_convert_init(cc, &ctx, NULL, bio, sector, 1);
+
+ /*
+ * The allocated buffers can be smaller than the whole bio,
+ * so repeat the whole process until all the data can be handled.
+ */
+ while (remaining) {
+ clone = crypt_clone(cc, io, bio, sector, &bvec_idx, &ctx);
+ if (!clone)
+ goto cleanup;
+
+ if (!io->first_clone) {
+ /*
+ * hold a reference to the first clone, because it
+ * holds the bio_vec array and that can't be freed
+ * before all other clones are released
+ */
+ bio_get(clone);
+ io->first_clone = clone;
+ }
+ atomic_inc(&io->pending);
+
+ remaining -= clone->bi_size;
+ sector += bio_sectors(clone);
+
+ generic_make_request(clone);
+
+ /* out of memory -> run queues */
+ if (remaining)
+ blk_congestion_wait(bio_data_dir(clone), HZ/100);
+ }
+
+ /* drop reference, clones could have returned before we reach this */
+ dec_pending(io, 0);
+ return 0;
+
+cleanup:
+ if (io->first_clone) {
+ dec_pending(io, -ENOMEM);
+ return 0;
+ }
+
+ /* if no bio has been dispatched yet, we can directly return the error */
+ mempool_free(io, cc->io_pool);
+ return -ENOMEM;
+}
+
+static int crypt_status(struct dm_target *ti, status_type_t type,
+ char *result, unsigned int maxlen)
+{
+ struct crypt_config *cc = (struct crypt_config *) ti->private;
+ const char *cipher;
+ const char *chainmode = NULL;
+ unsigned int sz = 0;
+
+ switch (type) {
+ case STATUSTYPE_INFO:
+ result[0] = '\0';
+ break;
+
+ case STATUSTYPE_TABLE:
+ cipher = crypto_tfm_alg_name(cc->tfm);
+
+ switch(cc->tfm->crt_cipher.cit_mode) {
+ case CRYPTO_TFM_MODE_CBC:
+ chainmode = "cbc";
+ break;
+ case CRYPTO_TFM_MODE_ECB:
+ chainmode = "ecb";
+ break;
+ default:
+ BUG();
+ }
+
+ if (cc->iv_mode)
+ DMEMIT("%s-%s-%s ", cipher, chainmode, cc->iv_mode);
+ else
+ DMEMIT("%s-%s ", cipher, chainmode);
+
+ if (cc->key_size > 0) {
+ if ((maxlen - sz) < ((cc->key_size << 1) + 1))
+ return -ENOMEM;
+
+ crypt_encode_key(result + sz, cc->key, cc->key_size);
+ sz += cc->key_size << 1;
+ } else {
+ if (sz >= maxlen)
+ return -ENOMEM;
+ result[sz++] = '-';
+ }
+
+ DMEMIT(" " SECTOR_FORMAT " %s " SECTOR_FORMAT,
+ cc->iv_offset, cc->dev->name, cc->start);
+ break;
+ }
+ return 0;
+}
+
+static struct target_type crypt_target = {
+ .name = "crypt",
+ .version= {1, 1, 0},
+ .module = THIS_MODULE,
+ .ctr = crypt_ctr,
+ .dtr = crypt_dtr,
+ .map = crypt_map,
+ .status = crypt_status,
+};
+
+static int __init dm_crypt_init(void)
+{
+ int r;
+
+ _crypt_io_pool = kmem_cache_create("dm-crypt_io",
+ sizeof(struct crypt_io),
+ 0, 0, NULL, NULL);
+ if (!_crypt_io_pool)
+ return -ENOMEM;
+
+ _kcryptd_workqueue = create_workqueue("kcryptd");
+ if (!_kcryptd_workqueue) {
+ r = -ENOMEM;
+ DMERR(PFX "couldn't create kcryptd");
+ goto bad1;
+ }
+
+ r = dm_register_target(&crypt_target);
+ if (r < 0) {
+ DMERR(PFX "register failed %d", r);
+ goto bad2;
+ }
+
+ return 0;
+
+bad2:
+ destroy_workqueue(_kcryptd_workqueue);
+bad1:
+ kmem_cache_destroy(_crypt_io_pool);
+ return r;
+}
+
+static void __exit dm_crypt_exit(void)
+{
+ int r = dm_unregister_target(&crypt_target);
+
+ if (r < 0)
+ DMERR(PFX "unregister failed %d", r);
+
+ destroy_workqueue(_kcryptd_workqueue);
+ kmem_cache_destroy(_crypt_io_pool);
+}
+
+module_init(dm_crypt_init);
+module_exit(dm_crypt_exit);
+
+MODULE_AUTHOR("Christophe Saout <christophe@saout.de>");
+MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-emc.c b/drivers/md/dm-emc.c
new file mode 100644
index 000000000000..700658664594
--- /dev/null
+++ b/drivers/md/dm-emc.c
@@ -0,0 +1,359 @@
+/*
+ * Copyright (C) 2004 SUSE LINUX Products GmbH. All rights reserved.
+ * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the GPL.
+ *
+ * Multipath support for EMC CLARiiON AX/CX-series hardware.
+ */
+
+#include "dm.h"
+#include "dm-hw-handler.h"
+#include <scsi/scsi.h>
+#include <scsi/scsi_cmnd.h>
+
+struct emc_handler {
+ spinlock_t lock;
+
+ /* Whether we should send the short trespass command (FC-series)
+ * or the long version (default for AX/CX CLARiiON arrays). */
+ unsigned short_trespass;
+ /* Whether or not to honor SCSI reservations when initiating a
+ * switch-over. Default: Don't. */
+ unsigned hr;
+
+ unsigned char sense[SCSI_SENSE_BUFFERSIZE];
+};
+
+#define TRESPASS_PAGE 0x22
+#define EMC_FAILOVER_TIMEOUT (60 * HZ)
+
+/* Code borrowed from dm-lsi-rdac by Mike Christie */
+
+static inline void free_bio(struct bio *bio)
+{
+ __free_page(bio->bi_io_vec[0].bv_page);
+ bio_put(bio);
+}
+
+static int emc_endio(struct bio *bio, unsigned int bytes_done, int error)
+{
+ struct path *path = bio->bi_private;
+
+ if (bio->bi_size)
+ return 1;
+
+ /* We also need to look at the sense keys here whether or not to
+ * switch to the next PG etc.
+ *
+ * For now simple logic: either it works or it doesn't.
+ */
+ if (error)
+ dm_pg_init_complete(path, MP_FAIL_PATH);
+ else
+ dm_pg_init_complete(path, 0);
+
+ /* request is freed in block layer */
+ free_bio(bio);
+
+ return 0;
+}
+
+static struct bio *get_failover_bio(struct path *path, unsigned data_size)
+{
+ struct bio *bio;
+ struct page *page;
+
+ bio = bio_alloc(GFP_ATOMIC, 1);
+ if (!bio) {
+ DMERR("dm-emc: get_failover_bio: bio_alloc() failed.");
+ return NULL;
+ }
+
+ bio->bi_rw |= (1 << BIO_RW);
+ bio->bi_bdev = path->dev->bdev;
+ bio->bi_sector = 0;
+ bio->bi_private = path;
+ bio->bi_end_io = emc_endio;
+
+ page = alloc_page(GFP_ATOMIC);
+ if (!page) {
+ DMERR("dm-emc: get_failover_bio: alloc_page() failed.");
+ bio_put(bio);
+ return NULL;
+ }
+
+ if (bio_add_page(bio, page, data_size, 0) != data_size) {
+ DMERR("dm-emc: get_failover_bio: alloc_page() failed.");
+ __free_page(page);
+ bio_put(bio);
+ return NULL;
+ }
+
+ return bio;
+}
+
+static struct request *get_failover_req(struct emc_handler *h,
+ struct bio *bio, struct path *path)
+{
+ struct request *rq;
+ struct block_device *bdev = bio->bi_bdev;
+ struct request_queue *q = bdev_get_queue(bdev);
+
+ /* FIXME: Figure out why it fails with GFP_ATOMIC. */
+ rq = blk_get_request(q, WRITE, __GFP_WAIT);
+ if (!rq) {
+ DMERR("dm-emc: get_failover_req: blk_get_request failed");
+ return NULL;
+ }
+
+ rq->bio = rq->biotail = bio;
+ blk_rq_bio_prep(q, rq, bio);
+
+ rq->rq_disk = bdev->bd_contains->bd_disk;
+
+ /* bio backed don't set data */
+ rq->buffer = rq->data = NULL;
+ /* rq data_len used for pc cmd's request_bufflen */
+ rq->data_len = bio->bi_size;
+
+ rq->sense = h->sense;
+ memset(rq->sense, 0, SCSI_SENSE_BUFFERSIZE);
+ rq->sense_len = 0;
+
+ memset(&rq->cmd, 0, BLK_MAX_CDB);
+
+ rq->timeout = EMC_FAILOVER_TIMEOUT;
+ rq->flags |= (REQ_BLOCK_PC | REQ_FAILFAST | REQ_NOMERGE);
+
+ return rq;
+}
+
+static struct request *emc_trespass_get(struct emc_handler *h,
+ struct path *path)
+{
+ struct bio *bio;
+ struct request *rq;
+ unsigned char *page22;
+ unsigned char long_trespass_pg[] = {
+ 0, 0, 0, 0,
+ TRESPASS_PAGE, /* Page code */
+ 0x09, /* Page length - 2 */
+ h->hr ? 0x01 : 0x81, /* Trespass code + Honor reservation bit */
+ 0xff, 0xff, /* Trespass target */
+ 0, 0, 0, 0, 0, 0 /* Reserved bytes / unknown */
+ };
+ unsigned char short_trespass_pg[] = {
+ 0, 0, 0, 0,
+ TRESPASS_PAGE, /* Page code */
+ 0x02, /* Page length - 2 */
+ h->hr ? 0x01 : 0x81, /* Trespass code + Honor reservation bit */
+ 0xff, /* Trespass target */
+ };
+ unsigned data_size = h->short_trespass ? sizeof(short_trespass_pg) :
+ sizeof(long_trespass_pg);
+
+ /* get bio backing */
+ if (data_size > PAGE_SIZE)
+ /* this should never happen */
+ return NULL;
+
+ bio = get_failover_bio(path, data_size);
+ if (!bio) {
+ DMERR("dm-emc: emc_trespass_get: no bio");
+ return NULL;
+ }
+
+ page22 = (unsigned char *)bio_data(bio);
+ memset(page22, 0, data_size);
+
+ memcpy(page22, h->short_trespass ?
+ short_trespass_pg : long_trespass_pg, data_size);
+
+ /* get request for block layer packet command */
+ rq = get_failover_req(h, bio, path);
+ if (!rq) {
+ DMERR("dm-emc: emc_trespass_get: no rq");
+ free_bio(bio);
+ return NULL;
+ }
+
+ /* Prepare the command. */
+ rq->cmd[0] = MODE_SELECT;
+ rq->cmd[1] = 0x10;
+ rq->cmd[4] = data_size;
+ rq->cmd_len = COMMAND_SIZE(rq->cmd[0]);
+
+ return rq;
+}
+
+static void emc_pg_init(struct hw_handler *hwh, unsigned bypassed,
+ struct path *path)
+{
+ struct request *rq;
+ struct request_queue *q = bdev_get_queue(path->dev->bdev);
+
+ /*
+ * We can either blindly init the pg (then look at the sense),
+ * or we can send some commands to get the state here (then
+ * possibly send the fo cmnd), or we can also have the
+ * initial state passed into us and then get an update here.
+ */
+ if (!q) {
+ DMINFO("dm-emc: emc_pg_init: no queue");
+ goto fail_path;
+ }
+
+ /* FIXME: The request should be pre-allocated. */
+ rq = emc_trespass_get(hwh->context, path);
+ if (!rq) {
+ DMERR("dm-emc: emc_pg_init: no rq");
+ goto fail_path;
+ }
+
+ DMINFO("dm-emc: emc_pg_init: sending switch-over command");
+ elv_add_request(q, rq, ELEVATOR_INSERT_FRONT, 1);
+ return;
+
+fail_path:
+ dm_pg_init_complete(path, MP_FAIL_PATH);
+}
+
+static struct emc_handler *alloc_emc_handler(void)
+{
+ struct emc_handler *h = kmalloc(sizeof(*h), GFP_KERNEL);
+
+ if (h)
+ spin_lock_init(&h->lock);
+
+ return h;
+}
+
+static int emc_create(struct hw_handler *hwh, unsigned argc, char **argv)
+{
+ struct emc_handler *h;
+ unsigned hr, short_trespass;
+
+ if (argc == 0) {
+ /* No arguments: use defaults */
+ hr = 0;
+ short_trespass = 0;
+ } else if (argc != 2) {
+ DMWARN("dm-emc hwhandler: incorrect number of arguments");
+ return -EINVAL;
+ } else {
+ if ((sscanf(argv[0], "%u", &short_trespass) != 1)
+ || (short_trespass > 1)) {
+ DMWARN("dm-emc: invalid trespass mode selected");
+ return -EINVAL;
+ }
+
+ if ((sscanf(argv[1], "%u", &hr) != 1)
+ || (hr > 1)) {
+ DMWARN("dm-emc: invalid honor reservation flag selected");
+ return -EINVAL;
+ }
+ }
+
+ h = alloc_emc_handler();
+ if (!h)
+ return -ENOMEM;
+
+ memset(h, 0, sizeof(*h));
+
+ hwh->context = h;
+
+ if ((h->short_trespass = short_trespass))
+ DMWARN("dm-emc: short trespass command will be send");
+ else
+ DMWARN("dm-emc: long trespass command will be send");
+
+ if ((h->hr = hr))
+ DMWARN("dm-emc: honor reservation bit will be set");
+ else
+ DMWARN("dm-emc: honor reservation bit will not be set (default)");
+
+ return 0;
+}
+
+static void emc_destroy(struct hw_handler *hwh)
+{
+ struct emc_handler *h = (struct emc_handler *) hwh->context;
+
+ kfree(h);
+ hwh->context = NULL;
+}
+
+static unsigned emc_error(struct hw_handler *hwh, struct bio *bio)
+{
+ /* FIXME: Patch from axboe still missing */
+#if 0
+ int sense;
+
+ if (bio->bi_error & BIO_SENSE) {
+ sense = bio->bi_error & 0xffffff; /* sense key / asc / ascq */
+
+ if (sense == 0x020403) {
+ /* LUN Not Ready - Manual Intervention Required
+ * indicates this is a passive path.
+ *
+ * FIXME: However, if this is seen and EVPD C0
+ * indicates that this is due to a NDU in
+ * progress, we should set FAIL_PATH too.
+ * This indicates we might have to do a SCSI
+ * inquiry in the end_io path. Ugh. */
+ return MP_BYPASS_PG | MP_RETRY_IO;
+ } else if (sense == 0x052501) {
+ /* An array based copy is in progress. Do not
+ * fail the path, do not bypass to another PG,
+ * do not retry. Fail the IO immediately.
+ * (Actually this is the same conclusion as in
+ * the default handler, but lets make sure.) */
+ return 0;
+ } else if (sense == 0x062900) {
+ /* Unit Attention Code. This is the first IO
+ * to the new path, so just retry. */
+ return MP_RETRY_IO;
+ }
+ }
+#endif
+
+ /* Try default handler */
+ return dm_scsi_err_handler(hwh, bio);
+}
+
+static struct hw_handler_type emc_hwh = {
+ .name = "emc",
+ .module = THIS_MODULE,
+ .create = emc_create,
+ .destroy = emc_destroy,
+ .pg_init = emc_pg_init,
+ .error = emc_error,
+};
+
+static int __init dm_emc_init(void)
+{
+ int r = dm_register_hw_handler(&emc_hwh);
+
+ if (r < 0)
+ DMERR("emc: register failed %d", r);
+
+ DMINFO("dm-emc version 0.0.3 loaded");
+
+ return r;
+}
+
+static void __exit dm_emc_exit(void)
+{
+ int r = dm_unregister_hw_handler(&emc_hwh);
+
+ if (r < 0)
+ DMERR("emc: unregister failed %d", r);
+}
+
+module_init(dm_emc_init);
+module_exit(dm_emc_exit);
+
+MODULE_DESCRIPTION(DM_NAME " EMC CX/AX/FC-family multipath");
+MODULE_AUTHOR("Lars Marowsky-Bree <lmb@suse.de>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-exception-store.c b/drivers/md/dm-exception-store.c
new file mode 100644
index 000000000000..17212b4201a1
--- /dev/null
+++ b/drivers/md/dm-exception-store.c
@@ -0,0 +1,648 @@
+/*
+ * dm-snapshot.c
+ *
+ * Copyright (C) 2001-2002 Sistina Software (UK) Limited.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+#include "dm-snap.h"
+#include "dm-io.h"
+#include "kcopyd.h"
+
+#include <linux/mm.h>
+#include <linux/pagemap.h>
+#include <linux/vmalloc.h>
+#include <linux/slab.h>
+
+/*-----------------------------------------------------------------
+ * Persistent snapshots, by persistent we mean that the snapshot
+ * will survive a reboot.
+ *---------------------------------------------------------------*/
+
+/*
+ * We need to store a record of which parts of the origin have
+ * been copied to the snapshot device. The snapshot code
+ * requires that we copy exception chunks to chunk aligned areas
+ * of the COW store. It makes sense therefore, to store the
+ * metadata in chunk size blocks.
+ *
+ * There is no backward or forward compatibility implemented,
+ * snapshots with different disk versions than the kernel will
+ * not be usable. It is expected that "lvcreate" will blank out
+ * the start of a fresh COW device before calling the snapshot
+ * constructor.
+ *
+ * The first chunk of the COW device just contains the header.
+ * After this there is a chunk filled with exception metadata,
+ * followed by as many exception chunks as can fit in the
+ * metadata areas.
+ *
+ * All on disk structures are in little-endian format. The end
+ * of the exceptions info is indicated by an exception with a
+ * new_chunk of 0, which is invalid since it would point to the
+ * header chunk.
+ */
+
+/*
+ * Magic for persistent snapshots: "SnAp" - Feeble isn't it.
+ */
+#define SNAP_MAGIC 0x70416e53
+
+/*
+ * The on-disk version of the metadata.
+ */
+#define SNAPSHOT_DISK_VERSION 1
+
+struct disk_header {
+ uint32_t magic;
+
+ /*
+ * Is this snapshot valid. There is no way of recovering
+ * an invalid snapshot.
+ */
+ uint32_t valid;
+
+ /*
+ * Simple, incrementing version. no backward
+ * compatibility.
+ */
+ uint32_t version;
+
+ /* In sectors */
+ uint32_t chunk_size;
+};
+
+struct disk_exception {
+ uint64_t old_chunk;
+ uint64_t new_chunk;
+};
+
+struct commit_callback {
+ void (*callback)(void *, int success);
+ void *context;
+};
+
+/*
+ * The top level structure for a persistent exception store.
+ */
+struct pstore {
+ struct dm_snapshot *snap; /* up pointer to my snapshot */
+ int version;
+ int valid;
+ uint32_t chunk_size;
+ uint32_t exceptions_per_area;
+
+ /*
+ * Now that we have an asynchronous kcopyd there is no
+ * need for large chunk sizes, so it wont hurt to have a
+ * whole chunks worth of metadata in memory at once.
+ */
+ void *area;
+
+ /*
+ * Used to keep track of which metadata area the data in
+ * 'chunk' refers to.
+ */
+ uint32_t current_area;
+
+ /*
+ * The next free chunk for an exception.
+ */
+ uint32_t next_free;
+
+ /*
+ * The index of next free exception in the current
+ * metadata area.
+ */
+ uint32_t current_committed;
+
+ atomic_t pending_count;
+ uint32_t callback_count;
+ struct commit_callback *callbacks;
+};
+
+static inline unsigned int sectors_to_pages(unsigned int sectors)
+{
+ return sectors / (PAGE_SIZE >> 9);
+}
+
+static int alloc_area(struct pstore *ps)
+{
+ int r = -ENOMEM;
+ size_t len;
+
+ len = ps->chunk_size << SECTOR_SHIFT;
+
+ /*
+ * Allocate the chunk_size block of memory that will hold
+ * a single metadata area.
+ */
+ ps->area = vmalloc(len);
+ if (!ps->area)
+ return r;
+
+ return 0;
+}
+
+static void free_area(struct pstore *ps)
+{
+ vfree(ps->area);
+}
+
+/*
+ * Read or write a chunk aligned and sized block of data from a device.
+ */
+static int chunk_io(struct pstore *ps, uint32_t chunk, int rw)
+{
+ struct io_region where;
+ unsigned long bits;
+
+ where.bdev = ps->snap->cow->bdev;
+ where.sector = ps->chunk_size * chunk;
+ where.count = ps->chunk_size;
+
+ return dm_io_sync_vm(1, &where, rw, ps->area, &bits);
+}
+
+/*
+ * Read or write a metadata area. Remembering to skip the first
+ * chunk which holds the header.
+ */
+static int area_io(struct pstore *ps, uint32_t area, int rw)
+{
+ int r;
+ uint32_t chunk;
+
+ /* convert a metadata area index to a chunk index */
+ chunk = 1 + ((ps->exceptions_per_area + 1) * area);
+
+ r = chunk_io(ps, chunk, rw);
+ if (r)
+ return r;
+
+ ps->current_area = area;
+ return 0;
+}
+
+static int zero_area(struct pstore *ps, uint32_t area)
+{
+ memset(ps->area, 0, ps->chunk_size << SECTOR_SHIFT);
+ return area_io(ps, area, WRITE);
+}
+
+static int read_header(struct pstore *ps, int *new_snapshot)
+{
+ int r;
+ struct disk_header *dh;
+
+ r = chunk_io(ps, 0, READ);
+ if (r)
+ return r;
+
+ dh = (struct disk_header *) ps->area;
+
+ if (le32_to_cpu(dh->magic) == 0) {
+ *new_snapshot = 1;
+
+ } else if (le32_to_cpu(dh->magic) == SNAP_MAGIC) {
+ *new_snapshot = 0;
+ ps->valid = le32_to_cpu(dh->valid);
+ ps->version = le32_to_cpu(dh->version);
+ ps->chunk_size = le32_to_cpu(dh->chunk_size);
+
+ } else {
+ DMWARN("Invalid/corrupt snapshot");
+ r = -ENXIO;
+ }
+
+ return r;
+}
+
+static int write_header(struct pstore *ps)
+{
+ struct disk_header *dh;
+
+ memset(ps->area, 0, ps->chunk_size << SECTOR_SHIFT);
+
+ dh = (struct disk_header *) ps->area;
+ dh->magic = cpu_to_le32(SNAP_MAGIC);
+ dh->valid = cpu_to_le32(ps->valid);
+ dh->version = cpu_to_le32(ps->version);
+ dh->chunk_size = cpu_to_le32(ps->chunk_size);
+
+ return chunk_io(ps, 0, WRITE);
+}
+
+/*
+ * Access functions for the disk exceptions, these do the endian conversions.
+ */
+static struct disk_exception *get_exception(struct pstore *ps, uint32_t index)
+{
+ if (index >= ps->exceptions_per_area)
+ return NULL;
+
+ return ((struct disk_exception *) ps->area) + index;
+}
+
+static int read_exception(struct pstore *ps,
+ uint32_t index, struct disk_exception *result)
+{
+ struct disk_exception *e;
+
+ e = get_exception(ps, index);
+ if (!e)
+ return -EINVAL;
+
+ /* copy it */
+ result->old_chunk = le64_to_cpu(e->old_chunk);
+ result->new_chunk = le64_to_cpu(e->new_chunk);
+
+ return 0;
+}
+
+static int write_exception(struct pstore *ps,
+ uint32_t index, struct disk_exception *de)
+{
+ struct disk_exception *e;
+
+ e = get_exception(ps, index);
+ if (!e)
+ return -EINVAL;
+
+ /* copy it */
+ e->old_chunk = cpu_to_le64(de->old_chunk);
+ e->new_chunk = cpu_to_le64(de->new_chunk);
+
+ return 0;
+}
+
+/*
+ * Registers the exceptions that are present in the current area.
+ * 'full' is filled in to indicate if the area has been
+ * filled.
+ */
+static int insert_exceptions(struct pstore *ps, int *full)
+{
+ int r;
+ unsigned int i;
+ struct disk_exception de;
+
+ /* presume the area is full */
+ *full = 1;
+
+ for (i = 0; i < ps->exceptions_per_area; i++) {
+ r = read_exception(ps, i, &de);
+
+ if (r)
+ return r;
+
+ /*
+ * If the new_chunk is pointing at the start of
+ * the COW device, where the first metadata area
+ * is we know that we've hit the end of the
+ * exceptions. Therefore the area is not full.
+ */
+ if (de.new_chunk == 0LL) {
+ ps->current_committed = i;
+ *full = 0;
+ break;
+ }
+
+ /*
+ * Keep track of the start of the free chunks.
+ */
+ if (ps->next_free <= de.new_chunk)
+ ps->next_free = de.new_chunk + 1;
+
+ /*
+ * Otherwise we add the exception to the snapshot.
+ */
+ r = dm_add_exception(ps->snap, de.old_chunk, de.new_chunk);
+ if (r)
+ return r;
+ }
+
+ return 0;
+}
+
+static int read_exceptions(struct pstore *ps)
+{
+ uint32_t area;
+ int r, full = 1;
+
+ /*
+ * Keeping reading chunks and inserting exceptions until
+ * we find a partially full area.
+ */
+ for (area = 0; full; area++) {
+ r = area_io(ps, area, READ);
+ if (r)
+ return r;
+
+ r = insert_exceptions(ps, &full);
+ if (r)
+ return r;
+ }
+
+ return 0;
+}
+
+static inline struct pstore *get_info(struct exception_store *store)
+{
+ return (struct pstore *) store->context;
+}
+
+static void persistent_fraction_full(struct exception_store *store,
+ sector_t *numerator, sector_t *denominator)
+{
+ *numerator = get_info(store)->next_free * store->snap->chunk_size;
+ *denominator = get_dev_size(store->snap->cow->bdev);
+}
+
+static void persistent_destroy(struct exception_store *store)
+{
+ struct pstore *ps = get_info(store);
+
+ dm_io_put(sectors_to_pages(ps->chunk_size));
+ vfree(ps->callbacks);
+ free_area(ps);
+ kfree(ps);
+}
+
+static int persistent_read_metadata(struct exception_store *store)
+{
+ int r, new_snapshot;
+ struct pstore *ps = get_info(store);
+
+ /*
+ * Read the snapshot header.
+ */
+ r = read_header(ps, &new_snapshot);
+ if (r)
+ return r;
+
+ /*
+ * Do we need to setup a new snapshot ?
+ */
+ if (new_snapshot) {
+ r = write_header(ps);
+ if (r) {
+ DMWARN("write_header failed");
+ return r;
+ }
+
+ r = zero_area(ps, 0);
+ if (r) {
+ DMWARN("zero_area(0) failed");
+ return r;
+ }
+
+ } else {
+ /*
+ * Sanity checks.
+ */
+ if (!ps->valid) {
+ DMWARN("snapshot is marked invalid");
+ return -EINVAL;
+ }
+
+ if (ps->version != SNAPSHOT_DISK_VERSION) {
+ DMWARN("unable to handle snapshot disk version %d",
+ ps->version);
+ return -EINVAL;
+ }
+
+ /*
+ * Read the metadata.
+ */
+ r = read_exceptions(ps);
+ if (r)
+ return r;
+ }
+
+ return 0;
+}
+
+static int persistent_prepare(struct exception_store *store,
+ struct exception *e)
+{
+ struct pstore *ps = get_info(store);
+ uint32_t stride;
+ sector_t size = get_dev_size(store->snap->cow->bdev);
+
+ /* Is there enough room ? */
+ if (size < ((ps->next_free + 1) * store->snap->chunk_size))
+ return -ENOSPC;
+
+ e->new_chunk = ps->next_free;
+
+ /*
+ * Move onto the next free pending, making sure to take
+ * into account the location of the metadata chunks.
+ */
+ stride = (ps->exceptions_per_area + 1);
+ if ((++ps->next_free % stride) == 1)
+ ps->next_free++;
+
+ atomic_inc(&ps->pending_count);
+ return 0;
+}
+
+static void persistent_commit(struct exception_store *store,
+ struct exception *e,
+ void (*callback) (void *, int success),
+ void *callback_context)
+{
+ int r;
+ unsigned int i;
+ struct pstore *ps = get_info(store);
+ struct disk_exception de;
+ struct commit_callback *cb;
+
+ de.old_chunk = e->old_chunk;
+ de.new_chunk = e->new_chunk;
+ write_exception(ps, ps->current_committed++, &de);
+
+ /*
+ * Add the callback to the back of the array. This code
+ * is the only place where the callback array is
+ * manipulated, and we know that it will never be called
+ * multiple times concurrently.
+ */
+ cb = ps->callbacks + ps->callback_count++;
+ cb->callback = callback;
+ cb->context = callback_context;
+
+ /*
+ * If there are no more exceptions in flight, or we have
+ * filled this metadata area we commit the exceptions to
+ * disk.
+ */
+ if (atomic_dec_and_test(&ps->pending_count) ||
+ (ps->current_committed == ps->exceptions_per_area)) {
+ r = area_io(ps, ps->current_area, WRITE);
+ if (r)
+ ps->valid = 0;
+
+ for (i = 0; i < ps->callback_count; i++) {
+ cb = ps->callbacks + i;
+ cb->callback(cb->context, r == 0 ? 1 : 0);
+ }
+
+ ps->callback_count = 0;
+ }
+
+ /*
+ * Have we completely filled the current area ?
+ */
+ if (ps->current_committed == ps->exceptions_per_area) {
+ ps->current_committed = 0;
+ r = zero_area(ps, ps->current_area + 1);
+ if (r)
+ ps->valid = 0;
+ }
+}
+
+static void persistent_drop(struct exception_store *store)
+{
+ struct pstore *ps = get_info(store);
+
+ ps->valid = 0;
+ if (write_header(ps))
+ DMWARN("write header failed");
+}
+
+int dm_create_persistent(struct exception_store *store, uint32_t chunk_size)
+{
+ int r;
+ struct pstore *ps;
+
+ r = dm_io_get(sectors_to_pages(chunk_size));
+ if (r)
+ return r;
+
+ /* allocate the pstore */
+ ps = kmalloc(sizeof(*ps), GFP_KERNEL);
+ if (!ps) {
+ r = -ENOMEM;
+ goto bad;
+ }
+
+ ps->snap = store->snap;
+ ps->valid = 1;
+ ps->version = SNAPSHOT_DISK_VERSION;
+ ps->chunk_size = chunk_size;
+ ps->exceptions_per_area = (chunk_size << SECTOR_SHIFT) /
+ sizeof(struct disk_exception);
+ ps->next_free = 2; /* skipping the header and first area */
+ ps->current_committed = 0;
+
+ r = alloc_area(ps);
+ if (r)
+ goto bad;
+
+ /*
+ * Allocate space for all the callbacks.
+ */
+ ps->callback_count = 0;
+ atomic_set(&ps->pending_count, 0);
+ ps->callbacks = dm_vcalloc(ps->exceptions_per_area,
+ sizeof(*ps->callbacks));
+
+ if (!ps->callbacks) {
+ r = -ENOMEM;
+ goto bad;
+ }
+
+ store->destroy = persistent_destroy;
+ store->read_metadata = persistent_read_metadata;
+ store->prepare_exception = persistent_prepare;
+ store->commit_exception = persistent_commit;
+ store->drop_snapshot = persistent_drop;
+ store->fraction_full = persistent_fraction_full;
+ store->context = ps;
+
+ return 0;
+
+ bad:
+ dm_io_put(sectors_to_pages(chunk_size));
+ if (ps) {
+ if (ps->area)
+ free_area(ps);
+
+ kfree(ps);
+ }
+ return r;
+}
+
+/*-----------------------------------------------------------------
+ * Implementation of the store for non-persistent snapshots.
+ *---------------------------------------------------------------*/
+struct transient_c {
+ sector_t next_free;
+};
+
+static void transient_destroy(struct exception_store *store)
+{
+ kfree(store->context);
+}
+
+static int transient_read_metadata(struct exception_store *store)
+{
+ return 0;
+}
+
+static int transient_prepare(struct exception_store *store, struct exception *e)
+{
+ struct transient_c *tc = (struct transient_c *) store->context;
+ sector_t size = get_dev_size(store->snap->cow->bdev);
+
+ if (size < (tc->next_free + store->snap->chunk_size))
+ return -1;
+
+ e->new_chunk = sector_to_chunk(store->snap, tc->next_free);
+ tc->next_free += store->snap->chunk_size;
+
+ return 0;
+}
+
+static void transient_commit(struct exception_store *store,
+ struct exception *e,
+ void (*callback) (void *, int success),
+ void *callback_context)
+{
+ /* Just succeed */
+ callback(callback_context, 1);
+}
+
+static void transient_fraction_full(struct exception_store *store,
+ sector_t *numerator, sector_t *denominator)
+{
+ *numerator = ((struct transient_c *) store->context)->next_free;
+ *denominator = get_dev_size(store->snap->cow->bdev);
+}
+
+int dm_create_transient(struct exception_store *store,
+ struct dm_snapshot *s, int blocksize)
+{
+ struct transient_c *tc;
+
+ memset(store, 0, sizeof(*store));
+ store->destroy = transient_destroy;
+ store->read_metadata = transient_read_metadata;
+ store->prepare_exception = transient_prepare;
+ store->commit_exception = transient_commit;
+ store->fraction_full = transient_fraction_full;
+ store->snap = s;
+
+ tc = kmalloc(sizeof(struct transient_c), GFP_KERNEL);
+ if (!tc)
+ return -ENOMEM;
+
+ tc->next_free = 0;
+ store->context = tc;
+
+ return 0;
+}
diff --git a/drivers/md/dm-hw-handler.c b/drivers/md/dm-hw-handler.c
new file mode 100644
index 000000000000..ae63772e44c9
--- /dev/null
+++ b/drivers/md/dm-hw-handler.c
@@ -0,0 +1,216 @@
+/*
+ * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the GPL.
+ *
+ * Multipath hardware handler registration.
+ */
+
+#include "dm.h"
+#include "dm-hw-handler.h"
+
+#include <linux/slab.h>
+
+struct hwh_internal {
+ struct hw_handler_type hwht;
+
+ struct list_head list;
+ long use;
+};
+
+#define hwht_to_hwhi(__hwht) container_of((__hwht), struct hwh_internal, hwht)
+
+static LIST_HEAD(_hw_handlers);
+static DECLARE_RWSEM(_hwh_lock);
+
+struct hwh_internal *__find_hw_handler_type(const char *name)
+{
+ struct hwh_internal *hwhi;
+
+ list_for_each_entry(hwhi, &_hw_handlers, list) {
+ if (!strcmp(name, hwhi->hwht.name))
+ return hwhi;
+ }
+
+ return NULL;
+}
+
+static struct hwh_internal *get_hw_handler(const char *name)
+{
+ struct hwh_internal *hwhi;
+
+ down_read(&_hwh_lock);
+ hwhi = __find_hw_handler_type(name);
+ if (hwhi) {
+ if ((hwhi->use == 0) && !try_module_get(hwhi->hwht.module))
+ hwhi = NULL;
+ else
+ hwhi->use++;
+ }
+ up_read(&_hwh_lock);
+
+ return hwhi;
+}
+
+struct hw_handler_type *dm_get_hw_handler(const char *name)
+{
+ struct hwh_internal *hwhi;
+
+ if (!name)
+ return NULL;
+
+ hwhi = get_hw_handler(name);
+ if (!hwhi) {
+ request_module("dm-%s", name);
+ hwhi = get_hw_handler(name);
+ }
+
+ return hwhi ? &hwhi->hwht : NULL;
+}
+
+void dm_put_hw_handler(struct hw_handler_type *hwht)
+{
+ struct hwh_internal *hwhi;
+
+ if (!hwht)
+ return;
+
+ down_read(&_hwh_lock);
+ hwhi = __find_hw_handler_type(hwht->name);
+ if (!hwhi)
+ goto out;
+
+ if (--hwhi->use == 0)
+ module_put(hwhi->hwht.module);
+
+ if (hwhi->use < 0)
+ BUG();
+
+ out:
+ up_read(&_hwh_lock);
+}
+
+static struct hwh_internal *_alloc_hw_handler(struct hw_handler_type *hwht)
+{
+ struct hwh_internal *hwhi = kmalloc(sizeof(*hwhi), GFP_KERNEL);
+
+ if (hwhi) {
+ memset(hwhi, 0, sizeof(*hwhi));
+ hwhi->hwht = *hwht;
+ }
+
+ return hwhi;
+}
+
+int dm_register_hw_handler(struct hw_handler_type *hwht)
+{
+ int r = 0;
+ struct hwh_internal *hwhi = _alloc_hw_handler(hwht);
+
+ if (!hwhi)
+ return -ENOMEM;
+
+ down_write(&_hwh_lock);
+
+ if (__find_hw_handler_type(hwht->name)) {
+ kfree(hwhi);
+ r = -EEXIST;
+ } else
+ list_add(&hwhi->list, &_hw_handlers);
+
+ up_write(&_hwh_lock);
+
+ return r;
+}
+
+int dm_unregister_hw_handler(struct hw_handler_type *hwht)
+{
+ struct hwh_internal *hwhi;
+
+ down_write(&_hwh_lock);
+
+ hwhi = __find_hw_handler_type(hwht->name);
+ if (!hwhi) {
+ up_write(&_hwh_lock);
+ return -EINVAL;
+ }
+
+ if (hwhi->use) {
+ up_write(&_hwh_lock);
+ return -ETXTBSY;
+ }
+
+ list_del(&hwhi->list);
+
+ up_write(&_hwh_lock);
+
+ kfree(hwhi);
+
+ return 0;
+}
+
+unsigned dm_scsi_err_handler(struct hw_handler *hwh, struct bio *bio)
+{
+#if 0
+ int sense_key, asc, ascq;
+
+ if (bio->bi_error & BIO_SENSE) {
+ /* FIXME: This is just an initial guess. */
+ /* key / asc / ascq */
+ sense_key = (bio->bi_error >> 16) & 0xff;
+ asc = (bio->bi_error >> 8) & 0xff;
+ ascq = bio->bi_error & 0xff;
+
+ switch (sense_key) {
+ /* This block as a whole comes from the device.
+ * So no point retrying on another path. */
+ case 0x03: /* Medium error */
+ case 0x05: /* Illegal request */
+ case 0x07: /* Data protect */
+ case 0x08: /* Blank check */
+ case 0x0a: /* copy aborted */
+ case 0x0c: /* obsolete - no clue ;-) */
+ case 0x0d: /* volume overflow */
+ case 0x0e: /* data miscompare */
+ case 0x0f: /* reserved - no idea either. */
+ return MP_ERROR_IO;
+
+ /* For these errors it's unclear whether they
+ * come from the device or the controller.
+ * So just lets try a different path, and if
+ * it eventually succeeds, user-space will clear
+ * the paths again... */
+ case 0x02: /* Not ready */
+ case 0x04: /* Hardware error */
+ case 0x09: /* vendor specific */
+ case 0x0b: /* Aborted command */
+ return MP_FAIL_PATH;
+
+ case 0x06: /* Unit attention - might want to decode */
+ if (asc == 0x04 && ascq == 0x01)
+ /* "Unit in the process of
+ * becoming ready" */
+ return 0;
+ return MP_FAIL_PATH;
+
+ /* FIXME: For Unit Not Ready we may want
+ * to have a generic pg activation
+ * feature (START_UNIT). */
+
+ /* Should these two ever end up in the
+ * error path? I don't think so. */
+ case 0x00: /* No sense */
+ case 0x01: /* Recovered error */
+ return 0;
+ }
+ }
+#endif
+
+ /* We got no idea how to decode the other kinds of errors ->
+ * assume generic error condition. */
+ return MP_FAIL_PATH;
+}
+
+EXPORT_SYMBOL_GPL(dm_register_hw_handler);
+EXPORT_SYMBOL_GPL(dm_unregister_hw_handler);
+EXPORT_SYMBOL_GPL(dm_scsi_err_handler);
diff --git a/drivers/md/dm-hw-handler.h b/drivers/md/dm-hw-handler.h
new file mode 100644
index 000000000000..15f5629e231a
--- /dev/null
+++ b/drivers/md/dm-hw-handler.h
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the GPL.
+ *
+ * Multipath hardware handler registration.
+ */
+
+#ifndef DM_HW_HANDLER_H
+#define DM_HW_HANDLER_H
+
+#include <linux/device-mapper.h>
+
+#include "dm-mpath.h"
+
+struct hw_handler_type;
+struct hw_handler {
+ struct hw_handler_type *type;
+ void *context;
+};
+
+/*
+ * Constructs a hardware handler object, takes custom arguments
+ */
+/* Information about a hardware handler type */
+struct hw_handler_type {
+ char *name;
+ struct module *module;
+
+ int (*create) (struct hw_handler *handler, unsigned int argc,
+ char **argv);
+ void (*destroy) (struct hw_handler *hwh);
+
+ void (*pg_init) (struct hw_handler *hwh, unsigned bypassed,
+ struct path *path);
+ unsigned (*error) (struct hw_handler *hwh, struct bio *bio);
+ int (*status) (struct hw_handler *hwh, status_type_t type,
+ char *result, unsigned int maxlen);
+};
+
+/* Register a hardware handler */
+int dm_register_hw_handler(struct hw_handler_type *type);
+
+/* Unregister a hardware handler */
+int dm_unregister_hw_handler(struct hw_handler_type *type);
+
+/* Returns a registered hardware handler type */
+struct hw_handler_type *dm_get_hw_handler(const char *name);
+
+/* Releases a hardware handler */
+void dm_put_hw_handler(struct hw_handler_type *hwht);
+
+/* Default err function */
+unsigned dm_scsi_err_handler(struct hw_handler *hwh, struct bio *bio);
+
+/* Error flags for err and dm_pg_init_complete */
+#define MP_FAIL_PATH 1
+#define MP_BYPASS_PG 2
+#define MP_ERROR_IO 4 /* Don't retry this I/O */
+
+#endif
diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c
new file mode 100644
index 000000000000..45754bb6a799
--- /dev/null
+++ b/drivers/md/dm-io.c
@@ -0,0 +1,426 @@
+/*
+ * Copyright (C) 2003 Sistina Software
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm-io.h"
+
+#include <linux/bio.h>
+#include <linux/mempool.h>
+#include <linux/module.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+
+static struct bio_set *_bios;
+
+/* FIXME: can we shrink this ? */
+struct io {
+ unsigned long error;
+ atomic_t count;
+ struct task_struct *sleeper;
+ io_notify_fn callback;
+ void *context;
+};
+
+/*
+ * io contexts are only dynamically allocated for asynchronous
+ * io. Since async io is likely to be the majority of io we'll
+ * have the same number of io contexts as buffer heads ! (FIXME:
+ * must reduce this).
+ */
+static unsigned _num_ios;
+static mempool_t *_io_pool;
+
+static void *alloc_io(unsigned int __nocast gfp_mask, void *pool_data)
+{
+ return kmalloc(sizeof(struct io), gfp_mask);
+}
+
+static void free_io(void *element, void *pool_data)
+{
+ kfree(element);
+}
+
+static unsigned int pages_to_ios(unsigned int pages)
+{
+ return 4 * pages; /* too many ? */
+}
+
+static int resize_pool(unsigned int new_ios)
+{
+ int r = 0;
+
+ if (_io_pool) {
+ if (new_ios == 0) {
+ /* free off the pool */
+ mempool_destroy(_io_pool);
+ _io_pool = NULL;
+ bioset_free(_bios);
+
+ } else {
+ /* resize the pool */
+ r = mempool_resize(_io_pool, new_ios, GFP_KERNEL);
+ }
+
+ } else {
+ /* create new pool */
+ _io_pool = mempool_create(new_ios, alloc_io, free_io, NULL);
+ if (!_io_pool)
+ return -ENOMEM;
+
+ _bios = bioset_create(16, 16, 4);
+ if (!_bios) {
+ mempool_destroy(_io_pool);
+ _io_pool = NULL;
+ return -ENOMEM;
+ }
+ }
+
+ if (!r)
+ _num_ios = new_ios;
+
+ return r;
+}
+
+int dm_io_get(unsigned int num_pages)
+{
+ return resize_pool(_num_ios + pages_to_ios(num_pages));
+}
+
+void dm_io_put(unsigned int num_pages)
+{
+ resize_pool(_num_ios - pages_to_ios(num_pages));
+}
+
+/*-----------------------------------------------------------------
+ * We need to keep track of which region a bio is doing io for.
+ * In order to save a memory allocation we store this the last
+ * bvec which we know is unused (blech).
+ * XXX This is ugly and can OOPS with some configs... find another way.
+ *---------------------------------------------------------------*/
+static inline void bio_set_region(struct bio *bio, unsigned region)
+{
+ bio->bi_io_vec[bio->bi_max_vecs - 1].bv_len = region;
+}
+
+static inline unsigned bio_get_region(struct bio *bio)
+{
+ return bio->bi_io_vec[bio->bi_max_vecs - 1].bv_len;
+}
+
+/*-----------------------------------------------------------------
+ * We need an io object to keep track of the number of bios that
+ * have been dispatched for a particular io.
+ *---------------------------------------------------------------*/
+static void dec_count(struct io *io, unsigned int region, int error)
+{
+ if (error)
+ set_bit(region, &io->error);
+
+ if (atomic_dec_and_test(&io->count)) {
+ if (io->sleeper)
+ wake_up_process(io->sleeper);
+
+ else {
+ int r = io->error;
+ io_notify_fn fn = io->callback;
+ void *context = io->context;
+
+ mempool_free(io, _io_pool);
+ fn(r, context);
+ }
+ }
+}
+
+static int endio(struct bio *bio, unsigned int done, int error)
+{
+ struct io *io = (struct io *) bio->bi_private;
+
+ /* keep going until we've finished */
+ if (bio->bi_size)
+ return 1;
+
+ if (error && bio_data_dir(bio) == READ)
+ zero_fill_bio(bio);
+
+ dec_count(io, bio_get_region(bio), error);
+ bio_put(bio);
+
+ return 0;
+}
+
+/*-----------------------------------------------------------------
+ * These little objects provide an abstraction for getting a new
+ * destination page for io.
+ *---------------------------------------------------------------*/
+struct dpages {
+ void (*get_page)(struct dpages *dp,
+ struct page **p, unsigned long *len, unsigned *offset);
+ void (*next_page)(struct dpages *dp);
+
+ unsigned context_u;
+ void *context_ptr;
+};
+
+/*
+ * Functions for getting the pages from a list.
+ */
+static void list_get_page(struct dpages *dp,
+ struct page **p, unsigned long *len, unsigned *offset)
+{
+ unsigned o = dp->context_u;
+ struct page_list *pl = (struct page_list *) dp->context_ptr;
+
+ *p = pl->page;
+ *len = PAGE_SIZE - o;
+ *offset = o;
+}
+
+static void list_next_page(struct dpages *dp)
+{
+ struct page_list *pl = (struct page_list *) dp->context_ptr;
+ dp->context_ptr = pl->next;
+ dp->context_u = 0;
+}
+
+static void list_dp_init(struct dpages *dp, struct page_list *pl, unsigned offset)
+{
+ dp->get_page = list_get_page;
+ dp->next_page = list_next_page;
+ dp->context_u = offset;
+ dp->context_ptr = pl;
+}
+
+/*
+ * Functions for getting the pages from a bvec.
+ */
+static void bvec_get_page(struct dpages *dp,
+ struct page **p, unsigned long *len, unsigned *offset)
+{
+ struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr;
+ *p = bvec->bv_page;
+ *len = bvec->bv_len;
+ *offset = bvec->bv_offset;
+}
+
+static void bvec_next_page(struct dpages *dp)
+{
+ struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr;
+ dp->context_ptr = bvec + 1;
+}
+
+static void bvec_dp_init(struct dpages *dp, struct bio_vec *bvec)
+{
+ dp->get_page = bvec_get_page;
+ dp->next_page = bvec_next_page;
+ dp->context_ptr = bvec;
+}
+
+static void vm_get_page(struct dpages *dp,
+ struct page **p, unsigned long *len, unsigned *offset)
+{
+ *p = vmalloc_to_page(dp->context_ptr);
+ *offset = dp->context_u;
+ *len = PAGE_SIZE - dp->context_u;
+}
+
+static void vm_next_page(struct dpages *dp)
+{
+ dp->context_ptr += PAGE_SIZE - dp->context_u;
+ dp->context_u = 0;
+}
+
+static void vm_dp_init(struct dpages *dp, void *data)
+{
+ dp->get_page = vm_get_page;
+ dp->next_page = vm_next_page;
+ dp->context_u = ((unsigned long) data) & (PAGE_SIZE - 1);
+ dp->context_ptr = data;
+}
+
+/*-----------------------------------------------------------------
+ * IO routines that accept a list of pages.
+ *---------------------------------------------------------------*/
+static void do_region(int rw, unsigned int region, struct io_region *where,
+ struct dpages *dp, struct io *io)
+{
+ struct bio *bio;
+ struct page *page;
+ unsigned long len;
+ unsigned offset;
+ unsigned num_bvecs;
+ sector_t remaining = where->count;
+
+ while (remaining) {
+ /*
+ * Allocate a suitably sized bio, we add an extra
+ * bvec for bio_get/set_region().
+ */
+ num_bvecs = (remaining / (PAGE_SIZE >> 9)) + 2;
+ bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, _bios);
+ bio->bi_sector = where->sector + (where->count - remaining);
+ bio->bi_bdev = where->bdev;
+ bio->bi_end_io = endio;
+ bio->bi_private = io;
+ bio_set_region(bio, region);
+
+ /*
+ * Try and add as many pages as possible.
+ */
+ while (remaining) {
+ dp->get_page(dp, &page, &len, &offset);
+ len = min(len, to_bytes(remaining));
+ if (!bio_add_page(bio, page, len, offset))
+ break;
+
+ offset = 0;
+ remaining -= to_sector(len);
+ dp->next_page(dp);
+ }
+
+ atomic_inc(&io->count);
+ submit_bio(rw, bio);
+ }
+}
+
+static void dispatch_io(int rw, unsigned int num_regions,
+ struct io_region *where, struct dpages *dp,
+ struct io *io, int sync)
+{
+ int i;
+ struct dpages old_pages = *dp;
+
+ if (sync)
+ rw |= (1 << BIO_RW_SYNC);
+
+ /*
+ * For multiple regions we need to be careful to rewind
+ * the dp object for each call to do_region.
+ */
+ for (i = 0; i < num_regions; i++) {
+ *dp = old_pages;
+ if (where[i].count)
+ do_region(rw, i, where + i, dp, io);
+ }
+
+ /*
+ * Drop the extra refence that we were holding to avoid
+ * the io being completed too early.
+ */
+ dec_count(io, 0, 0);
+}
+
+static int sync_io(unsigned int num_regions, struct io_region *where,
+ int rw, struct dpages *dp, unsigned long *error_bits)
+{
+ struct io io;
+
+ if (num_regions > 1 && rw != WRITE) {
+ WARN_ON(1);
+ return -EIO;
+ }
+
+ io.error = 0;
+ atomic_set(&io.count, 1); /* see dispatch_io() */
+ io.sleeper = current;
+
+ dispatch_io(rw, num_regions, where, dp, &io, 1);
+
+ while (1) {
+ set_current_state(TASK_UNINTERRUPTIBLE);
+
+ if (!atomic_read(&io.count) || signal_pending(current))
+ break;
+
+ io_schedule();
+ }
+ set_current_state(TASK_RUNNING);
+
+ if (atomic_read(&io.count))
+ return -EINTR;
+
+ *error_bits = io.error;
+ return io.error ? -EIO : 0;
+}
+
+static int async_io(unsigned int num_regions, struct io_region *where, int rw,
+ struct dpages *dp, io_notify_fn fn, void *context)
+{
+ struct io *io;
+
+ if (num_regions > 1 && rw != WRITE) {
+ WARN_ON(1);
+ fn(1, context);
+ return -EIO;
+ }
+
+ io = mempool_alloc(_io_pool, GFP_NOIO);
+ io->error = 0;
+ atomic_set(&io->count, 1); /* see dispatch_io() */
+ io->sleeper = NULL;
+ io->callback = fn;
+ io->context = context;
+
+ dispatch_io(rw, num_regions, where, dp, io, 0);
+ return 0;
+}
+
+int dm_io_sync(unsigned int num_regions, struct io_region *where, int rw,
+ struct page_list *pl, unsigned int offset,
+ unsigned long *error_bits)
+{
+ struct dpages dp;
+ list_dp_init(&dp, pl, offset);
+ return sync_io(num_regions, where, rw, &dp, error_bits);
+}
+
+int dm_io_sync_bvec(unsigned int num_regions, struct io_region *where, int rw,
+ struct bio_vec *bvec, unsigned long *error_bits)
+{
+ struct dpages dp;
+ bvec_dp_init(&dp, bvec);
+ return sync_io(num_regions, where, rw, &dp, error_bits);
+}
+
+int dm_io_sync_vm(unsigned int num_regions, struct io_region *where, int rw,
+ void *data, unsigned long *error_bits)
+{
+ struct dpages dp;
+ vm_dp_init(&dp, data);
+ return sync_io(num_regions, where, rw, &dp, error_bits);
+}
+
+int dm_io_async(unsigned int num_regions, struct io_region *where, int rw,
+ struct page_list *pl, unsigned int offset,
+ io_notify_fn fn, void *context)
+{
+ struct dpages dp;
+ list_dp_init(&dp, pl, offset);
+ return async_io(num_regions, where, rw, &dp, fn, context);
+}
+
+int dm_io_async_bvec(unsigned int num_regions, struct io_region *where, int rw,
+ struct bio_vec *bvec, io_notify_fn fn, void *context)
+{
+ struct dpages dp;
+ bvec_dp_init(&dp, bvec);
+ return async_io(num_regions, where, rw, &dp, fn, context);
+}
+
+int dm_io_async_vm(unsigned int num_regions, struct io_region *where, int rw,
+ void *data, io_notify_fn fn, void *context)
+{
+ struct dpages dp;
+ vm_dp_init(&dp, data);
+ return async_io(num_regions, where, rw, &dp, fn, context);
+}
+
+EXPORT_SYMBOL(dm_io_get);
+EXPORT_SYMBOL(dm_io_put);
+EXPORT_SYMBOL(dm_io_sync);
+EXPORT_SYMBOL(dm_io_async);
+EXPORT_SYMBOL(dm_io_sync_bvec);
+EXPORT_SYMBOL(dm_io_async_bvec);
+EXPORT_SYMBOL(dm_io_sync_vm);
+EXPORT_SYMBOL(dm_io_async_vm);
diff --git a/drivers/md/dm-io.h b/drivers/md/dm-io.h
new file mode 100644
index 000000000000..1a77f3265706
--- /dev/null
+++ b/drivers/md/dm-io.h
@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2003 Sistina Software
+ *
+ * This file is released under the GPL.
+ */
+
+#ifndef _DM_IO_H
+#define _DM_IO_H
+
+#include "dm.h"
+
+/* FIXME make this configurable */
+#define DM_MAX_IO_REGIONS 8
+
+struct io_region {
+ struct block_device *bdev;
+ sector_t sector;
+ sector_t count;
+};
+
+struct page_list {
+ struct page_list *next;
+ struct page *page;
+};
+
+
+/*
+ * 'error' is a bitset, with each bit indicating whether an error
+ * occurred doing io to the corresponding region.
+ */
+typedef void (*io_notify_fn)(unsigned long error, void *context);
+
+
+/*
+ * Before anyone uses the IO interface they should call
+ * dm_io_get(), specifying roughly how many pages they are
+ * expecting to perform io on concurrently.
+ *
+ * This function may block.
+ */
+int dm_io_get(unsigned int num_pages);
+void dm_io_put(unsigned int num_pages);
+
+/*
+ * Synchronous IO.
+ *
+ * Please ensure that the rw flag in the next two functions is
+ * either READ or WRITE, ie. we don't take READA. Any
+ * regions with a zero count field will be ignored.
+ */
+int dm_io_sync(unsigned int num_regions, struct io_region *where, int rw,
+ struct page_list *pl, unsigned int offset,
+ unsigned long *error_bits);
+
+int dm_io_sync_bvec(unsigned int num_regions, struct io_region *where, int rw,
+ struct bio_vec *bvec, unsigned long *error_bits);
+
+int dm_io_sync_vm(unsigned int num_regions, struct io_region *where, int rw,
+ void *data, unsigned long *error_bits);
+
+/*
+ * Aynchronous IO.
+ *
+ * The 'where' array may be safely allocated on the stack since
+ * the function takes a copy.
+ */
+int dm_io_async(unsigned int num_regions, struct io_region *where, int rw,
+ struct page_list *pl, unsigned int offset,
+ io_notify_fn fn, void *context);
+
+int dm_io_async_bvec(unsigned int num_regions, struct io_region *where, int rw,
+ struct bio_vec *bvec, io_notify_fn fn, void *context);
+
+int dm_io_async_vm(unsigned int num_regions, struct io_region *where, int rw,
+ void *data, io_notify_fn fn, void *context);
+
+#endif
diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c
new file mode 100644
index 000000000000..ee3c869d9701
--- /dev/null
+++ b/drivers/md/dm-ioctl.c
@@ -0,0 +1,1416 @@
+/*
+ * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
+ * Copyright (C) 2004 - 2005 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+
+#include <linux/module.h>
+#include <linux/vmalloc.h>
+#include <linux/miscdevice.h>
+#include <linux/init.h>
+#include <linux/wait.h>
+#include <linux/slab.h>
+#include <linux/devfs_fs_kernel.h>
+#include <linux/dm-ioctl.h>
+
+#include <asm/uaccess.h>
+
+#define DM_DRIVER_EMAIL "dm-devel@redhat.com"
+
+/*-----------------------------------------------------------------
+ * The ioctl interface needs to be able to look up devices by
+ * name or uuid.
+ *---------------------------------------------------------------*/
+struct hash_cell {
+ struct list_head name_list;
+ struct list_head uuid_list;
+
+ char *name;
+ char *uuid;
+ struct mapped_device *md;
+ struct dm_table *new_map;
+};
+
+struct vers_iter {
+ size_t param_size;
+ struct dm_target_versions *vers, *old_vers;
+ char *end;
+ uint32_t flags;
+};
+
+
+#define NUM_BUCKETS 64
+#define MASK_BUCKETS (NUM_BUCKETS - 1)
+static struct list_head _name_buckets[NUM_BUCKETS];
+static struct list_head _uuid_buckets[NUM_BUCKETS];
+
+static void dm_hash_remove_all(void);
+
+/*
+ * Guards access to both hash tables.
+ */
+static DECLARE_RWSEM(_hash_lock);
+
+static void init_buckets(struct list_head *buckets)
+{
+ unsigned int i;
+
+ for (i = 0; i < NUM_BUCKETS; i++)
+ INIT_LIST_HEAD(buckets + i);
+}
+
+static int dm_hash_init(void)
+{
+ init_buckets(_name_buckets);
+ init_buckets(_uuid_buckets);
+ devfs_mk_dir(DM_DIR);
+ return 0;
+}
+
+static void dm_hash_exit(void)
+{
+ dm_hash_remove_all();
+ devfs_remove(DM_DIR);
+}
+
+/*-----------------------------------------------------------------
+ * Hash function:
+ * We're not really concerned with the str hash function being
+ * fast since it's only used by the ioctl interface.
+ *---------------------------------------------------------------*/
+static unsigned int hash_str(const char *str)
+{
+ const unsigned int hash_mult = 2654435387U;
+ unsigned int h = 0;
+
+ while (*str)
+ h = (h + (unsigned int) *str++) * hash_mult;
+
+ return h & MASK_BUCKETS;
+}
+
+/*-----------------------------------------------------------------
+ * Code for looking up a device by name
+ *---------------------------------------------------------------*/
+static struct hash_cell *__get_name_cell(const char *str)
+{
+ struct hash_cell *hc;
+ unsigned int h = hash_str(str);
+
+ list_for_each_entry (hc, _name_buckets + h, name_list)
+ if (!strcmp(hc->name, str))
+ return hc;
+
+ return NULL;
+}
+
+static struct hash_cell *__get_uuid_cell(const char *str)
+{
+ struct hash_cell *hc;
+ unsigned int h = hash_str(str);
+
+ list_for_each_entry (hc, _uuid_buckets + h, uuid_list)
+ if (!strcmp(hc->uuid, str))
+ return hc;
+
+ return NULL;
+}
+
+/*-----------------------------------------------------------------
+ * Inserting, removing and renaming a device.
+ *---------------------------------------------------------------*/
+static inline char *kstrdup(const char *str)
+{
+ char *r = kmalloc(strlen(str) + 1, GFP_KERNEL);
+ if (r)
+ strcpy(r, str);
+ return r;
+}
+
+static struct hash_cell *alloc_cell(const char *name, const char *uuid,
+ struct mapped_device *md)
+{
+ struct hash_cell *hc;
+
+ hc = kmalloc(sizeof(*hc), GFP_KERNEL);
+ if (!hc)
+ return NULL;
+
+ hc->name = kstrdup(name);
+ if (!hc->name) {
+ kfree(hc);
+ return NULL;
+ }
+
+ if (!uuid)
+ hc->uuid = NULL;
+
+ else {
+ hc->uuid = kstrdup(uuid);
+ if (!hc->uuid) {
+ kfree(hc->name);
+ kfree(hc);
+ return NULL;
+ }
+ }
+
+ INIT_LIST_HEAD(&hc->name_list);
+ INIT_LIST_HEAD(&hc->uuid_list);
+ hc->md = md;
+ hc->new_map = NULL;
+ return hc;
+}
+
+static void free_cell(struct hash_cell *hc)
+{
+ if (hc) {
+ kfree(hc->name);
+ kfree(hc->uuid);
+ kfree(hc);
+ }
+}
+
+/*
+ * devfs stuff.
+ */
+static int register_with_devfs(struct hash_cell *hc)
+{
+ struct gendisk *disk = dm_disk(hc->md);
+
+ devfs_mk_bdev(MKDEV(disk->major, disk->first_minor),
+ S_IFBLK | S_IRUSR | S_IWUSR | S_IRGRP,
+ DM_DIR "/%s", hc->name);
+ return 0;
+}
+
+static int unregister_with_devfs(struct hash_cell *hc)
+{
+ devfs_remove(DM_DIR"/%s", hc->name);
+ return 0;
+}
+
+/*
+ * The kdev_t and uuid of a device can never change once it is
+ * initially inserted.
+ */
+static int dm_hash_insert(const char *name, const char *uuid, struct mapped_device *md)
+{
+ struct hash_cell *cell;
+
+ /*
+ * Allocate the new cells.
+ */
+ cell = alloc_cell(name, uuid, md);
+ if (!cell)
+ return -ENOMEM;
+
+ /*
+ * Insert the cell into both hash tables.
+ */
+ down_write(&_hash_lock);
+ if (__get_name_cell(name))
+ goto bad;
+
+ list_add(&cell->name_list, _name_buckets + hash_str(name));
+
+ if (uuid) {
+ if (__get_uuid_cell(uuid)) {
+ list_del(&cell->name_list);
+ goto bad;
+ }
+ list_add(&cell->uuid_list, _uuid_buckets + hash_str(uuid));
+ }
+ register_with_devfs(cell);
+ dm_get(md);
+ dm_set_mdptr(md, cell);
+ up_write(&_hash_lock);
+
+ return 0;
+
+ bad:
+ up_write(&_hash_lock);
+ free_cell(cell);
+ return -EBUSY;
+}
+
+static void __hash_remove(struct hash_cell *hc)
+{
+ /* remove from the dev hash */
+ list_del(&hc->uuid_list);
+ list_del(&hc->name_list);
+ unregister_with_devfs(hc);
+ dm_set_mdptr(hc->md, NULL);
+ dm_put(hc->md);
+ if (hc->new_map)
+ dm_table_put(hc->new_map);
+ free_cell(hc);
+}
+
+static void dm_hash_remove_all(void)
+{
+ int i;
+ struct hash_cell *hc;
+ struct list_head *tmp, *n;
+
+ down_write(&_hash_lock);
+ for (i = 0; i < NUM_BUCKETS; i++) {
+ list_for_each_safe (tmp, n, _name_buckets + i) {
+ hc = list_entry(tmp, struct hash_cell, name_list);
+ __hash_remove(hc);
+ }
+ }
+ up_write(&_hash_lock);
+}
+
+static int dm_hash_rename(const char *old, const char *new)
+{
+ char *new_name, *old_name;
+ struct hash_cell *hc;
+
+ /*
+ * duplicate new.
+ */
+ new_name = kstrdup(new);
+ if (!new_name)
+ return -ENOMEM;
+
+ down_write(&_hash_lock);
+
+ /*
+ * Is new free ?
+ */
+ hc = __get_name_cell(new);
+ if (hc) {
+ DMWARN("asked to rename to an already existing name %s -> %s",
+ old, new);
+ up_write(&_hash_lock);
+ kfree(new_name);
+ return -EBUSY;
+ }
+
+ /*
+ * Is there such a device as 'old' ?
+ */
+ hc = __get_name_cell(old);
+ if (!hc) {
+ DMWARN("asked to rename a non existent device %s -> %s",
+ old, new);
+ up_write(&_hash_lock);
+ kfree(new_name);
+ return -ENXIO;
+ }
+
+ /*
+ * rename and move the name cell.
+ */
+ unregister_with_devfs(hc);
+
+ list_del(&hc->name_list);
+ old_name = hc->name;
+ hc->name = new_name;
+ list_add(&hc->name_list, _name_buckets + hash_str(new_name));
+
+ /* rename the device node in devfs */
+ register_with_devfs(hc);
+
+ up_write(&_hash_lock);
+ kfree(old_name);
+ return 0;
+}
+
+/*-----------------------------------------------------------------
+ * Implementation of the ioctl commands
+ *---------------------------------------------------------------*/
+/*
+ * All the ioctl commands get dispatched to functions with this
+ * prototype.
+ */
+typedef int (*ioctl_fn)(struct dm_ioctl *param, size_t param_size);
+
+static int remove_all(struct dm_ioctl *param, size_t param_size)
+{
+ dm_hash_remove_all();
+ param->data_size = 0;
+ return 0;
+}
+
+/*
+ * Round up the ptr to an 8-byte boundary.
+ */
+#define ALIGN_MASK 7
+static inline void *align_ptr(void *ptr)
+{
+ return (void *) (((size_t) (ptr + ALIGN_MASK)) & ~ALIGN_MASK);
+}
+
+/*
+ * Retrieves the data payload buffer from an already allocated
+ * struct dm_ioctl.
+ */
+static void *get_result_buffer(struct dm_ioctl *param, size_t param_size,
+ size_t *len)
+{
+ param->data_start = align_ptr(param + 1) - (void *) param;
+
+ if (param->data_start < param_size)
+ *len = param_size - param->data_start;
+ else
+ *len = 0;
+
+ return ((void *) param) + param->data_start;
+}
+
+static int list_devices(struct dm_ioctl *param, size_t param_size)
+{
+ unsigned int i;
+ struct hash_cell *hc;
+ size_t len, needed = 0;
+ struct gendisk *disk;
+ struct dm_name_list *nl, *old_nl = NULL;
+
+ down_write(&_hash_lock);
+
+ /*
+ * Loop through all the devices working out how much
+ * space we need.
+ */
+ for (i = 0; i < NUM_BUCKETS; i++) {
+ list_for_each_entry (hc, _name_buckets + i, name_list) {
+ needed += sizeof(struct dm_name_list);
+ needed += strlen(hc->name) + 1;
+ needed += ALIGN_MASK;
+ }
+ }
+
+ /*
+ * Grab our output buffer.
+ */
+ nl = get_result_buffer(param, param_size, &len);
+ if (len < needed) {
+ param->flags |= DM_BUFFER_FULL_FLAG;
+ goto out;
+ }
+ param->data_size = param->data_start + needed;
+
+ nl->dev = 0; /* Flags no data */
+
+ /*
+ * Now loop through filling out the names.
+ */
+ for (i = 0; i < NUM_BUCKETS; i++) {
+ list_for_each_entry (hc, _name_buckets + i, name_list) {
+ if (old_nl)
+ old_nl->next = (uint32_t) ((void *) nl -
+ (void *) old_nl);
+ disk = dm_disk(hc->md);
+ nl->dev = huge_encode_dev(MKDEV(disk->major, disk->first_minor));
+ nl->next = 0;
+ strcpy(nl->name, hc->name);
+
+ old_nl = nl;
+ nl = align_ptr(((void *) ++nl) + strlen(hc->name) + 1);
+ }
+ }
+
+ out:
+ up_write(&_hash_lock);
+ return 0;
+}
+
+static void list_version_get_needed(struct target_type *tt, void *needed_param)
+{
+ size_t *needed = needed_param;
+
+ *needed += strlen(tt->name);
+ *needed += sizeof(tt->version);
+ *needed += ALIGN_MASK;
+}
+
+static void list_version_get_info(struct target_type *tt, void *param)
+{
+ struct vers_iter *info = param;
+
+ /* Check space - it might have changed since the first iteration */
+ if ((char *)info->vers + sizeof(tt->version) + strlen(tt->name) + 1 >
+ info->end) {
+
+ info->flags = DM_BUFFER_FULL_FLAG;
+ return;
+ }
+
+ if (info->old_vers)
+ info->old_vers->next = (uint32_t) ((void *)info->vers -
+ (void *)info->old_vers);
+ info->vers->version[0] = tt->version[0];
+ info->vers->version[1] = tt->version[1];
+ info->vers->version[2] = tt->version[2];
+ info->vers->next = 0;
+ strcpy(info->vers->name, tt->name);
+
+ info->old_vers = info->vers;
+ info->vers = align_ptr(((void *) ++info->vers) + strlen(tt->name) + 1);
+}
+
+static int list_versions(struct dm_ioctl *param, size_t param_size)
+{
+ size_t len, needed = 0;
+ struct dm_target_versions *vers;
+ struct vers_iter iter_info;
+
+ /*
+ * Loop through all the devices working out how much
+ * space we need.
+ */
+ dm_target_iterate(list_version_get_needed, &needed);
+
+ /*
+ * Grab our output buffer.
+ */
+ vers = get_result_buffer(param, param_size, &len);
+ if (len < needed) {
+ param->flags |= DM_BUFFER_FULL_FLAG;
+ goto out;
+ }
+ param->data_size = param->data_start + needed;
+
+ iter_info.param_size = param_size;
+ iter_info.old_vers = NULL;
+ iter_info.vers = vers;
+ iter_info.flags = 0;
+ iter_info.end = (char *)vers+len;
+
+ /*
+ * Now loop through filling out the names & versions.
+ */
+ dm_target_iterate(list_version_get_info, &iter_info);
+ param->flags |= iter_info.flags;
+
+ out:
+ return 0;
+}
+
+
+
+static int check_name(const char *name)
+{
+ if (strchr(name, '/')) {
+ DMWARN("invalid device name");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/*
+ * Fills in a dm_ioctl structure, ready for sending back to
+ * userland.
+ */
+static int __dev_status(struct mapped_device *md, struct dm_ioctl *param)
+{
+ struct gendisk *disk = dm_disk(md);
+ struct dm_table *table;
+ struct block_device *bdev;
+
+ param->flags &= ~(DM_SUSPEND_FLAG | DM_READONLY_FLAG |
+ DM_ACTIVE_PRESENT_FLAG);
+
+ if (dm_suspended(md))
+ param->flags |= DM_SUSPEND_FLAG;
+
+ param->dev = huge_encode_dev(MKDEV(disk->major, disk->first_minor));
+
+ if (!(param->flags & DM_SKIP_BDGET_FLAG)) {
+ bdev = bdget_disk(disk, 0);
+ if (!bdev)
+ return -ENXIO;
+
+ /*
+ * Yes, this will be out of date by the time it gets back
+ * to userland, but it is still very useful for
+ * debugging.
+ */
+ param->open_count = bdev->bd_openers;
+ bdput(bdev);
+ } else
+ param->open_count = -1;
+
+ if (disk->policy)
+ param->flags |= DM_READONLY_FLAG;
+
+ param->event_nr = dm_get_event_nr(md);
+
+ table = dm_get_table(md);
+ if (table) {
+ param->flags |= DM_ACTIVE_PRESENT_FLAG;
+ param->target_count = dm_table_get_num_targets(table);
+ dm_table_put(table);
+ } else
+ param->target_count = 0;
+
+ return 0;
+}
+
+static int dev_create(struct dm_ioctl *param, size_t param_size)
+{
+ int r;
+ struct mapped_device *md;
+
+ r = check_name(param->name);
+ if (r)
+ return r;
+
+ if (param->flags & DM_PERSISTENT_DEV_FLAG)
+ r = dm_create_with_minor(MINOR(huge_decode_dev(param->dev)), &md);
+ else
+ r = dm_create(&md);
+
+ if (r)
+ return r;
+
+ r = dm_hash_insert(param->name, *param->uuid ? param->uuid : NULL, md);
+ if (r) {
+ dm_put(md);
+ return r;
+ }
+
+ param->flags &= ~DM_INACTIVE_PRESENT_FLAG;
+
+ r = __dev_status(md, param);
+ dm_put(md);
+
+ return r;
+}
+
+/*
+ * Always use UUID for lookups if it's present, otherwise use name or dev.
+ */
+static inline struct hash_cell *__find_device_hash_cell(struct dm_ioctl *param)
+{
+ if (*param->uuid)
+ return __get_uuid_cell(param->uuid);
+ else if (*param->name)
+ return __get_name_cell(param->name);
+ else
+ return dm_get_mdptr(huge_decode_dev(param->dev));
+}
+
+static inline struct mapped_device *find_device(struct dm_ioctl *param)
+{
+ struct hash_cell *hc;
+ struct mapped_device *md = NULL;
+
+ down_read(&_hash_lock);
+ hc = __find_device_hash_cell(param);
+ if (hc) {
+ md = hc->md;
+ dm_get(md);
+
+ /*
+ * Sneakily write in both the name and the uuid
+ * while we have the cell.
+ */
+ strncpy(param->name, hc->name, sizeof(param->name));
+ if (hc->uuid)
+ strncpy(param->uuid, hc->uuid, sizeof(param->uuid)-1);
+ else
+ param->uuid[0] = '\0';
+
+ if (hc->new_map)
+ param->flags |= DM_INACTIVE_PRESENT_FLAG;
+ else
+ param->flags &= ~DM_INACTIVE_PRESENT_FLAG;
+ }
+ up_read(&_hash_lock);
+
+ return md;
+}
+
+static int dev_remove(struct dm_ioctl *param, size_t param_size)
+{
+ struct hash_cell *hc;
+
+ down_write(&_hash_lock);
+ hc = __find_device_hash_cell(param);
+
+ if (!hc) {
+ DMWARN("device doesn't appear to be in the dev hash table.");
+ up_write(&_hash_lock);
+ return -ENXIO;
+ }
+
+ __hash_remove(hc);
+ up_write(&_hash_lock);
+ param->data_size = 0;
+ return 0;
+}
+
+/*
+ * Check a string doesn't overrun the chunk of
+ * memory we copied from userland.
+ */
+static int invalid_str(char *str, void *end)
+{
+ while ((void *) str < end)
+ if (!*str++)
+ return 0;
+
+ return -EINVAL;
+}
+
+static int dev_rename(struct dm_ioctl *param, size_t param_size)
+{
+ int r;
+ char *new_name = (char *) param + param->data_start;
+
+ if (new_name < (char *) (param + 1) ||
+ invalid_str(new_name, (void *) param + param_size)) {
+ DMWARN("Invalid new logical volume name supplied.");
+ return -EINVAL;
+ }
+
+ r = check_name(new_name);
+ if (r)
+ return r;
+
+ param->data_size = 0;
+ return dm_hash_rename(param->name, new_name);
+}
+
+static int do_suspend(struct dm_ioctl *param)
+{
+ int r = 0;
+ struct mapped_device *md;
+
+ md = find_device(param);
+ if (!md)
+ return -ENXIO;
+
+ if (!dm_suspended(md))
+ r = dm_suspend(md);
+
+ if (!r)
+ r = __dev_status(md, param);
+
+ dm_put(md);
+ return r;
+}
+
+static int do_resume(struct dm_ioctl *param)
+{
+ int r = 0;
+ struct hash_cell *hc;
+ struct mapped_device *md;
+ struct dm_table *new_map;
+
+ down_write(&_hash_lock);
+
+ hc = __find_device_hash_cell(param);
+ if (!hc) {
+ DMWARN("device doesn't appear to be in the dev hash table.");
+ up_write(&_hash_lock);
+ return -ENXIO;
+ }
+
+ md = hc->md;
+ dm_get(md);
+
+ new_map = hc->new_map;
+ hc->new_map = NULL;
+ param->flags &= ~DM_INACTIVE_PRESENT_FLAG;
+
+ up_write(&_hash_lock);
+
+ /* Do we need to load a new map ? */
+ if (new_map) {
+ /* Suspend if it isn't already suspended */
+ if (!dm_suspended(md))
+ dm_suspend(md);
+
+ r = dm_swap_table(md, new_map);
+ if (r) {
+ dm_put(md);
+ dm_table_put(new_map);
+ return r;
+ }
+
+ if (dm_table_get_mode(new_map) & FMODE_WRITE)
+ set_disk_ro(dm_disk(md), 0);
+ else
+ set_disk_ro(dm_disk(md), 1);
+
+ dm_table_put(new_map);
+ }
+
+ if (dm_suspended(md))
+ r = dm_resume(md);
+
+ if (!r)
+ r = __dev_status(md, param);
+
+ dm_put(md);
+ return r;
+}
+
+/*
+ * Set or unset the suspension state of a device.
+ * If the device already is in the requested state we just return its status.
+ */
+static int dev_suspend(struct dm_ioctl *param, size_t param_size)
+{
+ if (param->flags & DM_SUSPEND_FLAG)
+ return do_suspend(param);
+
+ return do_resume(param);
+}
+
+/*
+ * Copies device info back to user space, used by
+ * the create and info ioctls.
+ */
+static int dev_status(struct dm_ioctl *param, size_t param_size)
+{
+ int r;
+ struct mapped_device *md;
+
+ md = find_device(param);
+ if (!md)
+ return -ENXIO;
+
+ r = __dev_status(md, param);
+ dm_put(md);
+ return r;
+}
+
+/*
+ * Build up the status struct for each target
+ */
+static void retrieve_status(struct dm_table *table,
+ struct dm_ioctl *param, size_t param_size)
+{
+ unsigned int i, num_targets;
+ struct dm_target_spec *spec;
+ char *outbuf, *outptr;
+ status_type_t type;
+ size_t remaining, len, used = 0;
+
+ outptr = outbuf = get_result_buffer(param, param_size, &len);
+
+ if (param->flags & DM_STATUS_TABLE_FLAG)
+ type = STATUSTYPE_TABLE;
+ else
+ type = STATUSTYPE_INFO;
+
+ /* Get all the target info */
+ num_targets = dm_table_get_num_targets(table);
+ for (i = 0; i < num_targets; i++) {
+ struct dm_target *ti = dm_table_get_target(table, i);
+
+ remaining = len - (outptr - outbuf);
+ if (remaining <= sizeof(struct dm_target_spec)) {
+ param->flags |= DM_BUFFER_FULL_FLAG;
+ break;
+ }
+
+ spec = (struct dm_target_spec *) outptr;
+
+ spec->status = 0;
+ spec->sector_start = ti->begin;
+ spec->length = ti->len;
+ strncpy(spec->target_type, ti->type->name,
+ sizeof(spec->target_type));
+
+ outptr += sizeof(struct dm_target_spec);
+ remaining = len - (outptr - outbuf);
+ if (remaining <= 0) {
+ param->flags |= DM_BUFFER_FULL_FLAG;
+ break;
+ }
+
+ /* Get the status/table string from the target driver */
+ if (ti->type->status) {
+ if (ti->type->status(ti, type, outptr, remaining)) {
+ param->flags |= DM_BUFFER_FULL_FLAG;
+ break;
+ }
+ } else
+ outptr[0] = '\0';
+
+ outptr += strlen(outptr) + 1;
+ used = param->data_start + (outptr - outbuf);
+
+ outptr = align_ptr(outptr);
+ spec->next = outptr - outbuf;
+ }
+
+ if (used)
+ param->data_size = used;
+
+ param->target_count = num_targets;
+}
+
+/*
+ * Wait for a device to report an event
+ */
+static int dev_wait(struct dm_ioctl *param, size_t param_size)
+{
+ int r;
+ struct mapped_device *md;
+ struct dm_table *table;
+
+ md = find_device(param);
+ if (!md)
+ return -ENXIO;
+
+ /*
+ * Wait for a notification event
+ */
+ if (dm_wait_event(md, param->event_nr)) {
+ r = -ERESTARTSYS;
+ goto out;
+ }
+
+ /*
+ * The userland program is going to want to know what
+ * changed to trigger the event, so we may as well tell
+ * him and save an ioctl.
+ */
+ r = __dev_status(md, param);
+ if (r)
+ goto out;
+
+ table = dm_get_table(md);
+ if (table) {
+ retrieve_status(table, param, param_size);
+ dm_table_put(table);
+ }
+
+ out:
+ dm_put(md);
+ return r;
+}
+
+static inline int get_mode(struct dm_ioctl *param)
+{
+ int mode = FMODE_READ | FMODE_WRITE;
+
+ if (param->flags & DM_READONLY_FLAG)
+ mode = FMODE_READ;
+
+ return mode;
+}
+
+static int next_target(struct dm_target_spec *last, uint32_t next, void *end,
+ struct dm_target_spec **spec, char **target_params)
+{
+ *spec = (struct dm_target_spec *) ((unsigned char *) last + next);
+ *target_params = (char *) (*spec + 1);
+
+ if (*spec < (last + 1))
+ return -EINVAL;
+
+ return invalid_str(*target_params, end);
+}
+
+static int populate_table(struct dm_table *table,
+ struct dm_ioctl *param, size_t param_size)
+{
+ int r;
+ unsigned int i = 0;
+ struct dm_target_spec *spec = (struct dm_target_spec *) param;
+ uint32_t next = param->data_start;
+ void *end = (void *) param + param_size;
+ char *target_params;
+
+ if (!param->target_count) {
+ DMWARN("populate_table: no targets specified");
+ return -EINVAL;
+ }
+
+ for (i = 0; i < param->target_count; i++) {
+
+ r = next_target(spec, next, end, &spec, &target_params);
+ if (r) {
+ DMWARN("unable to find target");
+ return r;
+ }
+
+ r = dm_table_add_target(table, spec->target_type,
+ (sector_t) spec->sector_start,
+ (sector_t) spec->length,
+ target_params);
+ if (r) {
+ DMWARN("error adding target to table");
+ return r;
+ }
+
+ next = spec->next;
+ }
+
+ return dm_table_complete(table);
+}
+
+static int table_load(struct dm_ioctl *param, size_t param_size)
+{
+ int r;
+ struct hash_cell *hc;
+ struct dm_table *t;
+
+ r = dm_table_create(&t, get_mode(param), param->target_count);
+ if (r)
+ return r;
+
+ r = populate_table(t, param, param_size);
+ if (r) {
+ dm_table_put(t);
+ return r;
+ }
+
+ down_write(&_hash_lock);
+ hc = __find_device_hash_cell(param);
+ if (!hc) {
+ DMWARN("device doesn't appear to be in the dev hash table.");
+ up_write(&_hash_lock);
+ return -ENXIO;
+ }
+
+ if (hc->new_map)
+ dm_table_put(hc->new_map);
+ hc->new_map = t;
+ param->flags |= DM_INACTIVE_PRESENT_FLAG;
+
+ r = __dev_status(hc->md, param);
+ up_write(&_hash_lock);
+ return r;
+}
+
+static int table_clear(struct dm_ioctl *param, size_t param_size)
+{
+ int r;
+ struct hash_cell *hc;
+
+ down_write(&_hash_lock);
+
+ hc = __find_device_hash_cell(param);
+ if (!hc) {
+ DMWARN("device doesn't appear to be in the dev hash table.");
+ up_write(&_hash_lock);
+ return -ENXIO;
+ }
+
+ if (hc->new_map) {
+ dm_table_put(hc->new_map);
+ hc->new_map = NULL;
+ }
+
+ param->flags &= ~DM_INACTIVE_PRESENT_FLAG;
+
+ r = __dev_status(hc->md, param);
+ up_write(&_hash_lock);
+ return r;
+}
+
+/*
+ * Retrieves a list of devices used by a particular dm device.
+ */
+static void retrieve_deps(struct dm_table *table,
+ struct dm_ioctl *param, size_t param_size)
+{
+ unsigned int count = 0;
+ struct list_head *tmp;
+ size_t len, needed;
+ struct dm_dev *dd;
+ struct dm_target_deps *deps;
+
+ deps = get_result_buffer(param, param_size, &len);
+
+ /*
+ * Count the devices.
+ */
+ list_for_each (tmp, dm_table_get_devices(table))
+ count++;
+
+ /*
+ * Check we have enough space.
+ */
+ needed = sizeof(*deps) + (sizeof(*deps->dev) * count);
+ if (len < needed) {
+ param->flags |= DM_BUFFER_FULL_FLAG;
+ return;
+ }
+
+ /*
+ * Fill in the devices.
+ */
+ deps->count = count;
+ count = 0;
+ list_for_each_entry (dd, dm_table_get_devices(table), list)
+ deps->dev[count++] = huge_encode_dev(dd->bdev->bd_dev);
+
+ param->data_size = param->data_start + needed;
+}
+
+static int table_deps(struct dm_ioctl *param, size_t param_size)
+{
+ int r = 0;
+ struct mapped_device *md;
+ struct dm_table *table;
+
+ md = find_device(param);
+ if (!md)
+ return -ENXIO;
+
+ r = __dev_status(md, param);
+ if (r)
+ goto out;
+
+ table = dm_get_table(md);
+ if (table) {
+ retrieve_deps(table, param, param_size);
+ dm_table_put(table);
+ }
+
+ out:
+ dm_put(md);
+ return r;
+}
+
+/*
+ * Return the status of a device as a text string for each
+ * target.
+ */
+static int table_status(struct dm_ioctl *param, size_t param_size)
+{
+ int r;
+ struct mapped_device *md;
+ struct dm_table *table;
+
+ md = find_device(param);
+ if (!md)
+ return -ENXIO;
+
+ r = __dev_status(md, param);
+ if (r)
+ goto out;
+
+ table = dm_get_table(md);
+ if (table) {
+ retrieve_status(table, param, param_size);
+ dm_table_put(table);
+ }
+
+ out:
+ dm_put(md);
+ return r;
+}
+
+/*
+ * Pass a message to the target that's at the supplied device offset.
+ */
+static int target_message(struct dm_ioctl *param, size_t param_size)
+{
+ int r, argc;
+ char **argv;
+ struct mapped_device *md;
+ struct dm_table *table;
+ struct dm_target *ti;
+ struct dm_target_msg *tmsg = (void *) param + param->data_start;
+
+ md = find_device(param);
+ if (!md)
+ return -ENXIO;
+
+ r = __dev_status(md, param);
+ if (r)
+ goto out;
+
+ if (tmsg < (struct dm_target_msg *) (param + 1) ||
+ invalid_str(tmsg->message, (void *) param + param_size)) {
+ DMWARN("Invalid target message parameters.");
+ r = -EINVAL;
+ goto out;
+ }
+
+ r = dm_split_args(&argc, &argv, tmsg->message);
+ if (r) {
+ DMWARN("Failed to split target message parameters");
+ goto out;
+ }
+
+ table = dm_get_table(md);
+ if (!table)
+ goto out_argv;
+
+ if (tmsg->sector >= dm_table_get_size(table)) {
+ DMWARN("Target message sector outside device.");
+ r = -EINVAL;
+ goto out_table;
+ }
+
+ ti = dm_table_find_target(table, tmsg->sector);
+ if (ti->type->message)
+ r = ti->type->message(ti, argc, argv);
+ else {
+ DMWARN("Target type does not support messages");
+ r = -EINVAL;
+ }
+
+ out_table:
+ dm_table_put(table);
+ out_argv:
+ kfree(argv);
+ out:
+ param->data_size = 0;
+ dm_put(md);
+ return r;
+}
+
+/*-----------------------------------------------------------------
+ * Implementation of open/close/ioctl on the special char
+ * device.
+ *---------------------------------------------------------------*/
+static ioctl_fn lookup_ioctl(unsigned int cmd)
+{
+ static struct {
+ int cmd;
+ ioctl_fn fn;
+ } _ioctls[] = {
+ {DM_VERSION_CMD, NULL}, /* version is dealt with elsewhere */
+ {DM_REMOVE_ALL_CMD, remove_all},
+ {DM_LIST_DEVICES_CMD, list_devices},
+
+ {DM_DEV_CREATE_CMD, dev_create},
+ {DM_DEV_REMOVE_CMD, dev_remove},
+ {DM_DEV_RENAME_CMD, dev_rename},
+ {DM_DEV_SUSPEND_CMD, dev_suspend},
+ {DM_DEV_STATUS_CMD, dev_status},
+ {DM_DEV_WAIT_CMD, dev_wait},
+
+ {DM_TABLE_LOAD_CMD, table_load},
+ {DM_TABLE_CLEAR_CMD, table_clear},
+ {DM_TABLE_DEPS_CMD, table_deps},
+ {DM_TABLE_STATUS_CMD, table_status},
+
+ {DM_LIST_VERSIONS_CMD, list_versions},
+
+ {DM_TARGET_MSG_CMD, target_message}
+ };
+
+ return (cmd >= ARRAY_SIZE(_ioctls)) ? NULL : _ioctls[cmd].fn;
+}
+
+/*
+ * As well as checking the version compatibility this always
+ * copies the kernel interface version out.
+ */
+static int check_version(unsigned int cmd, struct dm_ioctl __user *user)
+{
+ uint32_t version[3];
+ int r = 0;
+
+ if (copy_from_user(version, user->version, sizeof(version)))
+ return -EFAULT;
+
+ if ((DM_VERSION_MAJOR != version[0]) ||
+ (DM_VERSION_MINOR < version[1])) {
+ DMWARN("ioctl interface mismatch: "
+ "kernel(%u.%u.%u), user(%u.%u.%u), cmd(%d)",
+ DM_VERSION_MAJOR, DM_VERSION_MINOR,
+ DM_VERSION_PATCHLEVEL,
+ version[0], version[1], version[2], cmd);
+ r = -EINVAL;
+ }
+
+ /*
+ * Fill in the kernel version.
+ */
+ version[0] = DM_VERSION_MAJOR;
+ version[1] = DM_VERSION_MINOR;
+ version[2] = DM_VERSION_PATCHLEVEL;
+ if (copy_to_user(user->version, version, sizeof(version)))
+ return -EFAULT;
+
+ return r;
+}
+
+static void free_params(struct dm_ioctl *param)
+{
+ vfree(param);
+}
+
+static int copy_params(struct dm_ioctl __user *user, struct dm_ioctl **param)
+{
+ struct dm_ioctl tmp, *dmi;
+
+ if (copy_from_user(&tmp, user, sizeof(tmp)))
+ return -EFAULT;
+
+ if (tmp.data_size < sizeof(tmp))
+ return -EINVAL;
+
+ dmi = (struct dm_ioctl *) vmalloc(tmp.data_size);
+ if (!dmi)
+ return -ENOMEM;
+
+ if (copy_from_user(dmi, user, tmp.data_size)) {
+ vfree(dmi);
+ return -EFAULT;
+ }
+
+ *param = dmi;
+ return 0;
+}
+
+static int validate_params(uint cmd, struct dm_ioctl *param)
+{
+ /* Always clear this flag */
+ param->flags &= ~DM_BUFFER_FULL_FLAG;
+
+ /* Ignores parameters */
+ if (cmd == DM_REMOVE_ALL_CMD ||
+ cmd == DM_LIST_DEVICES_CMD ||
+ cmd == DM_LIST_VERSIONS_CMD)
+ return 0;
+
+ if ((cmd == DM_DEV_CREATE_CMD)) {
+ if (!*param->name) {
+ DMWARN("name not supplied when creating device");
+ return -EINVAL;
+ }
+ } else if ((*param->uuid && *param->name)) {
+ DMWARN("only supply one of name or uuid, cmd(%u)", cmd);
+ return -EINVAL;
+ }
+
+ /* Ensure strings are terminated */
+ param->name[DM_NAME_LEN - 1] = '\0';
+ param->uuid[DM_UUID_LEN - 1] = '\0';
+
+ return 0;
+}
+
+static int ctl_ioctl(struct inode *inode, struct file *file,
+ uint command, ulong u)
+{
+ int r = 0;
+ unsigned int cmd;
+ struct dm_ioctl *param;
+ struct dm_ioctl __user *user = (struct dm_ioctl __user *) u;
+ ioctl_fn fn = NULL;
+ size_t param_size;
+
+ /* only root can play with this */
+ if (!capable(CAP_SYS_ADMIN))
+ return -EACCES;
+
+ if (_IOC_TYPE(command) != DM_IOCTL)
+ return -ENOTTY;
+
+ cmd = _IOC_NR(command);
+
+ /*
+ * Check the interface version passed in. This also
+ * writes out the kernel's interface version.
+ */
+ r = check_version(cmd, user);
+ if (r)
+ return r;
+
+ /*
+ * Nothing more to do for the version command.
+ */
+ if (cmd == DM_VERSION_CMD)
+ return 0;
+
+ fn = lookup_ioctl(cmd);
+ if (!fn) {
+ DMWARN("dm_ctl_ioctl: unknown command 0x%x", command);
+ return -ENOTTY;
+ }
+
+ /*
+ * Trying to avoid low memory issues when a device is
+ * suspended.
+ */
+ current->flags |= PF_MEMALLOC;
+
+ /*
+ * Copy the parameters into kernel space.
+ */
+ r = copy_params(user, &param);
+ if (r) {
+ current->flags &= ~PF_MEMALLOC;
+ return r;
+ }
+
+ /*
+ * FIXME: eventually we will remove the PF_MEMALLOC flag
+ * here. However the tools still do nasty things like
+ * 'load' while a device is suspended.
+ */
+
+ r = validate_params(cmd, param);
+ if (r)
+ goto out;
+
+ param_size = param->data_size;
+ param->data_size = sizeof(*param);
+ r = fn(param, param_size);
+
+ /*
+ * Copy the results back to userland.
+ */
+ if (!r && copy_to_user(user, param, param->data_size))
+ r = -EFAULT;
+
+ out:
+ free_params(param);
+ current->flags &= ~PF_MEMALLOC;
+ return r;
+}
+
+static struct file_operations _ctl_fops = {
+ .ioctl = ctl_ioctl,
+ .owner = THIS_MODULE,
+};
+
+static struct miscdevice _dm_misc = {
+ .minor = MISC_DYNAMIC_MINOR,
+ .name = DM_NAME,
+ .devfs_name = "mapper/control",
+ .fops = &_ctl_fops
+};
+
+/*
+ * Create misc character device and link to DM_DIR/control.
+ */
+int __init dm_interface_init(void)
+{
+ int r;
+
+ r = dm_hash_init();
+ if (r)
+ return r;
+
+ r = misc_register(&_dm_misc);
+ if (r) {
+ DMERR("misc_register failed for control device");
+ dm_hash_exit();
+ return r;
+ }
+
+ DMINFO("%d.%d.%d%s initialised: %s", DM_VERSION_MAJOR,
+ DM_VERSION_MINOR, DM_VERSION_PATCHLEVEL, DM_VERSION_EXTRA,
+ DM_DRIVER_EMAIL);
+ return 0;
+}
+
+void dm_interface_exit(void)
+{
+ if (misc_deregister(&_dm_misc) < 0)
+ DMERR("misc_deregister failed for control device");
+
+ dm_hash_exit();
+}
diff --git a/drivers/md/dm-linear.c b/drivers/md/dm-linear.c
new file mode 100644
index 000000000000..6a2cd5dc8a63
--- /dev/null
+++ b/drivers/md/dm-linear.c
@@ -0,0 +1,123 @@
+/*
+ * Copyright (C) 2001-2003 Sistina Software (UK) Limited.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/blkdev.h>
+#include <linux/bio.h>
+#include <linux/slab.h>
+
+/*
+ * Linear: maps a linear range of a device.
+ */
+struct linear_c {
+ struct dm_dev *dev;
+ sector_t start;
+};
+
+/*
+ * Construct a linear mapping: <dev_path> <offset>
+ */
+static int linear_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+ struct linear_c *lc;
+
+ if (argc != 2) {
+ ti->error = "dm-linear: Invalid argument count";
+ return -EINVAL;
+ }
+
+ lc = kmalloc(sizeof(*lc), GFP_KERNEL);
+ if (lc == NULL) {
+ ti->error = "dm-linear: Cannot allocate linear context";
+ return -ENOMEM;
+ }
+
+ if (sscanf(argv[1], SECTOR_FORMAT, &lc->start) != 1) {
+ ti->error = "dm-linear: Invalid device sector";
+ goto bad;
+ }
+
+ if (dm_get_device(ti, argv[0], lc->start, ti->len,
+ dm_table_get_mode(ti->table), &lc->dev)) {
+ ti->error = "dm-linear: Device lookup failed";
+ goto bad;
+ }
+
+ ti->private = lc;
+ return 0;
+
+ bad:
+ kfree(lc);
+ return -EINVAL;
+}
+
+static void linear_dtr(struct dm_target *ti)
+{
+ struct linear_c *lc = (struct linear_c *) ti->private;
+
+ dm_put_device(ti, lc->dev);
+ kfree(lc);
+}
+
+static int linear_map(struct dm_target *ti, struct bio *bio,
+ union map_info *map_context)
+{
+ struct linear_c *lc = (struct linear_c *) ti->private;
+
+ bio->bi_bdev = lc->dev->bdev;
+ bio->bi_sector = lc->start + (bio->bi_sector - ti->begin);
+
+ return 1;
+}
+
+static int linear_status(struct dm_target *ti, status_type_t type,
+ char *result, unsigned int maxlen)
+{
+ struct linear_c *lc = (struct linear_c *) ti->private;
+
+ switch (type) {
+ case STATUSTYPE_INFO:
+ result[0] = '\0';
+ break;
+
+ case STATUSTYPE_TABLE:
+ snprintf(result, maxlen, "%s " SECTOR_FORMAT, lc->dev->name,
+ lc->start);
+ break;
+ }
+ return 0;
+}
+
+static struct target_type linear_target = {
+ .name = "linear",
+ .version= {1, 0, 1},
+ .module = THIS_MODULE,
+ .ctr = linear_ctr,
+ .dtr = linear_dtr,
+ .map = linear_map,
+ .status = linear_status,
+};
+
+int __init dm_linear_init(void)
+{
+ int r = dm_register_target(&linear_target);
+
+ if (r < 0)
+ DMERR("linear: register failed %d", r);
+
+ return r;
+}
+
+void dm_linear_exit(void)
+{
+ int r = dm_unregister_target(&linear_target);
+
+ if (r < 0)
+ DMERR("linear: unregister failed %d", r);
+}
diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c
new file mode 100644
index 000000000000..e110655eabdb
--- /dev/null
+++ b/drivers/md/dm-log.c
@@ -0,0 +1,711 @@
+/*
+ * Copyright (C) 2003 Sistina Software
+ *
+ * This file is released under the LGPL.
+ */
+
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/vmalloc.h>
+
+#include "dm-log.h"
+#include "dm-io.h"
+
+static LIST_HEAD(_log_types);
+static DEFINE_SPINLOCK(_lock);
+
+int dm_register_dirty_log_type(struct dirty_log_type *type)
+{
+ spin_lock(&_lock);
+ type->use_count = 0;
+ list_add(&type->list, &_log_types);
+ spin_unlock(&_lock);
+
+ return 0;
+}
+
+int dm_unregister_dirty_log_type(struct dirty_log_type *type)
+{
+ spin_lock(&_lock);
+
+ if (type->use_count)
+ DMWARN("Attempt to unregister a log type that is still in use");
+ else
+ list_del(&type->list);
+
+ spin_unlock(&_lock);
+
+ return 0;
+}
+
+static struct dirty_log_type *get_type(const char *type_name)
+{
+ struct dirty_log_type *type;
+
+ spin_lock(&_lock);
+ list_for_each_entry (type, &_log_types, list)
+ if (!strcmp(type_name, type->name)) {
+ if (!type->use_count && !try_module_get(type->module)){
+ spin_unlock(&_lock);
+ return NULL;
+ }
+ type->use_count++;
+ spin_unlock(&_lock);
+ return type;
+ }
+
+ spin_unlock(&_lock);
+ return NULL;
+}
+
+static void put_type(struct dirty_log_type *type)
+{
+ spin_lock(&_lock);
+ if (!--type->use_count)
+ module_put(type->module);
+ spin_unlock(&_lock);
+}
+
+struct dirty_log *dm_create_dirty_log(const char *type_name, struct dm_target *ti,
+ unsigned int argc, char **argv)
+{
+ struct dirty_log_type *type;
+ struct dirty_log *log;
+
+ log = kmalloc(sizeof(*log), GFP_KERNEL);
+ if (!log)
+ return NULL;
+
+ type = get_type(type_name);
+ if (!type) {
+ kfree(log);
+ return NULL;
+ }
+
+ log->type = type;
+ if (type->ctr(log, ti, argc, argv)) {
+ kfree(log);
+ put_type(type);
+ return NULL;
+ }
+
+ return log;
+}
+
+void dm_destroy_dirty_log(struct dirty_log *log)
+{
+ log->type->dtr(log);
+ put_type(log->type);
+ kfree(log);
+}
+
+/*-----------------------------------------------------------------
+ * Persistent and core logs share a lot of their implementation.
+ * FIXME: need a reload method to be called from a resume
+ *---------------------------------------------------------------*/
+/*
+ * Magic for persistent mirrors: "MiRr"
+ */
+#define MIRROR_MAGIC 0x4D695272
+
+/*
+ * The on-disk version of the metadata.
+ */
+#define MIRROR_DISK_VERSION 1
+#define LOG_OFFSET 2
+
+struct log_header {
+ uint32_t magic;
+
+ /*
+ * Simple, incrementing version. no backward
+ * compatibility.
+ */
+ uint32_t version;
+ sector_t nr_regions;
+};
+
+struct log_c {
+ struct dm_target *ti;
+ int touched;
+ uint32_t region_size;
+ unsigned int region_count;
+ region_t sync_count;
+
+ unsigned bitset_uint32_count;
+ uint32_t *clean_bits;
+ uint32_t *sync_bits;
+ uint32_t *recovering_bits; /* FIXME: this seems excessive */
+
+ int sync_search;
+
+ /* Resync flag */
+ enum sync {
+ DEFAULTSYNC, /* Synchronize if necessary */
+ NOSYNC, /* Devices known to be already in sync */
+ FORCESYNC, /* Force a sync to happen */
+ } sync;
+
+ /*
+ * Disk log fields
+ */
+ struct dm_dev *log_dev;
+ struct log_header header;
+
+ struct io_region header_location;
+ struct log_header *disk_header;
+
+ struct io_region bits_location;
+ uint32_t *disk_bits;
+};
+
+/*
+ * The touched member needs to be updated every time we access
+ * one of the bitsets.
+ */
+static inline int log_test_bit(uint32_t *bs, unsigned bit)
+{
+ return test_bit(bit, (unsigned long *) bs) ? 1 : 0;
+}
+
+static inline void log_set_bit(struct log_c *l,
+ uint32_t *bs, unsigned bit)
+{
+ set_bit(bit, (unsigned long *) bs);
+ l->touched = 1;
+}
+
+static inline void log_clear_bit(struct log_c *l,
+ uint32_t *bs, unsigned bit)
+{
+ clear_bit(bit, (unsigned long *) bs);
+ l->touched = 1;
+}
+
+/*----------------------------------------------------------------
+ * Header IO
+ *--------------------------------------------------------------*/
+static void header_to_disk(struct log_header *core, struct log_header *disk)
+{
+ disk->magic = cpu_to_le32(core->magic);
+ disk->version = cpu_to_le32(core->version);
+ disk->nr_regions = cpu_to_le64(core->nr_regions);
+}
+
+static void header_from_disk(struct log_header *core, struct log_header *disk)
+{
+ core->magic = le32_to_cpu(disk->magic);
+ core->version = le32_to_cpu(disk->version);
+ core->nr_regions = le64_to_cpu(disk->nr_regions);
+}
+
+static int read_header(struct log_c *log)
+{
+ int r;
+ unsigned long ebits;
+
+ r = dm_io_sync_vm(1, &log->header_location, READ,
+ log->disk_header, &ebits);
+ if (r)
+ return r;
+
+ header_from_disk(&log->header, log->disk_header);
+
+ /* New log required? */
+ if (log->sync != DEFAULTSYNC || log->header.magic != MIRROR_MAGIC) {
+ log->header.magic = MIRROR_MAGIC;
+ log->header.version = MIRROR_DISK_VERSION;
+ log->header.nr_regions = 0;
+ }
+
+ if (log->header.version != MIRROR_DISK_VERSION) {
+ DMWARN("incompatible disk log version");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static inline int write_header(struct log_c *log)
+{
+ unsigned long ebits;
+
+ header_to_disk(&log->header, log->disk_header);
+ return dm_io_sync_vm(1, &log->header_location, WRITE,
+ log->disk_header, &ebits);
+}
+
+/*----------------------------------------------------------------
+ * Bits IO
+ *--------------------------------------------------------------*/
+static inline void bits_to_core(uint32_t *core, uint32_t *disk, unsigned count)
+{
+ unsigned i;
+
+ for (i = 0; i < count; i++)
+ core[i] = le32_to_cpu(disk[i]);
+}
+
+static inline void bits_to_disk(uint32_t *core, uint32_t *disk, unsigned count)
+{
+ unsigned i;
+
+ /* copy across the clean/dirty bitset */
+ for (i = 0; i < count; i++)
+ disk[i] = cpu_to_le32(core[i]);
+}
+
+static int read_bits(struct log_c *log)
+{
+ int r;
+ unsigned long ebits;
+
+ r = dm_io_sync_vm(1, &log->bits_location, READ,
+ log->disk_bits, &ebits);
+ if (r)
+ return r;
+
+ bits_to_core(log->clean_bits, log->disk_bits,
+ log->bitset_uint32_count);
+ return 0;
+}
+
+static int write_bits(struct log_c *log)
+{
+ unsigned long ebits;
+ bits_to_disk(log->clean_bits, log->disk_bits,
+ log->bitset_uint32_count);
+ return dm_io_sync_vm(1, &log->bits_location, WRITE,
+ log->disk_bits, &ebits);
+}
+
+/*----------------------------------------------------------------
+ * core log constructor/destructor
+ *
+ * argv contains region_size followed optionally by [no]sync
+ *--------------------------------------------------------------*/
+#define BYTE_SHIFT 3
+static int core_ctr(struct dirty_log *log, struct dm_target *ti,
+ unsigned int argc, char **argv)
+{
+ enum sync sync = DEFAULTSYNC;
+
+ struct log_c *lc;
+ uint32_t region_size;
+ unsigned int region_count;
+ size_t bitset_size;
+
+ if (argc < 1 || argc > 2) {
+ DMWARN("wrong number of arguments to mirror log");
+ return -EINVAL;
+ }
+
+ if (argc > 1) {
+ if (!strcmp(argv[1], "sync"))
+ sync = FORCESYNC;
+ else if (!strcmp(argv[1], "nosync"))
+ sync = NOSYNC;
+ else {
+ DMWARN("unrecognised sync argument to mirror log: %s",
+ argv[1]);
+ return -EINVAL;
+ }
+ }
+
+ if (sscanf(argv[0], "%u", &region_size) != 1) {
+ DMWARN("invalid region size string");
+ return -EINVAL;
+ }
+
+ region_count = dm_sector_div_up(ti->len, region_size);
+
+ lc = kmalloc(sizeof(*lc), GFP_KERNEL);
+ if (!lc) {
+ DMWARN("couldn't allocate core log");
+ return -ENOMEM;
+ }
+
+ lc->ti = ti;
+ lc->touched = 0;
+ lc->region_size = region_size;
+ lc->region_count = region_count;
+ lc->sync = sync;
+
+ /*
+ * Work out how many words we need to hold the bitset.
+ */
+ bitset_size = dm_round_up(region_count,
+ sizeof(*lc->clean_bits) << BYTE_SHIFT);
+ bitset_size >>= BYTE_SHIFT;
+
+ lc->bitset_uint32_count = bitset_size / 4;
+ lc->clean_bits = vmalloc(bitset_size);
+ if (!lc->clean_bits) {
+ DMWARN("couldn't allocate clean bitset");
+ kfree(lc);
+ return -ENOMEM;
+ }
+ memset(lc->clean_bits, -1, bitset_size);
+
+ lc->sync_bits = vmalloc(bitset_size);
+ if (!lc->sync_bits) {
+ DMWARN("couldn't allocate sync bitset");
+ vfree(lc->clean_bits);
+ kfree(lc);
+ return -ENOMEM;
+ }
+ memset(lc->sync_bits, (sync == NOSYNC) ? -1 : 0, bitset_size);
+ lc->sync_count = (sync == NOSYNC) ? region_count : 0;
+
+ lc->recovering_bits = vmalloc(bitset_size);
+ if (!lc->recovering_bits) {
+ DMWARN("couldn't allocate sync bitset");
+ vfree(lc->sync_bits);
+ vfree(lc->clean_bits);
+ kfree(lc);
+ return -ENOMEM;
+ }
+ memset(lc->recovering_bits, 0, bitset_size);
+ lc->sync_search = 0;
+ log->context = lc;
+ return 0;
+}
+
+static void core_dtr(struct dirty_log *log)
+{
+ struct log_c *lc = (struct log_c *) log->context;
+ vfree(lc->clean_bits);
+ vfree(lc->sync_bits);
+ vfree(lc->recovering_bits);
+ kfree(lc);
+}
+
+/*----------------------------------------------------------------
+ * disk log constructor/destructor
+ *
+ * argv contains log_device region_size followed optionally by [no]sync
+ *--------------------------------------------------------------*/
+static int disk_ctr(struct dirty_log *log, struct dm_target *ti,
+ unsigned int argc, char **argv)
+{
+ int r;
+ size_t size;
+ struct log_c *lc;
+ struct dm_dev *dev;
+
+ if (argc < 2 || argc > 3) {
+ DMWARN("wrong number of arguments to disk mirror log");
+ return -EINVAL;
+ }
+
+ r = dm_get_device(ti, argv[0], 0, 0 /* FIXME */,
+ FMODE_READ | FMODE_WRITE, &dev);
+ if (r)
+ return r;
+
+ r = core_ctr(log, ti, argc - 1, argv + 1);
+ if (r) {
+ dm_put_device(ti, dev);
+ return r;
+ }
+
+ lc = (struct log_c *) log->context;
+ lc->log_dev = dev;
+
+ /* setup the disk header fields */
+ lc->header_location.bdev = lc->log_dev->bdev;
+ lc->header_location.sector = 0;
+ lc->header_location.count = 1;
+
+ /*
+ * We can't read less than this amount, even though we'll
+ * not be using most of this space.
+ */
+ lc->disk_header = vmalloc(1 << SECTOR_SHIFT);
+ if (!lc->disk_header)
+ goto bad;
+
+ /* setup the disk bitset fields */
+ lc->bits_location.bdev = lc->log_dev->bdev;
+ lc->bits_location.sector = LOG_OFFSET;
+
+ size = dm_round_up(lc->bitset_uint32_count * sizeof(uint32_t),
+ 1 << SECTOR_SHIFT);
+ lc->bits_location.count = size >> SECTOR_SHIFT;
+ lc->disk_bits = vmalloc(size);
+ if (!lc->disk_bits) {
+ vfree(lc->disk_header);
+ goto bad;
+ }
+ return 0;
+
+ bad:
+ dm_put_device(ti, lc->log_dev);
+ core_dtr(log);
+ return -ENOMEM;
+}
+
+static void disk_dtr(struct dirty_log *log)
+{
+ struct log_c *lc = (struct log_c *) log->context;
+ dm_put_device(lc->ti, lc->log_dev);
+ vfree(lc->disk_header);
+ vfree(lc->disk_bits);
+ core_dtr(log);
+}
+
+static int count_bits32(uint32_t *addr, unsigned size)
+{
+ int count = 0, i;
+
+ for (i = 0; i < size; i++) {
+ count += hweight32(*(addr+i));
+ }
+ return count;
+}
+
+static int disk_resume(struct dirty_log *log)
+{
+ int r;
+ unsigned i;
+ struct log_c *lc = (struct log_c *) log->context;
+ size_t size = lc->bitset_uint32_count * sizeof(uint32_t);
+
+ /* read the disk header */
+ r = read_header(lc);
+ if (r)
+ return r;
+
+ /* read the bits */
+ r = read_bits(lc);
+ if (r)
+ return r;
+
+ /* set or clear any new bits */
+ if (lc->sync == NOSYNC)
+ for (i = lc->header.nr_regions; i < lc->region_count; i++)
+ /* FIXME: amazingly inefficient */
+ log_set_bit(lc, lc->clean_bits, i);
+ else
+ for (i = lc->header.nr_regions; i < lc->region_count; i++)
+ /* FIXME: amazingly inefficient */
+ log_clear_bit(lc, lc->clean_bits, i);
+
+ /* copy clean across to sync */
+ memcpy(lc->sync_bits, lc->clean_bits, size);
+ lc->sync_count = count_bits32(lc->clean_bits, lc->bitset_uint32_count);
+
+ /* write the bits */
+ r = write_bits(lc);
+ if (r)
+ return r;
+
+ /* set the correct number of regions in the header */
+ lc->header.nr_regions = lc->region_count;
+
+ /* write the new header */
+ return write_header(lc);
+}
+
+static uint32_t core_get_region_size(struct dirty_log *log)
+{
+ struct log_c *lc = (struct log_c *) log->context;
+ return lc->region_size;
+}
+
+static int core_is_clean(struct dirty_log *log, region_t region)
+{
+ struct log_c *lc = (struct log_c *) log->context;
+ return log_test_bit(lc->clean_bits, region);
+}
+
+static int core_in_sync(struct dirty_log *log, region_t region, int block)
+{
+ struct log_c *lc = (struct log_c *) log->context;
+ return log_test_bit(lc->sync_bits, region);
+}
+
+static int core_flush(struct dirty_log *log)
+{
+ /* no op */
+ return 0;
+}
+
+static int disk_flush(struct dirty_log *log)
+{
+ int r;
+ struct log_c *lc = (struct log_c *) log->context;
+
+ /* only write if the log has changed */
+ if (!lc->touched)
+ return 0;
+
+ r = write_bits(lc);
+ if (!r)
+ lc->touched = 0;
+
+ return r;
+}
+
+static void core_mark_region(struct dirty_log *log, region_t region)
+{
+ struct log_c *lc = (struct log_c *) log->context;
+ log_clear_bit(lc, lc->clean_bits, region);
+}
+
+static void core_clear_region(struct dirty_log *log, region_t region)
+{
+ struct log_c *lc = (struct log_c *) log->context;
+ log_set_bit(lc, lc->clean_bits, region);
+}
+
+static int core_get_resync_work(struct dirty_log *log, region_t *region)
+{
+ struct log_c *lc = (struct log_c *) log->context;
+
+ if (lc->sync_search >= lc->region_count)
+ return 0;
+
+ do {
+ *region = find_next_zero_bit((unsigned long *) lc->sync_bits,
+ lc->region_count,
+ lc->sync_search);
+ lc->sync_search = *region + 1;
+
+ if (*region == lc->region_count)
+ return 0;
+
+ } while (log_test_bit(lc->recovering_bits, *region));
+
+ log_set_bit(lc, lc->recovering_bits, *region);
+ return 1;
+}
+
+static void core_complete_resync_work(struct dirty_log *log, region_t region,
+ int success)
+{
+ struct log_c *lc = (struct log_c *) log->context;
+
+ log_clear_bit(lc, lc->recovering_bits, region);
+ if (success) {
+ log_set_bit(lc, lc->sync_bits, region);
+ lc->sync_count++;
+ }
+}
+
+static region_t core_get_sync_count(struct dirty_log *log)
+{
+ struct log_c *lc = (struct log_c *) log->context;
+
+ return lc->sync_count;
+}
+
+#define DMEMIT_SYNC \
+ if (lc->sync != DEFAULTSYNC) \
+ DMEMIT("%ssync ", lc->sync == NOSYNC ? "no" : "")
+
+static int core_status(struct dirty_log *log, status_type_t status,
+ char *result, unsigned int maxlen)
+{
+ int sz = 0;
+ struct log_c *lc = log->context;
+
+ switch(status) {
+ case STATUSTYPE_INFO:
+ break;
+
+ case STATUSTYPE_TABLE:
+ DMEMIT("%s %u %u ", log->type->name,
+ lc->sync == DEFAULTSYNC ? 1 : 2, lc->region_size);
+ DMEMIT_SYNC;
+ }
+
+ return sz;
+}
+
+static int disk_status(struct dirty_log *log, status_type_t status,
+ char *result, unsigned int maxlen)
+{
+ int sz = 0;
+ char buffer[16];
+ struct log_c *lc = log->context;
+
+ switch(status) {
+ case STATUSTYPE_INFO:
+ break;
+
+ case STATUSTYPE_TABLE:
+ format_dev_t(buffer, lc->log_dev->bdev->bd_dev);
+ DMEMIT("%s %u %s %u ", log->type->name,
+ lc->sync == DEFAULTSYNC ? 2 : 3, buffer,
+ lc->region_size);
+ DMEMIT_SYNC;
+ }
+
+ return sz;
+}
+
+static struct dirty_log_type _core_type = {
+ .name = "core",
+ .module = THIS_MODULE,
+ .ctr = core_ctr,
+ .dtr = core_dtr,
+ .get_region_size = core_get_region_size,
+ .is_clean = core_is_clean,
+ .in_sync = core_in_sync,
+ .flush = core_flush,
+ .mark_region = core_mark_region,
+ .clear_region = core_clear_region,
+ .get_resync_work = core_get_resync_work,
+ .complete_resync_work = core_complete_resync_work,
+ .get_sync_count = core_get_sync_count,
+ .status = core_status,
+};
+
+static struct dirty_log_type _disk_type = {
+ .name = "disk",
+ .module = THIS_MODULE,
+ .ctr = disk_ctr,
+ .dtr = disk_dtr,
+ .suspend = disk_flush,
+ .resume = disk_resume,
+ .get_region_size = core_get_region_size,
+ .is_clean = core_is_clean,
+ .in_sync = core_in_sync,
+ .flush = disk_flush,
+ .mark_region = core_mark_region,
+ .clear_region = core_clear_region,
+ .get_resync_work = core_get_resync_work,
+ .complete_resync_work = core_complete_resync_work,
+ .get_sync_count = core_get_sync_count,
+ .status = disk_status,
+};
+
+int __init dm_dirty_log_init(void)
+{
+ int r;
+
+ r = dm_register_dirty_log_type(&_core_type);
+ if (r)
+ DMWARN("couldn't register core log");
+
+ r = dm_register_dirty_log_type(&_disk_type);
+ if (r) {
+ DMWARN("couldn't register disk type");
+ dm_unregister_dirty_log_type(&_core_type);
+ }
+
+ return r;
+}
+
+void dm_dirty_log_exit(void)
+{
+ dm_unregister_dirty_log_type(&_disk_type);
+ dm_unregister_dirty_log_type(&_core_type);
+}
+
+EXPORT_SYMBOL(dm_register_dirty_log_type);
+EXPORT_SYMBOL(dm_unregister_dirty_log_type);
+EXPORT_SYMBOL(dm_create_dirty_log);
+EXPORT_SYMBOL(dm_destroy_dirty_log);
diff --git a/drivers/md/dm-log.h b/drivers/md/dm-log.h
new file mode 100644
index 000000000000..5ae5309ebf28
--- /dev/null
+++ b/drivers/md/dm-log.h
@@ -0,0 +1,130 @@
+/*
+ * Copyright (C) 2003 Sistina Software
+ *
+ * This file is released under the LGPL.
+ */
+
+#ifndef DM_DIRTY_LOG
+#define DM_DIRTY_LOG
+
+#include "dm.h"
+
+typedef sector_t region_t;
+
+struct dirty_log_type;
+
+struct dirty_log {
+ struct dirty_log_type *type;
+ void *context;
+};
+
+struct dirty_log_type {
+ struct list_head list;
+ const char *name;
+ struct module *module;
+ unsigned int use_count;
+
+ int (*ctr)(struct dirty_log *log, struct dm_target *ti,
+ unsigned int argc, char **argv);
+ void (*dtr)(struct dirty_log *log);
+
+ /*
+ * There are times when we don't want the log to touch
+ * the disk.
+ */
+ int (*suspend)(struct dirty_log *log);
+ int (*resume)(struct dirty_log *log);
+
+ /*
+ * Retrieves the smallest size of region that the log can
+ * deal with.
+ */
+ uint32_t (*get_region_size)(struct dirty_log *log);
+
+ /*
+ * A predicate to say whether a region is clean or not.
+ * May block.
+ */
+ int (*is_clean)(struct dirty_log *log, region_t region);
+
+ /*
+ * Returns: 0, 1, -EWOULDBLOCK, < 0
+ *
+ * A predicate function to check the area given by
+ * [sector, sector + len) is in sync.
+ *
+ * If -EWOULDBLOCK is returned the state of the region is
+ * unknown, typically this will result in a read being
+ * passed to a daemon to deal with, since a daemon is
+ * allowed to block.
+ */
+ int (*in_sync)(struct dirty_log *log, region_t region, int can_block);
+
+ /*
+ * Flush the current log state (eg, to disk). This
+ * function may block.
+ */
+ int (*flush)(struct dirty_log *log);
+
+ /*
+ * Mark an area as clean or dirty. These functions may
+ * block, though for performance reasons blocking should
+ * be extremely rare (eg, allocating another chunk of
+ * memory for some reason).
+ */
+ void (*mark_region)(struct dirty_log *log, region_t region);
+ void (*clear_region)(struct dirty_log *log, region_t region);
+
+ /*
+ * Returns: <0 (error), 0 (no region), 1 (region)
+ *
+ * The mirrord will need perform recovery on regions of
+ * the mirror that are in the NOSYNC state. This
+ * function asks the log to tell the caller about the
+ * next region that this machine should recover.
+ *
+ * Do not confuse this function with 'in_sync()', one
+ * tells you if an area is synchronised, the other
+ * assigns recovery work.
+ */
+ int (*get_resync_work)(struct dirty_log *log, region_t *region);
+
+ /*
+ * This notifies the log that the resync of an area has
+ * been completed. The log should then mark this region
+ * as CLEAN.
+ */
+ void (*complete_resync_work)(struct dirty_log *log,
+ region_t region, int success);
+
+ /*
+ * Returns the number of regions that are in sync.
+ */
+ region_t (*get_sync_count)(struct dirty_log *log);
+
+ /*
+ * Support function for mirror status requests.
+ */
+ int (*status)(struct dirty_log *log, status_type_t status_type,
+ char *result, unsigned int maxlen);
+};
+
+int dm_register_dirty_log_type(struct dirty_log_type *type);
+int dm_unregister_dirty_log_type(struct dirty_log_type *type);
+
+
+/*
+ * Make sure you use these two functions, rather than calling
+ * type->constructor/destructor() directly.
+ */
+struct dirty_log *dm_create_dirty_log(const char *type_name, struct dm_target *ti,
+ unsigned int argc, char **argv);
+void dm_destroy_dirty_log(struct dirty_log *log);
+
+/*
+ * init/exit functions.
+ */
+int dm_dirty_log_init(void);
+void dm_dirty_log_exit(void);
+
+#endif
diff --git a/drivers/md/dm-mpath.c b/drivers/md/dm-mpath.c
new file mode 100644
index 000000000000..43763a0bd096
--- /dev/null
+++ b/drivers/md/dm-mpath.c
@@ -0,0 +1,1302 @@
+/*
+ * Copyright (C) 2003 Sistina Software Limited.
+ * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+#include "dm-path-selector.h"
+#include "dm-hw-handler.h"
+#include "dm-bio-list.h"
+#include "dm-bio-record.h"
+
+#include <linux/ctype.h>
+#include <linux/init.h>
+#include <linux/mempool.h>
+#include <linux/module.h>
+#include <linux/pagemap.h>
+#include <linux/slab.h>
+#include <linux/time.h>
+#include <linux/workqueue.h>
+#include <asm/atomic.h>
+
+#define MESG_STR(x) x, sizeof(x)
+
+/* Path properties */
+struct pgpath {
+ struct list_head list;
+
+ struct priority_group *pg; /* Owning PG */
+ unsigned fail_count; /* Cumulative failure count */
+
+ struct path path;
+};
+
+#define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)
+
+/*
+ * Paths are grouped into Priority Groups and numbered from 1 upwards.
+ * Each has a path selector which controls which path gets used.
+ */
+struct priority_group {
+ struct list_head list;
+
+ struct multipath *m; /* Owning multipath instance */
+ struct path_selector ps;
+
+ unsigned pg_num; /* Reference number */
+ unsigned bypassed; /* Temporarily bypass this PG? */
+
+ unsigned nr_pgpaths; /* Number of paths in PG */
+ struct list_head pgpaths;
+};
+
+/* Multipath context */
+struct multipath {
+ struct list_head list;
+ struct dm_target *ti;
+
+ spinlock_t lock;
+
+ struct hw_handler hw_handler;
+ unsigned nr_priority_groups;
+ struct list_head priority_groups;
+ unsigned pg_init_required; /* pg_init needs calling? */
+
+ unsigned nr_valid_paths; /* Total number of usable paths */
+ struct pgpath *current_pgpath;
+ struct priority_group *current_pg;
+ struct priority_group *next_pg; /* Switch to this PG if set */
+ unsigned repeat_count; /* I/Os left before calling PS again */
+
+ unsigned queue_io; /* Must we queue all I/O? */
+ unsigned queue_if_no_path; /* Queue I/O if last path fails? */
+ unsigned suspended; /* Has dm core suspended our I/O? */
+
+ struct work_struct process_queued_ios;
+ struct bio_list queued_ios;
+ unsigned queue_size;
+
+ struct work_struct trigger_event;
+
+ /*
+ * We must use a mempool of mpath_io structs so that we
+ * can resubmit bios on error.
+ */
+ mempool_t *mpio_pool;
+};
+
+/*
+ * Context information attached to each bio we process.
+ */
+struct mpath_io {
+ struct pgpath *pgpath;
+ struct dm_bio_details details;
+};
+
+typedef int (*action_fn) (struct pgpath *pgpath);
+
+#define MIN_IOS 256 /* Mempool size */
+
+static kmem_cache_t *_mpio_cache;
+
+static void process_queued_ios(void *data);
+static void trigger_event(void *data);
+
+
+/*-----------------------------------------------
+ * Allocation routines
+ *-----------------------------------------------*/
+
+static struct pgpath *alloc_pgpath(void)
+{
+ struct pgpath *pgpath = kmalloc(sizeof(*pgpath), GFP_KERNEL);
+
+ if (pgpath) {
+ memset(pgpath, 0, sizeof(*pgpath));
+ pgpath->path.is_active = 1;
+ }
+
+ return pgpath;
+}
+
+static inline void free_pgpath(struct pgpath *pgpath)
+{
+ kfree(pgpath);
+}
+
+static struct priority_group *alloc_priority_group(void)
+{
+ struct priority_group *pg;
+
+ pg = kmalloc(sizeof(*pg), GFP_KERNEL);
+ if (!pg)
+ return NULL;
+
+ memset(pg, 0, sizeof(*pg));
+ INIT_LIST_HEAD(&pg->pgpaths);
+
+ return pg;
+}
+
+static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
+{
+ struct pgpath *pgpath, *tmp;
+
+ list_for_each_entry_safe(pgpath, tmp, pgpaths, list) {
+ list_del(&pgpath->list);
+ dm_put_device(ti, pgpath->path.dev);
+ free_pgpath(pgpath);
+ }
+}
+
+static void free_priority_group(struct priority_group *pg,
+ struct dm_target *ti)
+{
+ struct path_selector *ps = &pg->ps;
+
+ if (ps->type) {
+ ps->type->destroy(ps);
+ dm_put_path_selector(ps->type);
+ }
+
+ free_pgpaths(&pg->pgpaths, ti);
+ kfree(pg);
+}
+
+static struct multipath *alloc_multipath(void)
+{
+ struct multipath *m;
+
+ m = kmalloc(sizeof(*m), GFP_KERNEL);
+ if (m) {
+ memset(m, 0, sizeof(*m));
+ INIT_LIST_HEAD(&m->priority_groups);
+ spin_lock_init(&m->lock);
+ m->queue_io = 1;
+ INIT_WORK(&m->process_queued_ios, process_queued_ios, m);
+ INIT_WORK(&m->trigger_event, trigger_event, m);
+ m->mpio_pool = mempool_create(MIN_IOS, mempool_alloc_slab,
+ mempool_free_slab, _mpio_cache);
+ if (!m->mpio_pool) {
+ kfree(m);
+ return NULL;
+ }
+ }
+
+ return m;
+}
+
+static void free_multipath(struct multipath *m)
+{
+ struct priority_group *pg, *tmp;
+ struct hw_handler *hwh = &m->hw_handler;
+
+ list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
+ list_del(&pg->list);
+ free_priority_group(pg, m->ti);
+ }
+
+ if (hwh->type) {
+ hwh->type->destroy(hwh);
+ dm_put_hw_handler(hwh->type);
+ }
+
+ mempool_destroy(m->mpio_pool);
+ kfree(m);
+}
+
+
+/*-----------------------------------------------
+ * Path selection
+ *-----------------------------------------------*/
+
+static void __switch_pg(struct multipath *m, struct pgpath *pgpath)
+{
+ struct hw_handler *hwh = &m->hw_handler;
+
+ m->current_pg = pgpath->pg;
+
+ /* Must we initialise the PG first, and queue I/O till it's ready? */
+ if (hwh->type && hwh->type->pg_init) {
+ m->pg_init_required = 1;
+ m->queue_io = 1;
+ } else {
+ m->pg_init_required = 0;
+ m->queue_io = 0;
+ }
+}
+
+static int __choose_path_in_pg(struct multipath *m, struct priority_group *pg)
+{
+ struct path *path;
+
+ path = pg->ps.type->select_path(&pg->ps, &m->repeat_count);
+ if (!path)
+ return -ENXIO;
+
+ m->current_pgpath = path_to_pgpath(path);
+
+ if (m->current_pg != pg)
+ __switch_pg(m, m->current_pgpath);
+
+ return 0;
+}
+
+static void __choose_pgpath(struct multipath *m)
+{
+ struct priority_group *pg;
+ unsigned bypassed = 1;
+
+ if (!m->nr_valid_paths)
+ goto failed;
+
+ /* Were we instructed to switch PG? */
+ if (m->next_pg) {
+ pg = m->next_pg;
+ m->next_pg = NULL;
+ if (!__choose_path_in_pg(m, pg))
+ return;
+ }
+
+ /* Don't change PG until it has no remaining paths */
+ if (m->current_pg && !__choose_path_in_pg(m, m->current_pg))
+ return;
+
+ /*
+ * Loop through priority groups until we find a valid path.
+ * First time we skip PGs marked 'bypassed'.
+ * Second time we only try the ones we skipped.
+ */
+ do {
+ list_for_each_entry(pg, &m->priority_groups, list) {
+ if (pg->bypassed == bypassed)
+ continue;
+ if (!__choose_path_in_pg(m, pg))
+ return;
+ }
+ } while (bypassed--);
+
+failed:
+ m->current_pgpath = NULL;
+ m->current_pg = NULL;
+}
+
+static int map_io(struct multipath *m, struct bio *bio, struct mpath_io *mpio,
+ unsigned was_queued)
+{
+ int r = 1;
+ unsigned long flags;
+ struct pgpath *pgpath;
+
+ spin_lock_irqsave(&m->lock, flags);
+
+ /* Do we need to select a new pgpath? */
+ if (!m->current_pgpath ||
+ (!m->queue_io && (m->repeat_count && --m->repeat_count == 0)))
+ __choose_pgpath(m);
+
+ pgpath = m->current_pgpath;
+
+ if (was_queued)
+ m->queue_size--;
+
+ if ((pgpath && m->queue_io) ||
+ (!pgpath && m->queue_if_no_path && !m->suspended)) {
+ /* Queue for the daemon to resubmit */
+ bio_list_add(&m->queued_ios, bio);
+ m->queue_size++;
+ if (m->pg_init_required || !m->queue_io)
+ schedule_work(&m->process_queued_ios);
+ pgpath = NULL;
+ r = 0;
+ } else if (!pgpath)
+ r = -EIO; /* Failed */
+ else
+ bio->bi_bdev = pgpath->path.dev->bdev;
+
+ mpio->pgpath = pgpath;
+
+ spin_unlock_irqrestore(&m->lock, flags);
+
+ return r;
+}
+
+/*
+ * If we run out of usable paths, should we queue I/O or error it?
+ */
+static int queue_if_no_path(struct multipath *m, unsigned queue_if_no_path)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&m->lock, flags);
+
+ m->queue_if_no_path = queue_if_no_path;
+ if (!m->queue_if_no_path)
+ schedule_work(&m->process_queued_ios);
+
+ spin_unlock_irqrestore(&m->lock, flags);
+
+ return 0;
+}
+
+/*-----------------------------------------------------------------
+ * The multipath daemon is responsible for resubmitting queued ios.
+ *---------------------------------------------------------------*/
+
+static void dispatch_queued_ios(struct multipath *m)
+{
+ int r;
+ unsigned long flags;
+ struct bio *bio = NULL, *next;
+ struct mpath_io *mpio;
+ union map_info *info;
+
+ spin_lock_irqsave(&m->lock, flags);
+ bio = bio_list_get(&m->queued_ios);
+ spin_unlock_irqrestore(&m->lock, flags);
+
+ while (bio) {
+ next = bio->bi_next;
+ bio->bi_next = NULL;
+
+ info = dm_get_mapinfo(bio);
+ mpio = info->ptr;
+
+ r = map_io(m, bio, mpio, 1);
+ if (r < 0)
+ bio_endio(bio, bio->bi_size, r);
+ else if (r == 1)
+ generic_make_request(bio);
+
+ bio = next;
+ }
+}
+
+static void process_queued_ios(void *data)
+{
+ struct multipath *m = (struct multipath *) data;
+ struct hw_handler *hwh = &m->hw_handler;
+ struct pgpath *pgpath;
+ unsigned init_required, must_queue = 0;
+ unsigned long flags;
+
+ spin_lock_irqsave(&m->lock, flags);
+
+ if (!m->current_pgpath)
+ __choose_pgpath(m);
+
+ pgpath = m->current_pgpath;
+
+ if ((pgpath && m->queue_io) ||
+ (!pgpath && m->queue_if_no_path && !m->suspended))
+ must_queue = 1;
+
+ init_required = m->pg_init_required;
+ if (init_required)
+ m->pg_init_required = 0;
+
+ spin_unlock_irqrestore(&m->lock, flags);
+
+ if (init_required)
+ hwh->type->pg_init(hwh, pgpath->pg->bypassed, &pgpath->path);
+
+ if (!must_queue)
+ dispatch_queued_ios(m);
+}
+
+/*
+ * An event is triggered whenever a path is taken out of use.
+ * Includes path failure and PG bypass.
+ */
+static void trigger_event(void *data)
+{
+ struct multipath *m = (struct multipath *) data;
+
+ dm_table_event(m->ti->table);
+}
+
+/*-----------------------------------------------------------------
+ * Constructor/argument parsing:
+ * <#multipath feature args> [<arg>]*
+ * <#hw_handler args> [hw_handler [<arg>]*]
+ * <#priority groups>
+ * <initial priority group>
+ * [<selector> <#selector args> [<arg>]*
+ * <#paths> <#per-path selector args>
+ * [<path> [<arg>]* ]+ ]+
+ *---------------------------------------------------------------*/
+struct param {
+ unsigned min;
+ unsigned max;
+ char *error;
+};
+
+#define ESTR(s) ("dm-multipath: " s)
+
+static int read_param(struct param *param, char *str, unsigned *v, char **error)
+{
+ if (!str ||
+ (sscanf(str, "%u", v) != 1) ||
+ (*v < param->min) ||
+ (*v > param->max)) {
+ *error = param->error;
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+struct arg_set {
+ unsigned argc;
+ char **argv;
+};
+
+static char *shift(struct arg_set *as)
+{
+ char *r;
+
+ if (as->argc) {
+ as->argc--;
+ r = *as->argv;
+ as->argv++;
+ return r;
+ }
+
+ return NULL;
+}
+
+static void consume(struct arg_set *as, unsigned n)
+{
+ BUG_ON (as->argc < n);
+ as->argc -= n;
+ as->argv += n;
+}
+
+static int parse_path_selector(struct arg_set *as, struct priority_group *pg,
+ struct dm_target *ti)
+{
+ int r;
+ struct path_selector_type *pst;
+ unsigned ps_argc;
+
+ static struct param _params[] = {
+ {0, 1024, ESTR("invalid number of path selector args")},
+ };
+
+ pst = dm_get_path_selector(shift(as));
+ if (!pst) {
+ ti->error = ESTR("unknown path selector type");
+ return -EINVAL;
+ }
+
+ r = read_param(_params, shift(as), &ps_argc, &ti->error);
+ if (r)
+ return -EINVAL;
+
+ r = pst->create(&pg->ps, ps_argc, as->argv);
+ if (r) {
+ dm_put_path_selector(pst);
+ ti->error = ESTR("path selector constructor failed");
+ return r;
+ }
+
+ pg->ps.type = pst;
+ consume(as, ps_argc);
+
+ return 0;
+}
+
+static struct pgpath *parse_path(struct arg_set *as, struct path_selector *ps,
+ struct dm_target *ti)
+{
+ int r;
+ struct pgpath *p;
+
+ /* we need at least a path arg */
+ if (as->argc < 1) {
+ ti->error = ESTR("no device given");
+ return NULL;
+ }
+
+ p = alloc_pgpath();
+ if (!p)
+ return NULL;
+
+ r = dm_get_device(ti, shift(as), ti->begin, ti->len,
+ dm_table_get_mode(ti->table), &p->path.dev);
+ if (r) {
+ ti->error = ESTR("error getting device");
+ goto bad;
+ }
+
+ r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
+ if (r) {
+ dm_put_device(ti, p->path.dev);
+ goto bad;
+ }
+
+ return p;
+
+ bad:
+ free_pgpath(p);
+ return NULL;
+}
+
+static struct priority_group *parse_priority_group(struct arg_set *as,
+ struct multipath *m,
+ struct dm_target *ti)
+{
+ static struct param _params[] = {
+ {1, 1024, ESTR("invalid number of paths")},
+ {0, 1024, ESTR("invalid number of selector args")}
+ };
+
+ int r;
+ unsigned i, nr_selector_args, nr_params;
+ struct priority_group *pg;
+
+ if (as->argc < 2) {
+ as->argc = 0;
+ ti->error = ESTR("not enough priority group aruments");
+ return NULL;
+ }
+
+ pg = alloc_priority_group();
+ if (!pg) {
+ ti->error = ESTR("couldn't allocate priority group");
+ return NULL;
+ }
+ pg->m = m;
+
+ r = parse_path_selector(as, pg, ti);
+ if (r)
+ goto bad;
+
+ /*
+ * read the paths
+ */
+ r = read_param(_params, shift(as), &pg->nr_pgpaths, &ti->error);
+ if (r)
+ goto bad;
+
+ r = read_param(_params + 1, shift(as), &nr_selector_args, &ti->error);
+ if (r)
+ goto bad;
+
+ nr_params = 1 + nr_selector_args;
+ for (i = 0; i < pg->nr_pgpaths; i++) {
+ struct pgpath *pgpath;
+ struct arg_set path_args;
+
+ if (as->argc < nr_params)
+ goto bad;
+
+ path_args.argc = nr_params;
+ path_args.argv = as->argv;
+
+ pgpath = parse_path(&path_args, &pg->ps, ti);
+ if (!pgpath)
+ goto bad;
+
+ pgpath->pg = pg;
+ list_add_tail(&pgpath->list, &pg->pgpaths);
+ consume(as, nr_params);
+ }
+
+ return pg;
+
+ bad:
+ free_priority_group(pg, ti);
+ return NULL;
+}
+
+static int parse_hw_handler(struct arg_set *as, struct multipath *m,
+ struct dm_target *ti)
+{
+ int r;
+ struct hw_handler_type *hwht;
+ unsigned hw_argc;
+
+ static struct param _params[] = {
+ {0, 1024, ESTR("invalid number of hardware handler args")},
+ };
+
+ r = read_param(_params, shift(as), &hw_argc, &ti->error);
+ if (r)
+ return -EINVAL;
+
+ if (!hw_argc)
+ return 0;
+
+ hwht = dm_get_hw_handler(shift(as));
+ if (!hwht) {
+ ti->error = ESTR("unknown hardware handler type");
+ return -EINVAL;
+ }
+
+ r = hwht->create(&m->hw_handler, hw_argc - 1, as->argv);
+ if (r) {
+ dm_put_hw_handler(hwht);
+ ti->error = ESTR("hardware handler constructor failed");
+ return r;
+ }
+
+ m->hw_handler.type = hwht;
+ consume(as, hw_argc - 1);
+
+ return 0;
+}
+
+static int parse_features(struct arg_set *as, struct multipath *m,
+ struct dm_target *ti)
+{
+ int r;
+ unsigned argc;
+
+ static struct param _params[] = {
+ {0, 1, ESTR("invalid number of feature args")},
+ };
+
+ r = read_param(_params, shift(as), &argc, &ti->error);
+ if (r)
+ return -EINVAL;
+
+ if (!argc)
+ return 0;
+
+ if (!strnicmp(shift(as), MESG_STR("queue_if_no_path")))
+ return queue_if_no_path(m, 1);
+ else {
+ ti->error = "Unrecognised multipath feature request";
+ return -EINVAL;
+ }
+}
+
+static int multipath_ctr(struct dm_target *ti, unsigned int argc,
+ char **argv)
+{
+ /* target parameters */
+ static struct param _params[] = {
+ {1, 1024, ESTR("invalid number of priority groups")},
+ {1, 1024, ESTR("invalid initial priority group number")},
+ };
+
+ int r;
+ struct multipath *m;
+ struct arg_set as;
+ unsigned pg_count = 0;
+ unsigned next_pg_num;
+
+ as.argc = argc;
+ as.argv = argv;
+
+ m = alloc_multipath();
+ if (!m) {
+ ti->error = ESTR("can't allocate multipath");
+ return -EINVAL;
+ }
+
+ r = parse_features(&as, m, ti);
+ if (r)
+ goto bad;
+
+ r = parse_hw_handler(&as, m, ti);
+ if (r)
+ goto bad;
+
+ r = read_param(_params, shift(&as), &m->nr_priority_groups, &ti->error);
+ if (r)
+ goto bad;
+
+ r = read_param(_params + 1, shift(&as), &next_pg_num, &ti->error);
+ if (r)
+ goto bad;
+
+ /* parse the priority groups */
+ while (as.argc) {
+ struct priority_group *pg;
+
+ pg = parse_priority_group(&as, m, ti);
+ if (!pg) {
+ r = -EINVAL;
+ goto bad;
+ }
+
+ m->nr_valid_paths += pg->nr_pgpaths;
+ list_add_tail(&pg->list, &m->priority_groups);
+ pg_count++;
+ pg->pg_num = pg_count;
+ if (!--next_pg_num)
+ m->next_pg = pg;
+ }
+
+ if (pg_count != m->nr_priority_groups) {
+ ti->error = ESTR("priority group count mismatch");
+ r = -EINVAL;
+ goto bad;
+ }
+
+ ti->private = m;
+ m->ti = ti;
+
+ return 0;
+
+ bad:
+ free_multipath(m);
+ return r;
+}
+
+static void multipath_dtr(struct dm_target *ti)
+{
+ struct multipath *m = (struct multipath *) ti->private;
+ free_multipath(m);
+}
+
+/*
+ * Map bios, recording original fields for later in case we have to resubmit
+ */
+static int multipath_map(struct dm_target *ti, struct bio *bio,
+ union map_info *map_context)
+{
+ int r;
+ struct mpath_io *mpio;
+ struct multipath *m = (struct multipath *) ti->private;
+
+ mpio = mempool_alloc(m->mpio_pool, GFP_NOIO);
+ dm_bio_record(&mpio->details, bio);
+
+ map_context->ptr = mpio;
+ bio->bi_rw |= (1 << BIO_RW_FAILFAST);
+ r = map_io(m, bio, mpio, 0);
+ if (r < 0)
+ mempool_free(mpio, m->mpio_pool);
+
+ return r;
+}
+
+/*
+ * Take a path out of use.
+ */
+static int fail_path(struct pgpath *pgpath)
+{
+ unsigned long flags;
+ struct multipath *m = pgpath->pg->m;
+
+ spin_lock_irqsave(&m->lock, flags);
+
+ if (!pgpath->path.is_active)
+ goto out;
+
+ DMWARN("dm-multipath: Failing path %s.", pgpath->path.dev->name);
+
+ pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
+ pgpath->path.is_active = 0;
+ pgpath->fail_count++;
+
+ m->nr_valid_paths--;
+
+ if (pgpath == m->current_pgpath)
+ m->current_pgpath = NULL;
+
+ schedule_work(&m->trigger_event);
+
+out:
+ spin_unlock_irqrestore(&m->lock, flags);
+
+ return 0;
+}
+
+/*
+ * Reinstate a previously-failed path
+ */
+static int reinstate_path(struct pgpath *pgpath)
+{
+ int r = 0;
+ unsigned long flags;
+ struct multipath *m = pgpath->pg->m;
+
+ spin_lock_irqsave(&m->lock, flags);
+
+ if (pgpath->path.is_active)
+ goto out;
+
+ if (!pgpath->pg->ps.type) {
+ DMWARN("Reinstate path not supported by path selector %s",
+ pgpath->pg->ps.type->name);
+ r = -EINVAL;
+ goto out;
+ }
+
+ r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
+ if (r)
+ goto out;
+
+ pgpath->path.is_active = 1;
+
+ m->current_pgpath = NULL;
+ if (!m->nr_valid_paths++)
+ schedule_work(&m->process_queued_ios);
+
+ schedule_work(&m->trigger_event);
+
+out:
+ spin_unlock_irqrestore(&m->lock, flags);
+
+ return r;
+}
+
+/*
+ * Fail or reinstate all paths that match the provided struct dm_dev.
+ */
+static int action_dev(struct multipath *m, struct dm_dev *dev,
+ action_fn action)
+{
+ int r = 0;
+ struct pgpath *pgpath;
+ struct priority_group *pg;
+
+ list_for_each_entry(pg, &m->priority_groups, list) {
+ list_for_each_entry(pgpath, &pg->pgpaths, list) {
+ if (pgpath->path.dev == dev)
+ r = action(pgpath);
+ }
+ }
+
+ return r;
+}
+
+/*
+ * Temporarily try to avoid having to use the specified PG
+ */
+static void bypass_pg(struct multipath *m, struct priority_group *pg,
+ int bypassed)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&m->lock, flags);
+
+ pg->bypassed = bypassed;
+ m->current_pgpath = NULL;
+ m->current_pg = NULL;
+
+ spin_unlock_irqrestore(&m->lock, flags);
+
+ schedule_work(&m->trigger_event);
+}
+
+/*
+ * Switch to using the specified PG from the next I/O that gets mapped
+ */
+static int switch_pg_num(struct multipath *m, const char *pgstr)
+{
+ struct priority_group *pg;
+ unsigned pgnum;
+ unsigned long flags;
+
+ if (!pgstr || (sscanf(pgstr, "%u", &pgnum) != 1) || !pgnum ||
+ (pgnum > m->nr_priority_groups)) {
+ DMWARN("invalid PG number supplied to switch_pg_num");
+ return -EINVAL;
+ }
+
+ spin_lock_irqsave(&m->lock, flags);
+ list_for_each_entry(pg, &m->priority_groups, list) {
+ pg->bypassed = 0;
+ if (--pgnum)
+ continue;
+
+ m->current_pgpath = NULL;
+ m->current_pg = NULL;
+ m->next_pg = pg;
+ }
+ spin_unlock_irqrestore(&m->lock, flags);
+
+ schedule_work(&m->trigger_event);
+ return 0;
+}
+
+/*
+ * Set/clear bypassed status of a PG.
+ * PGs are numbered upwards from 1 in the order they were declared.
+ */
+static int bypass_pg_num(struct multipath *m, const char *pgstr, int bypassed)
+{
+ struct priority_group *pg;
+ unsigned pgnum;
+
+ if (!pgstr || (sscanf(pgstr, "%u", &pgnum) != 1) || !pgnum ||
+ (pgnum > m->nr_priority_groups)) {
+ DMWARN("invalid PG number supplied to bypass_pg");
+ return -EINVAL;
+ }
+
+ list_for_each_entry(pg, &m->priority_groups, list) {
+ if (!--pgnum)
+ break;
+ }
+
+ bypass_pg(m, pg, bypassed);
+ return 0;
+}
+
+/*
+ * pg_init must call this when it has completed its initialisation
+ */
+void dm_pg_init_complete(struct path *path, unsigned err_flags)
+{
+ struct pgpath *pgpath = path_to_pgpath(path);
+ struct priority_group *pg = pgpath->pg;
+ struct multipath *m = pg->m;
+ unsigned long flags;
+
+ /* We insist on failing the path if the PG is already bypassed. */
+ if (err_flags && pg->bypassed)
+ err_flags |= MP_FAIL_PATH;
+
+ if (err_flags & MP_FAIL_PATH)
+ fail_path(pgpath);
+
+ if (err_flags & MP_BYPASS_PG)
+ bypass_pg(m, pg, 1);
+
+ spin_lock_irqsave(&m->lock, flags);
+ if (!err_flags)
+ m->queue_io = 0;
+ else {
+ m->current_pgpath = NULL;
+ m->current_pg = NULL;
+ }
+ schedule_work(&m->process_queued_ios);
+ spin_unlock_irqrestore(&m->lock, flags);
+}
+
+/*
+ * end_io handling
+ */
+static int do_end_io(struct multipath *m, struct bio *bio,
+ int error, struct mpath_io *mpio)
+{
+ struct hw_handler *hwh = &m->hw_handler;
+ unsigned err_flags = MP_FAIL_PATH; /* Default behavior */
+
+ if (!error)
+ return 0; /* I/O complete */
+
+ spin_lock(&m->lock);
+ if (!m->nr_valid_paths) {
+ if (!m->queue_if_no_path || m->suspended) {
+ spin_unlock(&m->lock);
+ return -EIO;
+ } else {
+ spin_unlock(&m->lock);
+ goto requeue;
+ }
+ }
+ spin_unlock(&m->lock);
+
+ if (hwh->type && hwh->type->error)
+ err_flags = hwh->type->error(hwh, bio);
+
+ if (mpio->pgpath) {
+ if (err_flags & MP_FAIL_PATH)
+ fail_path(mpio->pgpath);
+
+ if (err_flags & MP_BYPASS_PG)
+ bypass_pg(m, mpio->pgpath->pg, 1);
+ }
+
+ if (err_flags & MP_ERROR_IO)
+ return -EIO;
+
+ requeue:
+ dm_bio_restore(&mpio->details, bio);
+
+ /* queue for the daemon to resubmit or fail */
+ spin_lock(&m->lock);
+ bio_list_add(&m->queued_ios, bio);
+ m->queue_size++;
+ if (!m->queue_io)
+ schedule_work(&m->process_queued_ios);
+ spin_unlock(&m->lock);
+
+ return 1; /* io not complete */
+}
+
+static int multipath_end_io(struct dm_target *ti, struct bio *bio,
+ int error, union map_info *map_context)
+{
+ struct multipath *m = (struct multipath *) ti->private;
+ struct mpath_io *mpio = (struct mpath_io *) map_context->ptr;
+ struct pgpath *pgpath = mpio->pgpath;
+ struct path_selector *ps;
+ int r;
+
+ r = do_end_io(m, bio, error, mpio);
+ if (pgpath) {
+ ps = &pgpath->pg->ps;
+ if (ps->type->end_io)
+ ps->type->end_io(ps, &pgpath->path);
+ }
+ if (r <= 0)
+ mempool_free(mpio, m->mpio_pool);
+
+ return r;
+}
+
+/*
+ * Suspend can't complete until all the I/O is processed so if
+ * the last path failed we will now error any queued I/O.
+ */
+static void multipath_presuspend(struct dm_target *ti)
+{
+ struct multipath *m = (struct multipath *) ti->private;
+ unsigned long flags;
+
+ spin_lock_irqsave(&m->lock, flags);
+ m->suspended = 1;
+ if (m->queue_if_no_path)
+ schedule_work(&m->process_queued_ios);
+ spin_unlock_irqrestore(&m->lock, flags);
+}
+
+static void multipath_resume(struct dm_target *ti)
+{
+ struct multipath *m = (struct multipath *) ti->private;
+ unsigned long flags;
+
+ spin_lock_irqsave(&m->lock, flags);
+ m->suspended = 0;
+ spin_unlock_irqrestore(&m->lock, flags);
+}
+
+/*
+ * Info output has the following format:
+ * num_multipath_feature_args [multipath_feature_args]*
+ * num_handler_status_args [handler_status_args]*
+ * num_groups init_group_number
+ * [A|D|E num_ps_status_args [ps_status_args]*
+ * num_paths num_selector_args
+ * [path_dev A|F fail_count [selector_args]* ]+ ]+
+ *
+ * Table output has the following format (identical to the constructor string):
+ * num_feature_args [features_args]*
+ * num_handler_args hw_handler [hw_handler_args]*
+ * num_groups init_group_number
+ * [priority selector-name num_ps_args [ps_args]*
+ * num_paths num_selector_args [path_dev [selector_args]* ]+ ]+
+ */
+static int multipath_status(struct dm_target *ti, status_type_t type,
+ char *result, unsigned int maxlen)
+{
+ int sz = 0;
+ unsigned long flags;
+ struct multipath *m = (struct multipath *) ti->private;
+ struct hw_handler *hwh = &m->hw_handler;
+ struct priority_group *pg;
+ struct pgpath *p;
+ unsigned pg_num;
+ char state;
+
+ spin_lock_irqsave(&m->lock, flags);
+
+ /* Features */
+ if (type == STATUSTYPE_INFO)
+ DMEMIT("1 %u ", m->queue_size);
+ else if (m->queue_if_no_path)
+ DMEMIT("1 queue_if_no_path ");
+ else
+ DMEMIT("0 ");
+
+ if (hwh->type && hwh->type->status)
+ sz += hwh->type->status(hwh, type, result + sz, maxlen - sz);
+ else if (!hwh->type || type == STATUSTYPE_INFO)
+ DMEMIT("0 ");
+ else
+ DMEMIT("1 %s ", hwh->type->name);
+
+ DMEMIT("%u ", m->nr_priority_groups);
+
+ if (m->next_pg)
+ pg_num = m->next_pg->pg_num;
+ else if (m->current_pg)
+ pg_num = m->current_pg->pg_num;
+ else
+ pg_num = 1;
+
+ DMEMIT("%u ", pg_num);
+
+ switch (type) {
+ case STATUSTYPE_INFO:
+ list_for_each_entry(pg, &m->priority_groups, list) {
+ if (pg->bypassed)
+ state = 'D'; /* Disabled */
+ else if (pg == m->current_pg)
+ state = 'A'; /* Currently Active */
+ else
+ state = 'E'; /* Enabled */
+
+ DMEMIT("%c ", state);
+
+ if (pg->ps.type->status)
+ sz += pg->ps.type->status(&pg->ps, NULL, type,
+ result + sz,
+ maxlen - sz);
+ else
+ DMEMIT("0 ");
+
+ DMEMIT("%u %u ", pg->nr_pgpaths,
+ pg->ps.type->info_args);
+
+ list_for_each_entry(p, &pg->pgpaths, list) {
+ DMEMIT("%s %s %u ", p->path.dev->name,
+ p->path.is_active ? "A" : "F",
+ p->fail_count);
+ if (pg->ps.type->status)
+ sz += pg->ps.type->status(&pg->ps,
+ &p->path, type, result + sz,
+ maxlen - sz);
+ }
+ }
+ break;
+
+ case STATUSTYPE_TABLE:
+ list_for_each_entry(pg, &m->priority_groups, list) {
+ DMEMIT("%s ", pg->ps.type->name);
+
+ if (pg->ps.type->status)
+ sz += pg->ps.type->status(&pg->ps, NULL, type,
+ result + sz,
+ maxlen - sz);
+ else
+ DMEMIT("0 ");
+
+ DMEMIT("%u %u ", pg->nr_pgpaths,
+ pg->ps.type->table_args);
+
+ list_for_each_entry(p, &pg->pgpaths, list) {
+ DMEMIT("%s ", p->path.dev->name);
+ if (pg->ps.type->status)
+ sz += pg->ps.type->status(&pg->ps,
+ &p->path, type, result + sz,
+ maxlen - sz);
+ }
+ }
+ break;
+ }
+
+ spin_unlock_irqrestore(&m->lock, flags);
+
+ return 0;
+}
+
+static int multipath_message(struct dm_target *ti, unsigned argc, char **argv)
+{
+ int r;
+ struct dm_dev *dev;
+ struct multipath *m = (struct multipath *) ti->private;
+ action_fn action;
+
+ if (argc == 1) {
+ if (!strnicmp(argv[0], MESG_STR("queue_if_no_path")))
+ return queue_if_no_path(m, 1);
+ else if (!strnicmp(argv[0], MESG_STR("fail_if_no_path")))
+ return queue_if_no_path(m, 0);
+ }
+
+ if (argc != 2)
+ goto error;
+
+ if (!strnicmp(argv[0], MESG_STR("disable_group")))
+ return bypass_pg_num(m, argv[1], 1);
+ else if (!strnicmp(argv[0], MESG_STR("enable_group")))
+ return bypass_pg_num(m, argv[1], 0);
+ else if (!strnicmp(argv[0], MESG_STR("switch_group")))
+ return switch_pg_num(m, argv[1]);
+ else if (!strnicmp(argv[0], MESG_STR("reinstate_path")))
+ action = reinstate_path;
+ else if (!strnicmp(argv[0], MESG_STR("fail_path")))
+ action = fail_path;
+ else
+ goto error;
+
+ r = dm_get_device(ti, argv[1], ti->begin, ti->len,
+ dm_table_get_mode(ti->table), &dev);
+ if (r) {
+ DMWARN("dm-multipath message: error getting device %s",
+ argv[1]);
+ return -EINVAL;
+ }
+
+ r = action_dev(m, dev, action);
+
+ dm_put_device(ti, dev);
+
+ return r;
+
+error:
+ DMWARN("Unrecognised multipath message received.");
+ return -EINVAL;
+}
+
+/*-----------------------------------------------------------------
+ * Module setup
+ *---------------------------------------------------------------*/
+static struct target_type multipath_target = {
+ .name = "multipath",
+ .version = {1, 0, 4},
+ .module = THIS_MODULE,
+ .ctr = multipath_ctr,
+ .dtr = multipath_dtr,
+ .map = multipath_map,
+ .end_io = multipath_end_io,
+ .presuspend = multipath_presuspend,
+ .resume = multipath_resume,
+ .status = multipath_status,
+ .message = multipath_message,
+};
+
+static int __init dm_multipath_init(void)
+{
+ int r;
+
+ /* allocate a slab for the dm_ios */
+ _mpio_cache = kmem_cache_create("dm_mpath", sizeof(struct mpath_io),
+ 0, 0, NULL, NULL);
+ if (!_mpio_cache)
+ return -ENOMEM;
+
+ r = dm_register_target(&multipath_target);
+ if (r < 0) {
+ DMERR("%s: register failed %d", multipath_target.name, r);
+ kmem_cache_destroy(_mpio_cache);
+ return -EINVAL;
+ }
+
+ DMINFO("dm-multipath version %u.%u.%u loaded",
+ multipath_target.version[0], multipath_target.version[1],
+ multipath_target.version[2]);
+
+ return r;
+}
+
+static void __exit dm_multipath_exit(void)
+{
+ int r;
+
+ r = dm_unregister_target(&multipath_target);
+ if (r < 0)
+ DMERR("%s: target unregister failed %d",
+ multipath_target.name, r);
+ kmem_cache_destroy(_mpio_cache);
+}
+
+EXPORT_SYMBOL_GPL(dm_pg_init_complete);
+
+module_init(dm_multipath_init);
+module_exit(dm_multipath_exit);
+
+MODULE_DESCRIPTION(DM_NAME " multipath target");
+MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-mpath.h b/drivers/md/dm-mpath.h
new file mode 100644
index 000000000000..8a4bf2b6d52e
--- /dev/null
+++ b/drivers/md/dm-mpath.h
@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the GPL.
+ *
+ * Multipath.
+ */
+
+#ifndef DM_MPATH_H
+#define DM_MPATH_H
+
+struct dm_dev;
+
+struct path {
+ struct dm_dev *dev; /* Read-only */
+ unsigned is_active; /* Read-only */
+
+ void *pscontext; /* For path-selector use */
+ void *hwhcontext; /* For hw-handler use */
+};
+
+/* Callback for hwh_pg_init_fn to use when complete */
+void dm_pg_init_complete(struct path *path, unsigned err_flags);
+
+#endif
diff --git a/drivers/md/dm-path-selector.c b/drivers/md/dm-path-selector.c
new file mode 100644
index 000000000000..ac5c4bbec6c1
--- /dev/null
+++ b/drivers/md/dm-path-selector.c
@@ -0,0 +1,156 @@
+/*
+ * Copyright (C) 2003 Sistina Software.
+ * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+ *
+ * Module Author: Heinz Mauelshagen
+ *
+ * This file is released under the GPL.
+ *
+ * Path selector registration.
+ */
+
+#include "dm.h"
+#include "dm-path-selector.h"
+
+#include <linux/slab.h>
+
+struct ps_internal {
+ struct path_selector_type pst;
+
+ struct list_head list;
+ long use;
+};
+
+#define pst_to_psi(__pst) container_of((__pst), struct ps_internal, pst)
+
+static LIST_HEAD(_path_selectors);
+static DECLARE_RWSEM(_ps_lock);
+
+struct ps_internal *__find_path_selector_type(const char *name)
+{
+ struct ps_internal *psi;
+
+ list_for_each_entry(psi, &_path_selectors, list) {
+ if (!strcmp(name, psi->pst.name))
+ return psi;
+ }
+
+ return NULL;
+}
+
+static struct ps_internal *get_path_selector(const char *name)
+{
+ struct ps_internal *psi;
+
+ down_read(&_ps_lock);
+ psi = __find_path_selector_type(name);
+ if (psi) {
+ if ((psi->use == 0) && !try_module_get(psi->pst.module))
+ psi = NULL;
+ else
+ psi->use++;
+ }
+ up_read(&_ps_lock);
+
+ return psi;
+}
+
+struct path_selector_type *dm_get_path_selector(const char *name)
+{
+ struct ps_internal *psi;
+
+ if (!name)
+ return NULL;
+
+ psi = get_path_selector(name);
+ if (!psi) {
+ request_module("dm-%s", name);
+ psi = get_path_selector(name);
+ }
+
+ return psi ? &psi->pst : NULL;
+}
+
+void dm_put_path_selector(struct path_selector_type *pst)
+{
+ struct ps_internal *psi;
+
+ if (!pst)
+ return;
+
+ down_read(&_ps_lock);
+ psi = __find_path_selector_type(pst->name);
+ if (!psi)
+ goto out;
+
+ if (--psi->use == 0)
+ module_put(psi->pst.module);
+
+ if (psi->use < 0)
+ BUG();
+
+out:
+ up_read(&_ps_lock);
+}
+
+static struct ps_internal *_alloc_path_selector(struct path_selector_type *pst)
+{
+ struct ps_internal *psi = kmalloc(sizeof(*psi), GFP_KERNEL);
+
+ if (psi) {
+ memset(psi, 0, sizeof(*psi));
+ psi->pst = *pst;
+ }
+
+ return psi;
+}
+
+int dm_register_path_selector(struct path_selector_type *pst)
+{
+ int r = 0;
+ struct ps_internal *psi = _alloc_path_selector(pst);
+
+ if (!psi)
+ return -ENOMEM;
+
+ down_write(&_ps_lock);
+
+ if (__find_path_selector_type(pst->name)) {
+ kfree(psi);
+ r = -EEXIST;
+ } else
+ list_add(&psi->list, &_path_selectors);
+
+ up_write(&_ps_lock);
+
+ return r;
+}
+
+int dm_unregister_path_selector(struct path_selector_type *pst)
+{
+ struct ps_internal *psi;
+
+ down_write(&_ps_lock);
+
+ psi = __find_path_selector_type(pst->name);
+ if (!psi) {
+ up_write(&_ps_lock);
+ return -EINVAL;
+ }
+
+ if (psi->use) {
+ up_write(&_ps_lock);
+ return -ETXTBSY;
+ }
+
+ list_del(&psi->list);
+
+ up_write(&_ps_lock);
+
+ kfree(psi);
+
+ return 0;
+}
+
+EXPORT_SYMBOL_GPL(dm_register_path_selector);
+EXPORT_SYMBOL_GPL(dm_unregister_path_selector);
diff --git a/drivers/md/dm-path-selector.h b/drivers/md/dm-path-selector.h
new file mode 100644
index 000000000000..732d06a84f85
--- /dev/null
+++ b/drivers/md/dm-path-selector.h
@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) 2003 Sistina Software.
+ * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+ *
+ * Module Author: Heinz Mauelshagen
+ *
+ * This file is released under the GPL.
+ *
+ * Path-Selector registration.
+ */
+
+#ifndef DM_PATH_SELECTOR_H
+#define DM_PATH_SELECTOR_H
+
+#include <linux/device-mapper.h>
+
+#include "dm-mpath.h"
+
+/*
+ * We provide an abstraction for the code that chooses which path
+ * to send some io down.
+ */
+struct path_selector_type;
+struct path_selector {
+ struct path_selector_type *type;
+ void *context;
+};
+
+/* Information about a path selector type */
+struct path_selector_type {
+ char *name;
+ struct module *module;
+
+ unsigned int table_args;
+ unsigned int info_args;
+
+ /*
+ * Constructs a path selector object, takes custom arguments
+ */
+ int (*create) (struct path_selector *ps, unsigned argc, char **argv);
+ void (*destroy) (struct path_selector *ps);
+
+ /*
+ * Add an opaque path object, along with some selector specific
+ * path args (eg, path priority).
+ */
+ int (*add_path) (struct path_selector *ps, struct path *path,
+ int argc, char **argv, char **error);
+
+ /*
+ * Chooses a path for this io, if no paths are available then
+ * NULL will be returned.
+ *
+ * repeat_count is the number of times to use the path before
+ * calling the function again. 0 means don't call it again unless
+ * the path fails.
+ */
+ struct path *(*select_path) (struct path_selector *ps,
+ unsigned *repeat_count);
+
+ /*
+ * Notify the selector that a path has failed.
+ */
+ void (*fail_path) (struct path_selector *ps, struct path *p);
+
+ /*
+ * Ask selector to reinstate a path.
+ */
+ int (*reinstate_path) (struct path_selector *ps, struct path *p);
+
+ /*
+ * Table content based on parameters added in ps_add_path_fn
+ * or path selector status
+ */
+ int (*status) (struct path_selector *ps, struct path *path,
+ status_type_t type, char *result, unsigned int maxlen);
+
+ int (*end_io) (struct path_selector *ps, struct path *path);
+};
+
+/* Register a path selector */
+int dm_register_path_selector(struct path_selector_type *type);
+
+/* Unregister a path selector */
+int dm_unregister_path_selector(struct path_selector_type *type);
+
+/* Returns a registered path selector type */
+struct path_selector_type *dm_get_path_selector(const char *name);
+
+/* Releases a path selector */
+void dm_put_path_selector(struct path_selector_type *pst);
+
+#endif
diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c
new file mode 100644
index 000000000000..6e3cf7e13451
--- /dev/null
+++ b/drivers/md/dm-raid1.c
@@ -0,0 +1,1269 @@
+/*
+ * Copyright (C) 2003 Sistina Software Limited.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+#include "dm-bio-list.h"
+#include "dm-io.h"
+#include "dm-log.h"
+#include "kcopyd.h"
+
+#include <linux/ctype.h>
+#include <linux/init.h>
+#include <linux/mempool.h>
+#include <linux/module.h>
+#include <linux/pagemap.h>
+#include <linux/slab.h>
+#include <linux/time.h>
+#include <linux/vmalloc.h>
+#include <linux/workqueue.h>
+
+static struct workqueue_struct *_kmirrord_wq;
+static struct work_struct _kmirrord_work;
+
+static inline void wake(void)
+{
+ queue_work(_kmirrord_wq, &_kmirrord_work);
+}
+
+/*-----------------------------------------------------------------
+ * Region hash
+ *
+ * The mirror splits itself up into discrete regions. Each
+ * region can be in one of three states: clean, dirty,
+ * nosync. There is no need to put clean regions in the hash.
+ *
+ * In addition to being present in the hash table a region _may_
+ * be present on one of three lists.
+ *
+ * clean_regions: Regions on this list have no io pending to
+ * them, they are in sync, we are no longer interested in them,
+ * they are dull. rh_update_states() will remove them from the
+ * hash table.
+ *
+ * quiesced_regions: These regions have been spun down, ready
+ * for recovery. rh_recovery_start() will remove regions from
+ * this list and hand them to kmirrord, which will schedule the
+ * recovery io with kcopyd.
+ *
+ * recovered_regions: Regions that kcopyd has successfully
+ * recovered. rh_update_states() will now schedule any delayed
+ * io, up the recovery_count, and remove the region from the
+ * hash.
+ *
+ * There are 2 locks:
+ * A rw spin lock 'hash_lock' protects just the hash table,
+ * this is never held in write mode from interrupt context,
+ * which I believe means that we only have to disable irqs when
+ * doing a write lock.
+ *
+ * An ordinary spin lock 'region_lock' that protects the three
+ * lists in the region_hash, with the 'state', 'list' and
+ * 'bhs_delayed' fields of the regions. This is used from irq
+ * context, so all other uses will have to suspend local irqs.
+ *---------------------------------------------------------------*/
+struct mirror_set;
+struct region_hash {
+ struct mirror_set *ms;
+ uint32_t region_size;
+ unsigned region_shift;
+
+ /* holds persistent region state */
+ struct dirty_log *log;
+
+ /* hash table */
+ rwlock_t hash_lock;
+ mempool_t *region_pool;
+ unsigned int mask;
+ unsigned int nr_buckets;
+ struct list_head *buckets;
+
+ spinlock_t region_lock;
+ struct semaphore recovery_count;
+ struct list_head clean_regions;
+ struct list_head quiesced_regions;
+ struct list_head recovered_regions;
+};
+
+enum {
+ RH_CLEAN,
+ RH_DIRTY,
+ RH_NOSYNC,
+ RH_RECOVERING
+};
+
+struct region {
+ struct region_hash *rh; /* FIXME: can we get rid of this ? */
+ region_t key;
+ int state;
+
+ struct list_head hash_list;
+ struct list_head list;
+
+ atomic_t pending;
+ struct bio_list delayed_bios;
+};
+
+/*
+ * Conversion fns
+ */
+static inline region_t bio_to_region(struct region_hash *rh, struct bio *bio)
+{
+ return bio->bi_sector >> rh->region_shift;
+}
+
+static inline sector_t region_to_sector(struct region_hash *rh, region_t region)
+{
+ return region << rh->region_shift;
+}
+
+/* FIXME move this */
+static void queue_bio(struct mirror_set *ms, struct bio *bio, int rw);
+
+static void *region_alloc(unsigned int __nocast gfp_mask, void *pool_data)
+{
+ return kmalloc(sizeof(struct region), gfp_mask);
+}
+
+static void region_free(void *element, void *pool_data)
+{
+ kfree(element);
+}
+
+#define MIN_REGIONS 64
+#define MAX_RECOVERY 1
+static int rh_init(struct region_hash *rh, struct mirror_set *ms,
+ struct dirty_log *log, uint32_t region_size,
+ region_t nr_regions)
+{
+ unsigned int nr_buckets, max_buckets;
+ size_t i;
+
+ /*
+ * Calculate a suitable number of buckets for our hash
+ * table.
+ */
+ max_buckets = nr_regions >> 6;
+ for (nr_buckets = 128u; nr_buckets < max_buckets; nr_buckets <<= 1)
+ ;
+ nr_buckets >>= 1;
+
+ rh->ms = ms;
+ rh->log = log;
+ rh->region_size = region_size;
+ rh->region_shift = ffs(region_size) - 1;
+ rwlock_init(&rh->hash_lock);
+ rh->mask = nr_buckets - 1;
+ rh->nr_buckets = nr_buckets;
+
+ rh->buckets = vmalloc(nr_buckets * sizeof(*rh->buckets));
+ if (!rh->buckets) {
+ DMERR("unable to allocate region hash memory");
+ return -ENOMEM;
+ }
+
+ for (i = 0; i < nr_buckets; i++)
+ INIT_LIST_HEAD(rh->buckets + i);
+
+ spin_lock_init(&rh->region_lock);
+ sema_init(&rh->recovery_count, 0);
+ INIT_LIST_HEAD(&rh->clean_regions);
+ INIT_LIST_HEAD(&rh->quiesced_regions);
+ INIT_LIST_HEAD(&rh->recovered_regions);
+
+ rh->region_pool = mempool_create(MIN_REGIONS, region_alloc,
+ region_free, NULL);
+ if (!rh->region_pool) {
+ vfree(rh->buckets);
+ rh->buckets = NULL;
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void rh_exit(struct region_hash *rh)
+{
+ unsigned int h;
+ struct region *reg, *nreg;
+
+ BUG_ON(!list_empty(&rh->quiesced_regions));
+ for (h = 0; h < rh->nr_buckets; h++) {
+ list_for_each_entry_safe(reg, nreg, rh->buckets + h, hash_list) {
+ BUG_ON(atomic_read(&reg->pending));
+ mempool_free(reg, rh->region_pool);
+ }
+ }
+
+ if (rh->log)
+ dm_destroy_dirty_log(rh->log);
+ if (rh->region_pool)
+ mempool_destroy(rh->region_pool);
+ vfree(rh->buckets);
+}
+
+#define RH_HASH_MULT 2654435387U
+
+static inline unsigned int rh_hash(struct region_hash *rh, region_t region)
+{
+ return (unsigned int) ((region * RH_HASH_MULT) >> 12) & rh->mask;
+}
+
+static struct region *__rh_lookup(struct region_hash *rh, region_t region)
+{
+ struct region *reg;
+
+ list_for_each_entry (reg, rh->buckets + rh_hash(rh, region), hash_list)
+ if (reg->key == region)
+ return reg;
+
+ return NULL;
+}
+
+static void __rh_insert(struct region_hash *rh, struct region *reg)
+{
+ unsigned int h = rh_hash(rh, reg->key);
+ list_add(&reg->hash_list, rh->buckets + h);
+}
+
+static struct region *__rh_alloc(struct region_hash *rh, region_t region)
+{
+ struct region *reg, *nreg;
+
+ read_unlock(&rh->hash_lock);
+ nreg = mempool_alloc(rh->region_pool, GFP_NOIO);
+ nreg->state = rh->log->type->in_sync(rh->log, region, 1) ?
+ RH_CLEAN : RH_NOSYNC;
+ nreg->rh = rh;
+ nreg->key = region;
+
+ INIT_LIST_HEAD(&nreg->list);
+
+ atomic_set(&nreg->pending, 0);
+ bio_list_init(&nreg->delayed_bios);
+ write_lock_irq(&rh->hash_lock);
+
+ reg = __rh_lookup(rh, region);
+ if (reg)
+ /* we lost the race */
+ mempool_free(nreg, rh->region_pool);
+
+ else {
+ __rh_insert(rh, nreg);
+ if (nreg->state == RH_CLEAN) {
+ spin_lock(&rh->region_lock);
+ list_add(&nreg->list, &rh->clean_regions);
+ spin_unlock(&rh->region_lock);
+ }
+ reg = nreg;
+ }
+ write_unlock_irq(&rh->hash_lock);
+ read_lock(&rh->hash_lock);
+
+ return reg;
+}
+
+static inline struct region *__rh_find(struct region_hash *rh, region_t region)
+{
+ struct region *reg;
+
+ reg = __rh_lookup(rh, region);
+ if (!reg)
+ reg = __rh_alloc(rh, region);
+
+ return reg;
+}
+
+static int rh_state(struct region_hash *rh, region_t region, int may_block)
+{
+ int r;
+ struct region *reg;
+
+ read_lock(&rh->hash_lock);
+ reg = __rh_lookup(rh, region);
+ read_unlock(&rh->hash_lock);
+
+ if (reg)
+ return reg->state;
+
+ /*
+ * The region wasn't in the hash, so we fall back to the
+ * dirty log.
+ */
+ r = rh->log->type->in_sync(rh->log, region, may_block);
+
+ /*
+ * Any error from the dirty log (eg. -EWOULDBLOCK) gets
+ * taken as a RH_NOSYNC
+ */
+ return r == 1 ? RH_CLEAN : RH_NOSYNC;
+}
+
+static inline int rh_in_sync(struct region_hash *rh,
+ region_t region, int may_block)
+{
+ int state = rh_state(rh, region, may_block);
+ return state == RH_CLEAN || state == RH_DIRTY;
+}
+
+static void dispatch_bios(struct mirror_set *ms, struct bio_list *bio_list)
+{
+ struct bio *bio;
+
+ while ((bio = bio_list_pop(bio_list))) {
+ queue_bio(ms, bio, WRITE);
+ }
+}
+
+static void rh_update_states(struct region_hash *rh)
+{
+ struct region *reg, *next;
+
+ LIST_HEAD(clean);
+ LIST_HEAD(recovered);
+
+ /*
+ * Quickly grab the lists.
+ */
+ write_lock_irq(&rh->hash_lock);
+ spin_lock(&rh->region_lock);
+ if (!list_empty(&rh->clean_regions)) {
+ list_splice(&rh->clean_regions, &clean);
+ INIT_LIST_HEAD(&rh->clean_regions);
+
+ list_for_each_entry (reg, &clean, list) {
+ rh->log->type->clear_region(rh->log, reg->key);
+ list_del(&reg->hash_list);
+ }
+ }
+
+ if (!list_empty(&rh->recovered_regions)) {
+ list_splice(&rh->recovered_regions, &recovered);
+ INIT_LIST_HEAD(&rh->recovered_regions);
+
+ list_for_each_entry (reg, &recovered, list)
+ list_del(&reg->hash_list);
+ }
+ spin_unlock(&rh->region_lock);
+ write_unlock_irq(&rh->hash_lock);
+
+ /*
+ * All the regions on the recovered and clean lists have
+ * now been pulled out of the system, so no need to do
+ * any more locking.
+ */
+ list_for_each_entry_safe (reg, next, &recovered, list) {
+ rh->log->type->clear_region(rh->log, reg->key);
+ rh->log->type->complete_resync_work(rh->log, reg->key, 1);
+ dispatch_bios(rh->ms, &reg->delayed_bios);
+ up(&rh->recovery_count);
+ mempool_free(reg, rh->region_pool);
+ }
+
+ if (!list_empty(&recovered))
+ rh->log->type->flush(rh->log);
+
+ list_for_each_entry_safe (reg, next, &clean, list)
+ mempool_free(reg, rh->region_pool);
+}
+
+static void rh_inc(struct region_hash *rh, region_t region)
+{
+ struct region *reg;
+
+ read_lock(&rh->hash_lock);
+ reg = __rh_find(rh, region);
+ if (reg->state == RH_CLEAN) {
+ rh->log->type->mark_region(rh->log, reg->key);
+
+ spin_lock_irq(&rh->region_lock);
+ reg->state = RH_DIRTY;
+ list_del_init(&reg->list); /* take off the clean list */
+ spin_unlock_irq(&rh->region_lock);
+ }
+
+ atomic_inc(&reg->pending);
+ read_unlock(&rh->hash_lock);
+}
+
+static void rh_inc_pending(struct region_hash *rh, struct bio_list *bios)
+{
+ struct bio *bio;
+
+ for (bio = bios->head; bio; bio = bio->bi_next)
+ rh_inc(rh, bio_to_region(rh, bio));
+}
+
+static void rh_dec(struct region_hash *rh, region_t region)
+{
+ unsigned long flags;
+ struct region *reg;
+ int should_wake = 0;
+
+ read_lock(&rh->hash_lock);
+ reg = __rh_lookup(rh, region);
+ read_unlock(&rh->hash_lock);
+
+ if (atomic_dec_and_test(&reg->pending)) {
+ spin_lock_irqsave(&rh->region_lock, flags);
+ if (reg->state == RH_RECOVERING) {
+ list_add_tail(&reg->list, &rh->quiesced_regions);
+ } else {
+ reg->state = RH_CLEAN;
+ list_add(&reg->list, &rh->clean_regions);
+ }
+ spin_unlock_irqrestore(&rh->region_lock, flags);
+ should_wake = 1;
+ }
+
+ if (should_wake)
+ wake();
+}
+
+/*
+ * Starts quiescing a region in preparation for recovery.
+ */
+static int __rh_recovery_prepare(struct region_hash *rh)
+{
+ int r;
+ struct region *reg;
+ region_t region;
+
+ /*
+ * Ask the dirty log what's next.
+ */
+ r = rh->log->type->get_resync_work(rh->log, &region);
+ if (r <= 0)
+ return r;
+
+ /*
+ * Get this region, and start it quiescing by setting the
+ * recovering flag.
+ */
+ read_lock(&rh->hash_lock);
+ reg = __rh_find(rh, region);
+ read_unlock(&rh->hash_lock);
+
+ spin_lock_irq(&rh->region_lock);
+ reg->state = RH_RECOVERING;
+
+ /* Already quiesced ? */
+ if (atomic_read(&reg->pending))
+ list_del_init(&reg->list);
+
+ else {
+ list_del_init(&reg->list);
+ list_add(&reg->list, &rh->quiesced_regions);
+ }
+ spin_unlock_irq(&rh->region_lock);
+
+ return 1;
+}
+
+static void rh_recovery_prepare(struct region_hash *rh)
+{
+ while (!down_trylock(&rh->recovery_count))
+ if (__rh_recovery_prepare(rh) <= 0) {
+ up(&rh->recovery_count);
+ break;
+ }
+}
+
+/*
+ * Returns any quiesced regions.
+ */
+static struct region *rh_recovery_start(struct region_hash *rh)
+{
+ struct region *reg = NULL;
+
+ spin_lock_irq(&rh->region_lock);
+ if (!list_empty(&rh->quiesced_regions)) {
+ reg = list_entry(rh->quiesced_regions.next,
+ struct region, list);
+ list_del_init(&reg->list); /* remove from the quiesced list */
+ }
+ spin_unlock_irq(&rh->region_lock);
+
+ return reg;
+}
+
+/* FIXME: success ignored for now */
+static void rh_recovery_end(struct region *reg, int success)
+{
+ struct region_hash *rh = reg->rh;
+
+ spin_lock_irq(&rh->region_lock);
+ list_add(&reg->list, &reg->rh->recovered_regions);
+ spin_unlock_irq(&rh->region_lock);
+
+ wake();
+}
+
+static void rh_flush(struct region_hash *rh)
+{
+ rh->log->type->flush(rh->log);
+}
+
+static void rh_delay(struct region_hash *rh, struct bio *bio)
+{
+ struct region *reg;
+
+ read_lock(&rh->hash_lock);
+ reg = __rh_find(rh, bio_to_region(rh, bio));
+ bio_list_add(&reg->delayed_bios, bio);
+ read_unlock(&rh->hash_lock);
+}
+
+static void rh_stop_recovery(struct region_hash *rh)
+{
+ int i;
+
+ /* wait for any recovering regions */
+ for (i = 0; i < MAX_RECOVERY; i++)
+ down(&rh->recovery_count);
+}
+
+static void rh_start_recovery(struct region_hash *rh)
+{
+ int i;
+
+ for (i = 0; i < MAX_RECOVERY; i++)
+ up(&rh->recovery_count);
+
+ wake();
+}
+
+/*-----------------------------------------------------------------
+ * Mirror set structures.
+ *---------------------------------------------------------------*/
+struct mirror {
+ atomic_t error_count;
+ struct dm_dev *dev;
+ sector_t offset;
+};
+
+struct mirror_set {
+ struct dm_target *ti;
+ struct list_head list;
+ struct region_hash rh;
+ struct kcopyd_client *kcopyd_client;
+
+ spinlock_t lock; /* protects the next two lists */
+ struct bio_list reads;
+ struct bio_list writes;
+
+ /* recovery */
+ region_t nr_regions;
+ int in_sync;
+
+ unsigned int nr_mirrors;
+ struct mirror mirror[0];
+};
+
+/*
+ * Every mirror should look like this one.
+ */
+#define DEFAULT_MIRROR 0
+
+/*
+ * This is yucky. We squirrel the mirror_set struct away inside
+ * bi_next for write buffers. This is safe since the bh
+ * doesn't get submitted to the lower levels of block layer.
+ */
+static struct mirror_set *bio_get_ms(struct bio *bio)
+{
+ return (struct mirror_set *) bio->bi_next;
+}
+
+static void bio_set_ms(struct bio *bio, struct mirror_set *ms)
+{
+ bio->bi_next = (struct bio *) ms;
+}
+
+/*-----------------------------------------------------------------
+ * Recovery.
+ *
+ * When a mirror is first activated we may find that some regions
+ * are in the no-sync state. We have to recover these by
+ * recopying from the default mirror to all the others.
+ *---------------------------------------------------------------*/
+static void recovery_complete(int read_err, unsigned int write_err,
+ void *context)
+{
+ struct region *reg = (struct region *) context;
+
+ /* FIXME: better error handling */
+ rh_recovery_end(reg, read_err || write_err);
+}
+
+static int recover(struct mirror_set *ms, struct region *reg)
+{
+ int r;
+ unsigned int i;
+ struct io_region from, to[KCOPYD_MAX_REGIONS], *dest;
+ struct mirror *m;
+ unsigned long flags = 0;
+
+ /* fill in the source */
+ m = ms->mirror + DEFAULT_MIRROR;
+ from.bdev = m->dev->bdev;
+ from.sector = m->offset + region_to_sector(reg->rh, reg->key);
+ if (reg->key == (ms->nr_regions - 1)) {
+ /*
+ * The final region may be smaller than
+ * region_size.
+ */
+ from.count = ms->ti->len & (reg->rh->region_size - 1);
+ if (!from.count)
+ from.count = reg->rh->region_size;
+ } else
+ from.count = reg->rh->region_size;
+
+ /* fill in the destinations */
+ for (i = 0, dest = to; i < ms->nr_mirrors; i++) {
+ if (i == DEFAULT_MIRROR)
+ continue;
+
+ m = ms->mirror + i;
+ dest->bdev = m->dev->bdev;
+ dest->sector = m->offset + region_to_sector(reg->rh, reg->key);
+ dest->count = from.count;
+ dest++;
+ }
+
+ /* hand to kcopyd */
+ set_bit(KCOPYD_IGNORE_ERROR, &flags);
+ r = kcopyd_copy(ms->kcopyd_client, &from, ms->nr_mirrors - 1, to, flags,
+ recovery_complete, reg);
+
+ return r;
+}
+
+static void do_recovery(struct mirror_set *ms)
+{
+ int r;
+ struct region *reg;
+ struct dirty_log *log = ms->rh.log;
+
+ /*
+ * Start quiescing some regions.
+ */
+ rh_recovery_prepare(&ms->rh);
+
+ /*
+ * Copy any already quiesced regions.
+ */
+ while ((reg = rh_recovery_start(&ms->rh))) {
+ r = recover(ms, reg);
+ if (r)
+ rh_recovery_end(reg, 0);
+ }
+
+ /*
+ * Update the in sync flag.
+ */
+ if (!ms->in_sync &&
+ (log->type->get_sync_count(log) == ms->nr_regions)) {
+ /* the sync is complete */
+ dm_table_event(ms->ti->table);
+ ms->in_sync = 1;
+ }
+}
+
+/*-----------------------------------------------------------------
+ * Reads
+ *---------------------------------------------------------------*/
+static struct mirror *choose_mirror(struct mirror_set *ms, sector_t sector)
+{
+ /* FIXME: add read balancing */
+ return ms->mirror + DEFAULT_MIRROR;
+}
+
+/*
+ * remap a buffer to a particular mirror.
+ */
+static void map_bio(struct mirror_set *ms, struct mirror *m, struct bio *bio)
+{
+ bio->bi_bdev = m->dev->bdev;
+ bio->bi_sector = m->offset + (bio->bi_sector - ms->ti->begin);
+}
+
+static void do_reads(struct mirror_set *ms, struct bio_list *reads)
+{
+ region_t region;
+ struct bio *bio;
+ struct mirror *m;
+
+ while ((bio = bio_list_pop(reads))) {
+ region = bio_to_region(&ms->rh, bio);
+
+ /*
+ * We can only read balance if the region is in sync.
+ */
+ if (rh_in_sync(&ms->rh, region, 0))
+ m = choose_mirror(ms, bio->bi_sector);
+ else
+ m = ms->mirror + DEFAULT_MIRROR;
+
+ map_bio(ms, m, bio);
+ generic_make_request(bio);
+ }
+}
+
+/*-----------------------------------------------------------------
+ * Writes.
+ *
+ * We do different things with the write io depending on the
+ * state of the region that it's in:
+ *
+ * SYNC: increment pending, use kcopyd to write to *all* mirrors
+ * RECOVERING: delay the io until recovery completes
+ * NOSYNC: increment pending, just write to the default mirror
+ *---------------------------------------------------------------*/
+static void write_callback(unsigned long error, void *context)
+{
+ unsigned int i;
+ int uptodate = 1;
+ struct bio *bio = (struct bio *) context;
+ struct mirror_set *ms;
+
+ ms = bio_get_ms(bio);
+ bio_set_ms(bio, NULL);
+
+ /*
+ * NOTE: We don't decrement the pending count here,
+ * instead it is done by the targets endio function.
+ * This way we handle both writes to SYNC and NOSYNC
+ * regions with the same code.
+ */
+
+ if (error) {
+ /*
+ * only error the io if all mirrors failed.
+ * FIXME: bogus
+ */
+ uptodate = 0;
+ for (i = 0; i < ms->nr_mirrors; i++)
+ if (!test_bit(i, &error)) {
+ uptodate = 1;
+ break;
+ }
+ }
+ bio_endio(bio, bio->bi_size, 0);
+}
+
+static void do_write(struct mirror_set *ms, struct bio *bio)
+{
+ unsigned int i;
+ struct io_region io[KCOPYD_MAX_REGIONS+1];
+ struct mirror *m;
+
+ for (i = 0; i < ms->nr_mirrors; i++) {
+ m = ms->mirror + i;
+
+ io[i].bdev = m->dev->bdev;
+ io[i].sector = m->offset + (bio->bi_sector - ms->ti->begin);
+ io[i].count = bio->bi_size >> 9;
+ }
+
+ bio_set_ms(bio, ms);
+ dm_io_async_bvec(ms->nr_mirrors, io, WRITE,
+ bio->bi_io_vec + bio->bi_idx,
+ write_callback, bio);
+}
+
+static void do_writes(struct mirror_set *ms, struct bio_list *writes)
+{
+ int state;
+ struct bio *bio;
+ struct bio_list sync, nosync, recover, *this_list = NULL;
+
+ if (!writes->head)
+ return;
+
+ /*
+ * Classify each write.
+ */
+ bio_list_init(&sync);
+ bio_list_init(&nosync);
+ bio_list_init(&recover);
+
+ while ((bio = bio_list_pop(writes))) {
+ state = rh_state(&ms->rh, bio_to_region(&ms->rh, bio), 1);
+ switch (state) {
+ case RH_CLEAN:
+ case RH_DIRTY:
+ this_list = &sync;
+ break;
+
+ case RH_NOSYNC:
+ this_list = &nosync;
+ break;
+
+ case RH_RECOVERING:
+ this_list = &recover;
+ break;
+ }
+
+ bio_list_add(this_list, bio);
+ }
+
+ /*
+ * Increment the pending counts for any regions that will
+ * be written to (writes to recover regions are going to
+ * be delayed).
+ */
+ rh_inc_pending(&ms->rh, &sync);
+ rh_inc_pending(&ms->rh, &nosync);
+ rh_flush(&ms->rh);
+
+ /*
+ * Dispatch io.
+ */
+ while ((bio = bio_list_pop(&sync)))
+ do_write(ms, bio);
+
+ while ((bio = bio_list_pop(&recover)))
+ rh_delay(&ms->rh, bio);
+
+ while ((bio = bio_list_pop(&nosync))) {
+ map_bio(ms, ms->mirror + DEFAULT_MIRROR, bio);
+ generic_make_request(bio);
+ }
+}
+
+/*-----------------------------------------------------------------
+ * kmirrord
+ *---------------------------------------------------------------*/
+static LIST_HEAD(_mirror_sets);
+static DECLARE_RWSEM(_mirror_sets_lock);
+
+static void do_mirror(struct mirror_set *ms)
+{
+ struct bio_list reads, writes;
+
+ spin_lock(&ms->lock);
+ reads = ms->reads;
+ writes = ms->writes;
+ bio_list_init(&ms->reads);
+ bio_list_init(&ms->writes);
+ spin_unlock(&ms->lock);
+
+ rh_update_states(&ms->rh);
+ do_recovery(ms);
+ do_reads(ms, &reads);
+ do_writes(ms, &writes);
+}
+
+static void do_work(void *ignored)
+{
+ struct mirror_set *ms;
+
+ down_read(&_mirror_sets_lock);
+ list_for_each_entry (ms, &_mirror_sets, list)
+ do_mirror(ms);
+ up_read(&_mirror_sets_lock);
+}
+
+/*-----------------------------------------------------------------
+ * Target functions
+ *---------------------------------------------------------------*/
+static struct mirror_set *alloc_context(unsigned int nr_mirrors,
+ uint32_t region_size,
+ struct dm_target *ti,
+ struct dirty_log *dl)
+{
+ size_t len;
+ struct mirror_set *ms = NULL;
+
+ if (array_too_big(sizeof(*ms), sizeof(ms->mirror[0]), nr_mirrors))
+ return NULL;
+
+ len = sizeof(*ms) + (sizeof(ms->mirror[0]) * nr_mirrors);
+
+ ms = kmalloc(len, GFP_KERNEL);
+ if (!ms) {
+ ti->error = "dm-mirror: Cannot allocate mirror context";
+ return NULL;
+ }
+
+ memset(ms, 0, len);
+ spin_lock_init(&ms->lock);
+
+ ms->ti = ti;
+ ms->nr_mirrors = nr_mirrors;
+ ms->nr_regions = dm_sector_div_up(ti->len, region_size);
+ ms->in_sync = 0;
+
+ if (rh_init(&ms->rh, ms, dl, region_size, ms->nr_regions)) {
+ ti->error = "dm-mirror: Error creating dirty region hash";
+ kfree(ms);
+ return NULL;
+ }
+
+ return ms;
+}
+
+static void free_context(struct mirror_set *ms, struct dm_target *ti,
+ unsigned int m)
+{
+ while (m--)
+ dm_put_device(ti, ms->mirror[m].dev);
+
+ rh_exit(&ms->rh);
+ kfree(ms);
+}
+
+static inline int _check_region_size(struct dm_target *ti, uint32_t size)
+{
+ return !(size % (PAGE_SIZE >> 9) || (size & (size - 1)) ||
+ size > ti->len);
+}
+
+static int get_mirror(struct mirror_set *ms, struct dm_target *ti,
+ unsigned int mirror, char **argv)
+{
+ sector_t offset;
+
+ if (sscanf(argv[1], SECTOR_FORMAT, &offset) != 1) {
+ ti->error = "dm-mirror: Invalid offset";
+ return -EINVAL;
+ }
+
+ if (dm_get_device(ti, argv[0], offset, ti->len,
+ dm_table_get_mode(ti->table),
+ &ms->mirror[mirror].dev)) {
+ ti->error = "dm-mirror: Device lookup failure";
+ return -ENXIO;
+ }
+
+ ms->mirror[mirror].offset = offset;
+
+ return 0;
+}
+
+static int add_mirror_set(struct mirror_set *ms)
+{
+ down_write(&_mirror_sets_lock);
+ list_add_tail(&ms->list, &_mirror_sets);
+ up_write(&_mirror_sets_lock);
+ wake();
+
+ return 0;
+}
+
+static void del_mirror_set(struct mirror_set *ms)
+{
+ down_write(&_mirror_sets_lock);
+ list_del(&ms->list);
+ up_write(&_mirror_sets_lock);
+}
+
+/*
+ * Create dirty log: log_type #log_params <log_params>
+ */
+static struct dirty_log *create_dirty_log(struct dm_target *ti,
+ unsigned int argc, char **argv,
+ unsigned int *args_used)
+{
+ unsigned int param_count;
+ struct dirty_log *dl;
+
+ if (argc < 2) {
+ ti->error = "dm-mirror: Insufficient mirror log arguments";
+ return NULL;
+ }
+
+ if (sscanf(argv[1], "%u", &param_count) != 1) {
+ ti->error = "dm-mirror: Invalid mirror log argument count";
+ return NULL;
+ }
+
+ *args_used = 2 + param_count;
+
+ if (argc < *args_used) {
+ ti->error = "dm-mirror: Insufficient mirror log arguments";
+ return NULL;
+ }
+
+ dl = dm_create_dirty_log(argv[0], ti, param_count, argv + 2);
+ if (!dl) {
+ ti->error = "dm-mirror: Error creating mirror dirty log";
+ return NULL;
+ }
+
+ if (!_check_region_size(ti, dl->type->get_region_size(dl))) {
+ ti->error = "dm-mirror: Invalid region size";
+ dm_destroy_dirty_log(dl);
+ return NULL;
+ }
+
+ return dl;
+}
+
+/*
+ * Construct a mirror mapping:
+ *
+ * log_type #log_params <log_params>
+ * #mirrors [mirror_path offset]{2,}
+ *
+ * log_type is "core" or "disk"
+ * #log_params is between 1 and 3
+ */
+#define DM_IO_PAGES 64
+static int mirror_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+ int r;
+ unsigned int nr_mirrors, m, args_used;
+ struct mirror_set *ms;
+ struct dirty_log *dl;
+
+ dl = create_dirty_log(ti, argc, argv, &args_used);
+ if (!dl)
+ return -EINVAL;
+
+ argv += args_used;
+ argc -= args_used;
+
+ if (!argc || sscanf(argv[0], "%u", &nr_mirrors) != 1 ||
+ nr_mirrors < 2 || nr_mirrors > KCOPYD_MAX_REGIONS + 1) {
+ ti->error = "dm-mirror: Invalid number of mirrors";
+ dm_destroy_dirty_log(dl);
+ return -EINVAL;
+ }
+
+ argv++, argc--;
+
+ if (argc != nr_mirrors * 2) {
+ ti->error = "dm-mirror: Wrong number of mirror arguments";
+ dm_destroy_dirty_log(dl);
+ return -EINVAL;
+ }
+
+ ms = alloc_context(nr_mirrors, dl->type->get_region_size(dl), ti, dl);
+ if (!ms) {
+ dm_destroy_dirty_log(dl);
+ return -ENOMEM;
+ }
+
+ /* Get the mirror parameter sets */
+ for (m = 0; m < nr_mirrors; m++) {
+ r = get_mirror(ms, ti, m, argv);
+ if (r) {
+ free_context(ms, ti, m);
+ return r;
+ }
+ argv += 2;
+ argc -= 2;
+ }
+
+ ti->private = ms;
+
+ r = kcopyd_client_create(DM_IO_PAGES, &ms->kcopyd_client);
+ if (r) {
+ free_context(ms, ti, ms->nr_mirrors);
+ return r;
+ }
+
+ add_mirror_set(ms);
+ return 0;
+}
+
+static void mirror_dtr(struct dm_target *ti)
+{
+ struct mirror_set *ms = (struct mirror_set *) ti->private;
+
+ del_mirror_set(ms);
+ kcopyd_client_destroy(ms->kcopyd_client);
+ free_context(ms, ti, ms->nr_mirrors);
+}
+
+static void queue_bio(struct mirror_set *ms, struct bio *bio, int rw)
+{
+ int should_wake = 0;
+ struct bio_list *bl;
+
+ bl = (rw == WRITE) ? &ms->writes : &ms->reads;
+ spin_lock(&ms->lock);
+ should_wake = !(bl->head);
+ bio_list_add(bl, bio);
+ spin_unlock(&ms->lock);
+
+ if (should_wake)
+ wake();
+}
+
+/*
+ * Mirror mapping function
+ */
+static int mirror_map(struct dm_target *ti, struct bio *bio,
+ union map_info *map_context)
+{
+ int r, rw = bio_rw(bio);
+ struct mirror *m;
+ struct mirror_set *ms = ti->private;
+
+ map_context->ll = bio->bi_sector >> ms->rh.region_shift;
+
+ if (rw == WRITE) {
+ queue_bio(ms, bio, rw);
+ return 0;
+ }
+
+ r = ms->rh.log->type->in_sync(ms->rh.log,
+ bio_to_region(&ms->rh, bio), 0);
+ if (r < 0 && r != -EWOULDBLOCK)
+ return r;
+
+ if (r == -EWOULDBLOCK) /* FIXME: ugly */
+ r = 0;
+
+ /*
+ * We don't want to fast track a recovery just for a read
+ * ahead. So we just let it silently fail.
+ * FIXME: get rid of this.
+ */
+ if (!r && rw == READA)
+ return -EIO;
+
+ if (!r) {
+ /* Pass this io over to the daemon */
+ queue_bio(ms, bio, rw);
+ return 0;
+ }
+
+ m = choose_mirror(ms, bio->bi_sector);
+ if (!m)
+ return -EIO;
+
+ map_bio(ms, m, bio);
+ return 1;
+}
+
+static int mirror_end_io(struct dm_target *ti, struct bio *bio,
+ int error, union map_info *map_context)
+{
+ int rw = bio_rw(bio);
+ struct mirror_set *ms = (struct mirror_set *) ti->private;
+ region_t region = map_context->ll;
+
+ /*
+ * We need to dec pending if this was a write.
+ */
+ if (rw == WRITE)
+ rh_dec(&ms->rh, region);
+
+ return 0;
+}
+
+static void mirror_postsuspend(struct dm_target *ti)
+{
+ struct mirror_set *ms = (struct mirror_set *) ti->private;
+ struct dirty_log *log = ms->rh.log;
+
+ rh_stop_recovery(&ms->rh);
+ if (log->type->suspend && log->type->suspend(log))
+ /* FIXME: need better error handling */
+ DMWARN("log suspend failed");
+}
+
+static void mirror_resume(struct dm_target *ti)
+{
+ struct mirror_set *ms = (struct mirror_set *) ti->private;
+ struct dirty_log *log = ms->rh.log;
+ if (log->type->resume && log->type->resume(log))
+ /* FIXME: need better error handling */
+ DMWARN("log resume failed");
+ rh_start_recovery(&ms->rh);
+}
+
+static int mirror_status(struct dm_target *ti, status_type_t type,
+ char *result, unsigned int maxlen)
+{
+ unsigned int m, sz;
+ struct mirror_set *ms = (struct mirror_set *) ti->private;
+
+ sz = ms->rh.log->type->status(ms->rh.log, type, result, maxlen);
+
+ switch (type) {
+ case STATUSTYPE_INFO:
+ DMEMIT("%d ", ms->nr_mirrors);
+ for (m = 0; m < ms->nr_mirrors; m++)
+ DMEMIT("%s ", ms->mirror[m].dev->name);
+
+ DMEMIT(SECTOR_FORMAT "/" SECTOR_FORMAT,
+ ms->rh.log->type->get_sync_count(ms->rh.log),
+ ms->nr_regions);
+ break;
+
+ case STATUSTYPE_TABLE:
+ DMEMIT("%d ", ms->nr_mirrors);
+ for (m = 0; m < ms->nr_mirrors; m++)
+ DMEMIT("%s " SECTOR_FORMAT " ",
+ ms->mirror[m].dev->name, ms->mirror[m].offset);
+ }
+
+ return 0;
+}
+
+static struct target_type mirror_target = {
+ .name = "mirror",
+ .version = {1, 0, 1},
+ .module = THIS_MODULE,
+ .ctr = mirror_ctr,
+ .dtr = mirror_dtr,
+ .map = mirror_map,
+ .end_io = mirror_end_io,
+ .postsuspend = mirror_postsuspend,
+ .resume = mirror_resume,
+ .status = mirror_status,
+};
+
+static int __init dm_mirror_init(void)
+{
+ int r;
+
+ r = dm_dirty_log_init();
+ if (r)
+ return r;
+
+ _kmirrord_wq = create_workqueue("kmirrord");
+ if (!_kmirrord_wq) {
+ DMERR("couldn't start kmirrord");
+ dm_dirty_log_exit();
+ return r;
+ }
+ INIT_WORK(&_kmirrord_work, do_work, NULL);
+
+ r = dm_register_target(&mirror_target);
+ if (r < 0) {
+ DMERR("%s: Failed to register mirror target",
+ mirror_target.name);
+ dm_dirty_log_exit();
+ destroy_workqueue(_kmirrord_wq);
+ }
+
+ return r;
+}
+
+static void __exit dm_mirror_exit(void)
+{
+ int r;
+
+ r = dm_unregister_target(&mirror_target);
+ if (r < 0)
+ DMERR("%s: unregister failed %d", mirror_target.name, r);
+
+ destroy_workqueue(_kmirrord_wq);
+ dm_dirty_log_exit();
+}
+
+/* Module hooks */
+module_init(dm_mirror_init);
+module_exit(dm_mirror_exit);
+
+MODULE_DESCRIPTION(DM_NAME " mirror target");
+MODULE_AUTHOR("Joe Thornber");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-round-robin.c b/drivers/md/dm-round-robin.c
new file mode 100644
index 000000000000..d0024865a789
--- /dev/null
+++ b/drivers/md/dm-round-robin.c
@@ -0,0 +1,214 @@
+/*
+ * Copyright (C) 2003 Sistina Software.
+ * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
+ *
+ * Module Author: Heinz Mauelshagen
+ *
+ * This file is released under the GPL.
+ *
+ * Round-robin path selector.
+ */
+
+#include "dm.h"
+#include "dm-path-selector.h"
+
+#include <linux/slab.h>
+
+/*-----------------------------------------------------------------
+ * Path-handling code, paths are held in lists
+ *---------------------------------------------------------------*/
+struct path_info {
+ struct list_head list;
+ struct path *path;
+ unsigned repeat_count;
+};
+
+static void free_paths(struct list_head *paths)
+{
+ struct path_info *pi, *next;
+
+ list_for_each_entry_safe(pi, next, paths, list) {
+ list_del(&pi->list);
+ kfree(pi);
+ }
+}
+
+/*-----------------------------------------------------------------
+ * Round-robin selector
+ *---------------------------------------------------------------*/
+
+#define RR_MIN_IO 1000
+
+struct selector {
+ struct list_head valid_paths;
+ struct list_head invalid_paths;
+};
+
+static struct selector *alloc_selector(void)
+{
+ struct selector *s = kmalloc(sizeof(*s), GFP_KERNEL);
+
+ if (s) {
+ INIT_LIST_HEAD(&s->valid_paths);
+ INIT_LIST_HEAD(&s->invalid_paths);
+ }
+
+ return s;
+}
+
+static int rr_create(struct path_selector *ps, unsigned argc, char **argv)
+{
+ struct selector *s;
+
+ s = alloc_selector();
+ if (!s)
+ return -ENOMEM;
+
+ ps->context = s;
+ return 0;
+}
+
+static void rr_destroy(struct path_selector *ps)
+{
+ struct selector *s = (struct selector *) ps->context;
+
+ free_paths(&s->valid_paths);
+ free_paths(&s->invalid_paths);
+ kfree(s);
+ ps->context = NULL;
+}
+
+static int rr_status(struct path_selector *ps, struct path *path,
+ status_type_t type, char *result, unsigned int maxlen)
+{
+ struct path_info *pi;
+ int sz = 0;
+
+ if (!path)
+ DMEMIT("0 ");
+ else {
+ switch(type) {
+ case STATUSTYPE_INFO:
+ break;
+ case STATUSTYPE_TABLE:
+ pi = path->pscontext;
+ DMEMIT("%u ", pi->repeat_count);
+ break;
+ }
+ }
+
+ return sz;
+}
+
+/*
+ * Called during initialisation to register each path with an
+ * optional repeat_count.
+ */
+static int rr_add_path(struct path_selector *ps, struct path *path,
+ int argc, char **argv, char **error)
+{
+ struct selector *s = (struct selector *) ps->context;
+ struct path_info *pi;
+ unsigned repeat_count = RR_MIN_IO;
+
+ if (argc > 1) {
+ *error = "round-robin ps: incorrect number of arguments";
+ return -EINVAL;
+ }
+
+ /* First path argument is number of I/Os before switching path */
+ if ((argc == 1) && (sscanf(argv[0], "%u", &repeat_count) != 1)) {
+ *error = "round-robin ps: invalid repeat count";
+ return -EINVAL;
+ }
+
+ /* allocate the path */
+ pi = kmalloc(sizeof(*pi), GFP_KERNEL);
+ if (!pi) {
+ *error = "round-robin ps: Error allocating path context";
+ return -ENOMEM;
+ }
+
+ pi->path = path;
+ pi->repeat_count = repeat_count;
+
+ path->pscontext = pi;
+
+ list_add(&pi->list, &s->valid_paths);
+
+ return 0;
+}
+
+static void rr_fail_path(struct path_selector *ps, struct path *p)
+{
+ struct selector *s = (struct selector *) ps->context;
+ struct path_info *pi = p->pscontext;
+
+ list_move(&pi->list, &s->invalid_paths);
+}
+
+static int rr_reinstate_path(struct path_selector *ps, struct path *p)
+{
+ struct selector *s = (struct selector *) ps->context;
+ struct path_info *pi = p->pscontext;
+
+ list_move(&pi->list, &s->valid_paths);
+
+ return 0;
+}
+
+static struct path *rr_select_path(struct path_selector *ps,
+ unsigned *repeat_count)
+{
+ struct selector *s = (struct selector *) ps->context;
+ struct path_info *pi = NULL;
+
+ if (!list_empty(&s->valid_paths)) {
+ pi = list_entry(s->valid_paths.next, struct path_info, list);
+ list_move_tail(&pi->list, &s->valid_paths);
+ *repeat_count = pi->repeat_count;
+ }
+
+ return pi ? pi->path : NULL;
+}
+
+static struct path_selector_type rr_ps = {
+ .name = "round-robin",
+ .module = THIS_MODULE,
+ .table_args = 1,
+ .info_args = 0,
+ .create = rr_create,
+ .destroy = rr_destroy,
+ .status = rr_status,
+ .add_path = rr_add_path,
+ .fail_path = rr_fail_path,
+ .reinstate_path = rr_reinstate_path,
+ .select_path = rr_select_path,
+};
+
+static int __init dm_rr_init(void)
+{
+ int r = dm_register_path_selector(&rr_ps);
+
+ if (r < 0)
+ DMERR("round-robin: register failed %d", r);
+
+ DMINFO("dm-round-robin version 1.0.0 loaded");
+
+ return r;
+}
+
+static void __exit dm_rr_exit(void)
+{
+ int r = dm_unregister_path_selector(&rr_ps);
+
+ if (r < 0)
+ DMERR("round-robin: unregister failed %d", r);
+}
+
+module_init(dm_rr_init);
+module_exit(dm_rr_exit);
+
+MODULE_DESCRIPTION(DM_NAME " round-robin multipath path selector");
+MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c
new file mode 100644
index 000000000000..7e691ab9a748
--- /dev/null
+++ b/drivers/md/dm-snap.c
@@ -0,0 +1,1208 @@
+/*
+ * dm-snapshot.c
+ *
+ * Copyright (C) 2001-2002 Sistina Software (UK) Limited.
+ *
+ * This file is released under the GPL.
+ */
+
+#include <linux/blkdev.h>
+#include <linux/config.h>
+#include <linux/ctype.h>
+#include <linux/device-mapper.h>
+#include <linux/fs.h>
+#include <linux/init.h>
+#include <linux/kdev_t.h>
+#include <linux/list.h>
+#include <linux/mempool.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+
+#include "dm-snap.h"
+#include "dm-bio-list.h"
+#include "kcopyd.h"
+
+/*
+ * The percentage increment we will wake up users at
+ */
+#define WAKE_UP_PERCENT 5
+
+/*
+ * kcopyd priority of snapshot operations
+ */
+#define SNAPSHOT_COPY_PRIORITY 2
+
+/*
+ * Each snapshot reserves this many pages for io
+ */
+#define SNAPSHOT_PAGES 256
+
+struct pending_exception {
+ struct exception e;
+
+ /*
+ * Origin buffers waiting for this to complete are held
+ * in a bio list
+ */
+ struct bio_list origin_bios;
+ struct bio_list snapshot_bios;
+
+ /*
+ * Other pending_exceptions that are processing this
+ * chunk. When this list is empty, we know we can
+ * complete the origins.
+ */
+ struct list_head siblings;
+
+ /* Pointer back to snapshot context */
+ struct dm_snapshot *snap;
+
+ /*
+ * 1 indicates the exception has already been sent to
+ * kcopyd.
+ */
+ int started;
+};
+
+/*
+ * Hash table mapping origin volumes to lists of snapshots and
+ * a lock to protect it
+ */
+static kmem_cache_t *exception_cache;
+static kmem_cache_t *pending_cache;
+static mempool_t *pending_pool;
+
+/*
+ * One of these per registered origin, held in the snapshot_origins hash
+ */
+struct origin {
+ /* The origin device */
+ struct block_device *bdev;
+
+ struct list_head hash_list;
+
+ /* List of snapshots for this origin */
+ struct list_head snapshots;
+};
+
+/*
+ * Size of the hash table for origin volumes. If we make this
+ * the size of the minors list then it should be nearly perfect
+ */
+#define ORIGIN_HASH_SIZE 256
+#define ORIGIN_MASK 0xFF
+static struct list_head *_origins;
+static struct rw_semaphore _origins_lock;
+
+static int init_origin_hash(void)
+{
+ int i;
+
+ _origins = kmalloc(ORIGIN_HASH_SIZE * sizeof(struct list_head),
+ GFP_KERNEL);
+ if (!_origins) {
+ DMERR("Device mapper: Snapshot: unable to allocate memory");
+ return -ENOMEM;
+ }
+
+ for (i = 0; i < ORIGIN_HASH_SIZE; i++)
+ INIT_LIST_HEAD(_origins + i);
+ init_rwsem(&_origins_lock);
+
+ return 0;
+}
+
+static void exit_origin_hash(void)
+{
+ kfree(_origins);
+}
+
+static inline unsigned int origin_hash(struct block_device *bdev)
+{
+ return bdev->bd_dev & ORIGIN_MASK;
+}
+
+static struct origin *__lookup_origin(struct block_device *origin)
+{
+ struct list_head *ol;
+ struct origin *o;
+
+ ol = &_origins[origin_hash(origin)];
+ list_for_each_entry (o, ol, hash_list)
+ if (bdev_equal(o->bdev, origin))
+ return o;
+
+ return NULL;
+}
+
+static void __insert_origin(struct origin *o)
+{
+ struct list_head *sl = &_origins[origin_hash(o->bdev)];
+ list_add_tail(&o->hash_list, sl);
+}
+
+/*
+ * Make a note of the snapshot and its origin so we can look it
+ * up when the origin has a write on it.
+ */
+static int register_snapshot(struct dm_snapshot *snap)
+{
+ struct origin *o;
+ struct block_device *bdev = snap->origin->bdev;
+
+ down_write(&_origins_lock);
+ o = __lookup_origin(bdev);
+
+ if (!o) {
+ /* New origin */
+ o = kmalloc(sizeof(*o), GFP_KERNEL);
+ if (!o) {
+ up_write(&_origins_lock);
+ return -ENOMEM;
+ }
+
+ /* Initialise the struct */
+ INIT_LIST_HEAD(&o->snapshots);
+ o->bdev = bdev;
+
+ __insert_origin(o);
+ }
+
+ list_add_tail(&snap->list, &o->snapshots);
+
+ up_write(&_origins_lock);
+ return 0;
+}
+
+static void unregister_snapshot(struct dm_snapshot *s)
+{
+ struct origin *o;
+
+ down_write(&_origins_lock);
+ o = __lookup_origin(s->origin->bdev);
+
+ list_del(&s->list);
+ if (list_empty(&o->snapshots)) {
+ list_del(&o->hash_list);
+ kfree(o);
+ }
+
+ up_write(&_origins_lock);
+}
+
+/*
+ * Implementation of the exception hash tables.
+ */
+static int init_exception_table(struct exception_table *et, uint32_t size)
+{
+ unsigned int i;
+
+ et->hash_mask = size - 1;
+ et->table = dm_vcalloc(size, sizeof(struct list_head));
+ if (!et->table)
+ return -ENOMEM;
+
+ for (i = 0; i < size; i++)
+ INIT_LIST_HEAD(et->table + i);
+
+ return 0;
+}
+
+static void exit_exception_table(struct exception_table *et, kmem_cache_t *mem)
+{
+ struct list_head *slot;
+ struct exception *ex, *next;
+ int i, size;
+
+ size = et->hash_mask + 1;
+ for (i = 0; i < size; i++) {
+ slot = et->table + i;
+
+ list_for_each_entry_safe (ex, next, slot, hash_list)
+ kmem_cache_free(mem, ex);
+ }
+
+ vfree(et->table);
+}
+
+static inline uint32_t exception_hash(struct exception_table *et, chunk_t chunk)
+{
+ return chunk & et->hash_mask;
+}
+
+static void insert_exception(struct exception_table *eh, struct exception *e)
+{
+ struct list_head *l = &eh->table[exception_hash(eh, e->old_chunk)];
+ list_add(&e->hash_list, l);
+}
+
+static inline void remove_exception(struct exception *e)
+{
+ list_del(&e->hash_list);
+}
+
+/*
+ * Return the exception data for a sector, or NULL if not
+ * remapped.
+ */
+static struct exception *lookup_exception(struct exception_table *et,
+ chunk_t chunk)
+{
+ struct list_head *slot;
+ struct exception *e;
+
+ slot = &et->table[exception_hash(et, chunk)];
+ list_for_each_entry (e, slot, hash_list)
+ if (e->old_chunk == chunk)
+ return e;
+
+ return NULL;
+}
+
+static inline struct exception *alloc_exception(void)
+{
+ struct exception *e;
+
+ e = kmem_cache_alloc(exception_cache, GFP_NOIO);
+ if (!e)
+ e = kmem_cache_alloc(exception_cache, GFP_ATOMIC);
+
+ return e;
+}
+
+static inline void free_exception(struct exception *e)
+{
+ kmem_cache_free(exception_cache, e);
+}
+
+static inline struct pending_exception *alloc_pending_exception(void)
+{
+ return mempool_alloc(pending_pool, GFP_NOIO);
+}
+
+static inline void free_pending_exception(struct pending_exception *pe)
+{
+ mempool_free(pe, pending_pool);
+}
+
+int dm_add_exception(struct dm_snapshot *s, chunk_t old, chunk_t new)
+{
+ struct exception *e;
+
+ e = alloc_exception();
+ if (!e)
+ return -ENOMEM;
+
+ e->old_chunk = old;
+ e->new_chunk = new;
+ insert_exception(&s->complete, e);
+ return 0;
+}
+
+/*
+ * Hard coded magic.
+ */
+static int calc_max_buckets(void)
+{
+ /* use a fixed size of 2MB */
+ unsigned long mem = 2 * 1024 * 1024;
+ mem /= sizeof(struct list_head);
+
+ return mem;
+}
+
+/*
+ * Rounds a number down to a power of 2.
+ */
+static inline uint32_t round_down(uint32_t n)
+{
+ while (n & (n - 1))
+ n &= (n - 1);
+ return n;
+}
+
+/*
+ * Allocate room for a suitable hash table.
+ */
+static int init_hash_tables(struct dm_snapshot *s)
+{
+ sector_t hash_size, cow_dev_size, origin_dev_size, max_buckets;
+
+ /*
+ * Calculate based on the size of the original volume or
+ * the COW volume...
+ */
+ cow_dev_size = get_dev_size(s->cow->bdev);
+ origin_dev_size = get_dev_size(s->origin->bdev);
+ max_buckets = calc_max_buckets();
+
+ hash_size = min(origin_dev_size, cow_dev_size) >> s->chunk_shift;
+ hash_size = min(hash_size, max_buckets);
+
+ /* Round it down to a power of 2 */
+ hash_size = round_down(hash_size);
+ if (init_exception_table(&s->complete, hash_size))
+ return -ENOMEM;
+
+ /*
+ * Allocate hash table for in-flight exceptions
+ * Make this smaller than the real hash table
+ */
+ hash_size >>= 3;
+ if (hash_size < 64)
+ hash_size = 64;
+
+ if (init_exception_table(&s->pending, hash_size)) {
+ exit_exception_table(&s->complete, exception_cache);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+/*
+ * Round a number up to the nearest 'size' boundary. size must
+ * be a power of 2.
+ */
+static inline ulong round_up(ulong n, ulong size)
+{
+ size--;
+ return (n + size) & ~size;
+}
+
+/*
+ * Construct a snapshot mapping: <origin_dev> <COW-dev> <p/n> <chunk-size>
+ */
+static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+ struct dm_snapshot *s;
+ unsigned long chunk_size;
+ int r = -EINVAL;
+ char persistent;
+ char *origin_path;
+ char *cow_path;
+ char *value;
+ int blocksize;
+
+ if (argc < 4) {
+ ti->error = "dm-snapshot: requires exactly 4 arguments";
+ r = -EINVAL;
+ goto bad1;
+ }
+
+ origin_path = argv[0];
+ cow_path = argv[1];
+ persistent = toupper(*argv[2]);
+
+ if (persistent != 'P' && persistent != 'N') {
+ ti->error = "Persistent flag is not P or N";
+ r = -EINVAL;
+ goto bad1;
+ }
+
+ chunk_size = simple_strtoul(argv[3], &value, 10);
+ if (chunk_size == 0 || value == NULL) {
+ ti->error = "Invalid chunk size";
+ r = -EINVAL;
+ goto bad1;
+ }
+
+ s = kmalloc(sizeof(*s), GFP_KERNEL);
+ if (s == NULL) {
+ ti->error = "Cannot allocate snapshot context private "
+ "structure";
+ r = -ENOMEM;
+ goto bad1;
+ }
+
+ r = dm_get_device(ti, origin_path, 0, ti->len, FMODE_READ, &s->origin);
+ if (r) {
+ ti->error = "Cannot get origin device";
+ goto bad2;
+ }
+
+ r = dm_get_device(ti, cow_path, 0, 0,
+ FMODE_READ | FMODE_WRITE, &s->cow);
+ if (r) {
+ dm_put_device(ti, s->origin);
+ ti->error = "Cannot get COW device";
+ goto bad2;
+ }
+
+ /*
+ * Chunk size must be multiple of page size. Silently
+ * round up if it's not.
+ */
+ chunk_size = round_up(chunk_size, PAGE_SIZE >> 9);
+
+ /* Validate the chunk size against the device block size */
+ blocksize = s->cow->bdev->bd_disk->queue->hardsect_size;
+ if (chunk_size % (blocksize >> 9)) {
+ ti->error = "Chunk size is not a multiple of device blocksize";
+ r = -EINVAL;
+ goto bad3;
+ }
+
+ /* Check chunk_size is a power of 2 */
+ if (chunk_size & (chunk_size - 1)) {
+ ti->error = "Chunk size is not a power of 2";
+ r = -EINVAL;
+ goto bad3;
+ }
+
+ s->chunk_size = chunk_size;
+ s->chunk_mask = chunk_size - 1;
+ s->type = persistent;
+ s->chunk_shift = ffs(chunk_size) - 1;
+
+ s->valid = 1;
+ s->have_metadata = 0;
+ s->last_percent = 0;
+ init_rwsem(&s->lock);
+ s->table = ti->table;
+
+ /* Allocate hash table for COW data */
+ if (init_hash_tables(s)) {
+ ti->error = "Unable to allocate hash table space";
+ r = -ENOMEM;
+ goto bad3;
+ }
+
+ /*
+ * Check the persistent flag - done here because we need the iobuf
+ * to check the LV header
+ */
+ s->store.snap = s;
+
+ if (persistent == 'P')
+ r = dm_create_persistent(&s->store, chunk_size);
+ else
+ r = dm_create_transient(&s->store, s, blocksize);
+
+ if (r) {
+ ti->error = "Couldn't create exception store";
+ r = -EINVAL;
+ goto bad4;
+ }
+
+ r = kcopyd_client_create(SNAPSHOT_PAGES, &s->kcopyd_client);
+ if (r) {
+ ti->error = "Could not create kcopyd client";
+ goto bad5;
+ }
+
+ /* Add snapshot to the list of snapshots for this origin */
+ if (register_snapshot(s)) {
+ r = -EINVAL;
+ ti->error = "Cannot register snapshot origin";
+ goto bad6;
+ }
+
+ ti->private = s;
+ ti->split_io = chunk_size;
+
+ return 0;
+
+ bad6:
+ kcopyd_client_destroy(s->kcopyd_client);
+
+ bad5:
+ s->store.destroy(&s->store);
+
+ bad4:
+ exit_exception_table(&s->pending, pending_cache);
+ exit_exception_table(&s->complete, exception_cache);
+
+ bad3:
+ dm_put_device(ti, s->cow);
+ dm_put_device(ti, s->origin);
+
+ bad2:
+ kfree(s);
+
+ bad1:
+ return r;
+}
+
+static void snapshot_dtr(struct dm_target *ti)
+{
+ struct dm_snapshot *s = (struct dm_snapshot *) ti->private;
+
+ unregister_snapshot(s);
+
+ exit_exception_table(&s->pending, pending_cache);
+ exit_exception_table(&s->complete, exception_cache);
+
+ /* Deallocate memory used */
+ s->store.destroy(&s->store);
+
+ dm_put_device(ti, s->origin);
+ dm_put_device(ti, s->cow);
+ kcopyd_client_destroy(s->kcopyd_client);
+ kfree(s);
+}
+
+/*
+ * Flush a list of buffers.
+ */
+static void flush_bios(struct bio *bio)
+{
+ struct bio *n;
+
+ while (bio) {
+ n = bio->bi_next;
+ bio->bi_next = NULL;
+ generic_make_request(bio);
+ bio = n;
+ }
+}
+
+/*
+ * Error a list of buffers.
+ */
+static void error_bios(struct bio *bio)
+{
+ struct bio *n;
+
+ while (bio) {
+ n = bio->bi_next;
+ bio->bi_next = NULL;
+ bio_io_error(bio, bio->bi_size);
+ bio = n;
+ }
+}
+
+static struct bio *__flush_bios(struct pending_exception *pe)
+{
+ struct pending_exception *sibling;
+
+ if (list_empty(&pe->siblings))
+ return bio_list_get(&pe->origin_bios);
+
+ sibling = list_entry(pe->siblings.next,
+ struct pending_exception, siblings);
+
+ list_del(&pe->siblings);
+
+ /* This is fine as long as kcopyd is single-threaded. If kcopyd
+ * becomes multi-threaded, we'll need some locking here.
+ */
+ bio_list_merge(&sibling->origin_bios, &pe->origin_bios);
+
+ return NULL;
+}
+
+static void pending_complete(struct pending_exception *pe, int success)
+{
+ struct exception *e;
+ struct dm_snapshot *s = pe->snap;
+ struct bio *flush = NULL;
+
+ if (success) {
+ e = alloc_exception();
+ if (!e) {
+ DMWARN("Unable to allocate exception.");
+ down_write(&s->lock);
+ s->store.drop_snapshot(&s->store);
+ s->valid = 0;
+ flush = __flush_bios(pe);
+ up_write(&s->lock);
+
+ error_bios(bio_list_get(&pe->snapshot_bios));
+ goto out;
+ }
+ *e = pe->e;
+
+ /*
+ * Add a proper exception, and remove the
+ * in-flight exception from the list.
+ */
+ down_write(&s->lock);
+ insert_exception(&s->complete, e);
+ remove_exception(&pe->e);
+ flush = __flush_bios(pe);
+
+ /* Submit any pending write bios */
+ up_write(&s->lock);
+
+ flush_bios(bio_list_get(&pe->snapshot_bios));
+ } else {
+ /* Read/write error - snapshot is unusable */
+ down_write(&s->lock);
+ if (s->valid)
+ DMERR("Error reading/writing snapshot");
+ s->store.drop_snapshot(&s->store);
+ s->valid = 0;
+ remove_exception(&pe->e);
+ flush = __flush_bios(pe);
+ up_write(&s->lock);
+
+ error_bios(bio_list_get(&pe->snapshot_bios));
+
+ dm_table_event(s->table);
+ }
+
+ out:
+ free_pending_exception(pe);
+
+ if (flush)
+ flush_bios(flush);
+}
+
+static void commit_callback(void *context, int success)
+{
+ struct pending_exception *pe = (struct pending_exception *) context;
+ pending_complete(pe, success);
+}
+
+/*
+ * Called when the copy I/O has finished. kcopyd actually runs
+ * this code so don't block.
+ */
+static void copy_callback(int read_err, unsigned int write_err, void *context)
+{
+ struct pending_exception *pe = (struct pending_exception *) context;
+ struct dm_snapshot *s = pe->snap;
+
+ if (read_err || write_err)
+ pending_complete(pe, 0);
+
+ else
+ /* Update the metadata if we are persistent */
+ s->store.commit_exception(&s->store, &pe->e, commit_callback,
+ pe);
+}
+
+/*
+ * Dispatches the copy operation to kcopyd.
+ */
+static inline void start_copy(struct pending_exception *pe)
+{
+ struct dm_snapshot *s = pe->snap;
+ struct io_region src, dest;
+ struct block_device *bdev = s->origin->bdev;
+ sector_t dev_size;
+
+ dev_size = get_dev_size(bdev);
+
+ src.bdev = bdev;
+ src.sector = chunk_to_sector(s, pe->e.old_chunk);
+ src.count = min(s->chunk_size, dev_size - src.sector);
+
+ dest.bdev = s->cow->bdev;
+ dest.sector = chunk_to_sector(s, pe->e.new_chunk);
+ dest.count = src.count;
+
+ /* Hand over to kcopyd */
+ kcopyd_copy(s->kcopyd_client,
+ &src, 1, &dest, 0, copy_callback, pe);
+}
+
+/*
+ * Looks to see if this snapshot already has a pending exception
+ * for this chunk, otherwise it allocates a new one and inserts
+ * it into the pending table.
+ *
+ * NOTE: a write lock must be held on snap->lock before calling
+ * this.
+ */
+static struct pending_exception *
+__find_pending_exception(struct dm_snapshot *s, struct bio *bio)
+{
+ struct exception *e;
+ struct pending_exception *pe;
+ chunk_t chunk = sector_to_chunk(s, bio->bi_sector);
+
+ /*
+ * Is there a pending exception for this already ?
+ */
+ e = lookup_exception(&s->pending, chunk);
+ if (e) {
+ /* cast the exception to a pending exception */
+ pe = container_of(e, struct pending_exception, e);
+
+ } else {
+ /*
+ * Create a new pending exception, we don't want
+ * to hold the lock while we do this.
+ */
+ up_write(&s->lock);
+ pe = alloc_pending_exception();
+ down_write(&s->lock);
+
+ e = lookup_exception(&s->pending, chunk);
+ if (e) {
+ free_pending_exception(pe);
+ pe = container_of(e, struct pending_exception, e);
+ } else {
+ pe->e.old_chunk = chunk;
+ bio_list_init(&pe->origin_bios);
+ bio_list_init(&pe->snapshot_bios);
+ INIT_LIST_HEAD(&pe->siblings);
+ pe->snap = s;
+ pe->started = 0;
+
+ if (s->store.prepare_exception(&s->store, &pe->e)) {
+ free_pending_exception(pe);
+ s->valid = 0;
+ return NULL;
+ }
+
+ insert_exception(&s->pending, &pe->e);
+ }
+ }
+
+ return pe;
+}
+
+static inline void remap_exception(struct dm_snapshot *s, struct exception *e,
+ struct bio *bio)
+{
+ bio->bi_bdev = s->cow->bdev;
+ bio->bi_sector = chunk_to_sector(s, e->new_chunk) +
+ (bio->bi_sector & s->chunk_mask);
+}
+
+static int snapshot_map(struct dm_target *ti, struct bio *bio,
+ union map_info *map_context)
+{
+ struct exception *e;
+ struct dm_snapshot *s = (struct dm_snapshot *) ti->private;
+ int r = 1;
+ chunk_t chunk;
+ struct pending_exception *pe;
+
+ chunk = sector_to_chunk(s, bio->bi_sector);
+
+ /* Full snapshots are not usable */
+ if (!s->valid)
+ return -1;
+
+ /*
+ * Write to snapshot - higher level takes care of RW/RO
+ * flags so we should only get this if we are
+ * writeable.
+ */
+ if (bio_rw(bio) == WRITE) {
+
+ /* FIXME: should only take write lock if we need
+ * to copy an exception */
+ down_write(&s->lock);
+
+ /* If the block is already remapped - use that, else remap it */
+ e = lookup_exception(&s->complete, chunk);
+ if (e) {
+ remap_exception(s, e, bio);
+ up_write(&s->lock);
+
+ } else {
+ pe = __find_pending_exception(s, bio);
+
+ if (!pe) {
+ if (s->store.drop_snapshot)
+ s->store.drop_snapshot(&s->store);
+ s->valid = 0;
+ r = -EIO;
+ up_write(&s->lock);
+ } else {
+ remap_exception(s, &pe->e, bio);
+ bio_list_add(&pe->snapshot_bios, bio);
+
+ if (!pe->started) {
+ /* this is protected by snap->lock */
+ pe->started = 1;
+ up_write(&s->lock);
+ start_copy(pe);
+ } else
+ up_write(&s->lock);
+ r = 0;
+ }
+ }
+
+ } else {
+ /*
+ * FIXME: this read path scares me because we
+ * always use the origin when we have a pending
+ * exception. However I can't think of a
+ * situation where this is wrong - ejt.
+ */
+
+ /* Do reads */
+ down_read(&s->lock);
+
+ /* See if it it has been remapped */
+ e = lookup_exception(&s->complete, chunk);
+ if (e)
+ remap_exception(s, e, bio);
+ else
+ bio->bi_bdev = s->origin->bdev;
+
+ up_read(&s->lock);
+ }
+
+ return r;
+}
+
+static void snapshot_resume(struct dm_target *ti)
+{
+ struct dm_snapshot *s = (struct dm_snapshot *) ti->private;
+
+ if (s->have_metadata)
+ return;
+
+ if (s->store.read_metadata(&s->store)) {
+ down_write(&s->lock);
+ s->valid = 0;
+ up_write(&s->lock);
+ }
+
+ s->have_metadata = 1;
+}
+
+static int snapshot_status(struct dm_target *ti, status_type_t type,
+ char *result, unsigned int maxlen)
+{
+ struct dm_snapshot *snap = (struct dm_snapshot *) ti->private;
+
+ switch (type) {
+ case STATUSTYPE_INFO:
+ if (!snap->valid)
+ snprintf(result, maxlen, "Invalid");
+ else {
+ if (snap->store.fraction_full) {
+ sector_t numerator, denominator;
+ snap->store.fraction_full(&snap->store,
+ &numerator,
+ &denominator);
+ snprintf(result, maxlen,
+ SECTOR_FORMAT "/" SECTOR_FORMAT,
+ numerator, denominator);
+ }
+ else
+ snprintf(result, maxlen, "Unknown");
+ }
+ break;
+
+ case STATUSTYPE_TABLE:
+ /*
+ * kdevname returns a static pointer so we need
+ * to make private copies if the output is to
+ * make sense.
+ */
+ snprintf(result, maxlen, "%s %s %c " SECTOR_FORMAT,
+ snap->origin->name, snap->cow->name,
+ snap->type, snap->chunk_size);
+ break;
+ }
+
+ return 0;
+}
+
+/*-----------------------------------------------------------------
+ * Origin methods
+ *---------------------------------------------------------------*/
+static void list_merge(struct list_head *l1, struct list_head *l2)
+{
+ struct list_head *l1_n, *l2_p;
+
+ l1_n = l1->next;
+ l2_p = l2->prev;
+
+ l1->next = l2;
+ l2->prev = l1;
+
+ l2_p->next = l1_n;
+ l1_n->prev = l2_p;
+}
+
+static int __origin_write(struct list_head *snapshots, struct bio *bio)
+{
+ int r = 1, first = 1;
+ struct dm_snapshot *snap;
+ struct exception *e;
+ struct pending_exception *pe, *last = NULL;
+ chunk_t chunk;
+
+ /* Do all the snapshots on this origin */
+ list_for_each_entry (snap, snapshots, list) {
+
+ /* Only deal with valid snapshots */
+ if (!snap->valid)
+ continue;
+
+ down_write(&snap->lock);
+
+ /*
+ * Remember, different snapshots can have
+ * different chunk sizes.
+ */
+ chunk = sector_to_chunk(snap, bio->bi_sector);
+
+ /*
+ * Check exception table to see if block
+ * is already remapped in this snapshot
+ * and trigger an exception if not.
+ */
+ e = lookup_exception(&snap->complete, chunk);
+ if (!e) {
+ pe = __find_pending_exception(snap, bio);
+ if (!pe) {
+ snap->store.drop_snapshot(&snap->store);
+ snap->valid = 0;
+
+ } else {
+ if (last)
+ list_merge(&pe->siblings,
+ &last->siblings);
+
+ last = pe;
+ r = 0;
+ }
+ }
+
+ up_write(&snap->lock);
+ }
+
+ /*
+ * Now that we have a complete pe list we can start the copying.
+ */
+ if (last) {
+ pe = last;
+ do {
+ down_write(&pe->snap->lock);
+ if (first)
+ bio_list_add(&pe->origin_bios, bio);
+ if (!pe->started) {
+ pe->started = 1;
+ up_write(&pe->snap->lock);
+ start_copy(pe);
+ } else
+ up_write(&pe->snap->lock);
+ first = 0;
+ pe = list_entry(pe->siblings.next,
+ struct pending_exception, siblings);
+
+ } while (pe != last);
+ }
+
+ return r;
+}
+
+/*
+ * Called on a write from the origin driver.
+ */
+static int do_origin(struct dm_dev *origin, struct bio *bio)
+{
+ struct origin *o;
+ int r = 1;
+
+ down_read(&_origins_lock);
+ o = __lookup_origin(origin->bdev);
+ if (o)
+ r = __origin_write(&o->snapshots, bio);
+ up_read(&_origins_lock);
+
+ return r;
+}
+
+/*
+ * Origin: maps a linear range of a device, with hooks for snapshotting.
+ */
+
+/*
+ * Construct an origin mapping: <dev_path>
+ * The context for an origin is merely a 'struct dm_dev *'
+ * pointing to the real device.
+ */
+static int origin_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+ int r;
+ struct dm_dev *dev;
+
+ if (argc != 1) {
+ ti->error = "dm-origin: incorrect number of arguments";
+ return -EINVAL;
+ }
+
+ r = dm_get_device(ti, argv[0], 0, ti->len,
+ dm_table_get_mode(ti->table), &dev);
+ if (r) {
+ ti->error = "Cannot get target device";
+ return r;
+ }
+
+ ti->private = dev;
+ return 0;
+}
+
+static void origin_dtr(struct dm_target *ti)
+{
+ struct dm_dev *dev = (struct dm_dev *) ti->private;
+ dm_put_device(ti, dev);
+}
+
+static int origin_map(struct dm_target *ti, struct bio *bio,
+ union map_info *map_context)
+{
+ struct dm_dev *dev = (struct dm_dev *) ti->private;
+ bio->bi_bdev = dev->bdev;
+
+ /* Only tell snapshots if this is a write */
+ return (bio_rw(bio) == WRITE) ? do_origin(dev, bio) : 1;
+}
+
+#define min_not_zero(l, r) (l == 0) ? r : ((r == 0) ? l : min(l, r))
+
+/*
+ * Set the target "split_io" field to the minimum of all the snapshots'
+ * chunk sizes.
+ */
+static void origin_resume(struct dm_target *ti)
+{
+ struct dm_dev *dev = (struct dm_dev *) ti->private;
+ struct dm_snapshot *snap;
+ struct origin *o;
+ chunk_t chunk_size = 0;
+
+ down_read(&_origins_lock);
+ o = __lookup_origin(dev->bdev);
+ if (o)
+ list_for_each_entry (snap, &o->snapshots, list)
+ chunk_size = min_not_zero(chunk_size, snap->chunk_size);
+ up_read(&_origins_lock);
+
+ ti->split_io = chunk_size;
+}
+
+static int origin_status(struct dm_target *ti, status_type_t type, char *result,
+ unsigned int maxlen)
+{
+ struct dm_dev *dev = (struct dm_dev *) ti->private;
+
+ switch (type) {
+ case STATUSTYPE_INFO:
+ result[0] = '\0';
+ break;
+
+ case STATUSTYPE_TABLE:
+ snprintf(result, maxlen, "%s", dev->name);
+ break;
+ }
+
+ return 0;
+}
+
+static struct target_type origin_target = {
+ .name = "snapshot-origin",
+ .version = {1, 0, 1},
+ .module = THIS_MODULE,
+ .ctr = origin_ctr,
+ .dtr = origin_dtr,
+ .map = origin_map,
+ .resume = origin_resume,
+ .status = origin_status,
+};
+
+static struct target_type snapshot_target = {
+ .name = "snapshot",
+ .version = {1, 0, 1},
+ .module = THIS_MODULE,
+ .ctr = snapshot_ctr,
+ .dtr = snapshot_dtr,
+ .map = snapshot_map,
+ .resume = snapshot_resume,
+ .status = snapshot_status,
+};
+
+static int __init dm_snapshot_init(void)
+{
+ int r;
+
+ r = dm_register_target(&snapshot_target);
+ if (r) {
+ DMERR("snapshot target register failed %d", r);
+ return r;
+ }
+
+ r = dm_register_target(&origin_target);
+ if (r < 0) {
+ DMERR("Device mapper: Origin: register failed %d\n", r);
+ goto bad1;
+ }
+
+ r = init_origin_hash();
+ if (r) {
+ DMERR("init_origin_hash failed.");
+ goto bad2;
+ }
+
+ exception_cache = kmem_cache_create("dm-snapshot-ex",
+ sizeof(struct exception),
+ __alignof__(struct exception),
+ 0, NULL, NULL);
+ if (!exception_cache) {
+ DMERR("Couldn't create exception cache.");
+ r = -ENOMEM;
+ goto bad3;
+ }
+
+ pending_cache =
+ kmem_cache_create("dm-snapshot-in",
+ sizeof(struct pending_exception),
+ __alignof__(struct pending_exception),
+ 0, NULL, NULL);
+ if (!pending_cache) {
+ DMERR("Couldn't create pending cache.");
+ r = -ENOMEM;
+ goto bad4;
+ }
+
+ pending_pool = mempool_create(128, mempool_alloc_slab,
+ mempool_free_slab, pending_cache);
+ if (!pending_pool) {
+ DMERR("Couldn't create pending pool.");
+ r = -ENOMEM;
+ goto bad5;
+ }
+
+ return 0;
+
+ bad5:
+ kmem_cache_destroy(pending_cache);
+ bad4:
+ kmem_cache_destroy(exception_cache);
+ bad3:
+ exit_origin_hash();
+ bad2:
+ dm_unregister_target(&origin_target);
+ bad1:
+ dm_unregister_target(&snapshot_target);
+ return r;
+}
+
+static void __exit dm_snapshot_exit(void)
+{
+ int r;
+
+ r = dm_unregister_target(&snapshot_target);
+ if (r)
+ DMERR("snapshot unregister failed %d", r);
+
+ r = dm_unregister_target(&origin_target);
+ if (r)
+ DMERR("origin unregister failed %d", r);
+
+ exit_origin_hash();
+ mempool_destroy(pending_pool);
+ kmem_cache_destroy(pending_cache);
+ kmem_cache_destroy(exception_cache);
+}
+
+/* Module hooks */
+module_init(dm_snapshot_init);
+module_exit(dm_snapshot_exit);
+
+MODULE_DESCRIPTION(DM_NAME " snapshot target");
+MODULE_AUTHOR("Joe Thornber");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-snap.h b/drivers/md/dm-snap.h
new file mode 100644
index 000000000000..375aa24d4d7d
--- /dev/null
+++ b/drivers/md/dm-snap.h
@@ -0,0 +1,161 @@
+/*
+ * dm-snapshot.c
+ *
+ * Copyright (C) 2001-2002 Sistina Software (UK) Limited.
+ *
+ * This file is released under the GPL.
+ */
+
+#ifndef DM_SNAPSHOT_H
+#define DM_SNAPSHOT_H
+
+#include "dm.h"
+#include <linux/blkdev.h>
+
+struct exception_table {
+ uint32_t hash_mask;
+ struct list_head *table;
+};
+
+/*
+ * The snapshot code deals with largish chunks of the disk at a
+ * time. Typically 64k - 256k.
+ */
+/* FIXME: can we get away with limiting these to a uint32_t ? */
+typedef sector_t chunk_t;
+
+/*
+ * An exception is used where an old chunk of data has been
+ * replaced by a new one.
+ */
+struct exception {
+ struct list_head hash_list;
+
+ chunk_t old_chunk;
+ chunk_t new_chunk;
+};
+
+/*
+ * Abstraction to handle the meta/layout of exception stores (the
+ * COW device).
+ */
+struct exception_store {
+
+ /*
+ * Destroys this object when you've finished with it.
+ */
+ void (*destroy) (struct exception_store *store);
+
+ /*
+ * The target shouldn't read the COW device until this is
+ * called.
+ */
+ int (*read_metadata) (struct exception_store *store);
+
+ /*
+ * Find somewhere to store the next exception.
+ */
+ int (*prepare_exception) (struct exception_store *store,
+ struct exception *e);
+
+ /*
+ * Update the metadata with this exception.
+ */
+ void (*commit_exception) (struct exception_store *store,
+ struct exception *e,
+ void (*callback) (void *, int success),
+ void *callback_context);
+
+ /*
+ * The snapshot is invalid, note this in the metadata.
+ */
+ void (*drop_snapshot) (struct exception_store *store);
+
+ /*
+ * Return how full the snapshot is.
+ */
+ void (*fraction_full) (struct exception_store *store,
+ sector_t *numerator,
+ sector_t *denominator);
+
+ struct dm_snapshot *snap;
+ void *context;
+};
+
+struct dm_snapshot {
+ struct rw_semaphore lock;
+ struct dm_table *table;
+
+ struct dm_dev *origin;
+ struct dm_dev *cow;
+
+ /* List of snapshots per Origin */
+ struct list_head list;
+
+ /* Size of data blocks saved - must be a power of 2 */
+ chunk_t chunk_size;
+ chunk_t chunk_mask;
+ chunk_t chunk_shift;
+
+ /* You can't use a snapshot if this is 0 (e.g. if full) */
+ int valid;
+ int have_metadata;
+
+ /* Used for display of table */
+ char type;
+
+ /* The last percentage we notified */
+ int last_percent;
+
+ struct exception_table pending;
+ struct exception_table complete;
+
+ /* The on disk metadata handler */
+ struct exception_store store;
+
+ struct kcopyd_client *kcopyd_client;
+};
+
+/*
+ * Used by the exception stores to load exceptions hen
+ * initialising.
+ */
+int dm_add_exception(struct dm_snapshot *s, chunk_t old, chunk_t new);
+
+/*
+ * Constructor and destructor for the default persistent
+ * store.
+ */
+int dm_create_persistent(struct exception_store *store, uint32_t chunk_size);
+
+int dm_create_transient(struct exception_store *store,
+ struct dm_snapshot *s, int blocksize);
+
+/*
+ * Return the number of sectors in the device.
+ */
+static inline sector_t get_dev_size(struct block_device *bdev)
+{
+ return bdev->bd_inode->i_size >> SECTOR_SHIFT;
+}
+
+static inline chunk_t sector_to_chunk(struct dm_snapshot *s, sector_t sector)
+{
+ return (sector & ~s->chunk_mask) >> s->chunk_shift;
+}
+
+static inline sector_t chunk_to_sector(struct dm_snapshot *s, chunk_t chunk)
+{
+ return chunk << s->chunk_shift;
+}
+
+static inline int bdev_equal(struct block_device *lhs, struct block_device *rhs)
+{
+ /*
+ * There is only ever one instance of a particular block
+ * device so we can compare pointers safely.
+ */
+ return lhs == rhs;
+}
+
+#endif
diff --git a/drivers/md/dm-stripe.c b/drivers/md/dm-stripe.c
new file mode 100644
index 000000000000..ab89278a56bf
--- /dev/null
+++ b/drivers/md/dm-stripe.c
@@ -0,0 +1,234 @@
+/*
+ * Copyright (C) 2001-2003 Sistina Software (UK) Limited.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/blkdev.h>
+#include <linux/bio.h>
+#include <linux/slab.h>
+
+struct stripe {
+ struct dm_dev *dev;
+ sector_t physical_start;
+};
+
+struct stripe_c {
+ uint32_t stripes;
+
+ /* The size of this target / num. stripes */
+ sector_t stripe_width;
+
+ /* stripe chunk size */
+ uint32_t chunk_shift;
+ sector_t chunk_mask;
+
+ struct stripe stripe[0];
+};
+
+static inline struct stripe_c *alloc_context(unsigned int stripes)
+{
+ size_t len;
+
+ if (array_too_big(sizeof(struct stripe_c), sizeof(struct stripe),
+ stripes))
+ return NULL;
+
+ len = sizeof(struct stripe_c) + (sizeof(struct stripe) * stripes);
+
+ return kmalloc(len, GFP_KERNEL);
+}
+
+/*
+ * Parse a single <dev> <sector> pair
+ */
+static int get_stripe(struct dm_target *ti, struct stripe_c *sc,
+ unsigned int stripe, char **argv)
+{
+ sector_t start;
+
+ if (sscanf(argv[1], SECTOR_FORMAT, &start) != 1)
+ return -EINVAL;
+
+ if (dm_get_device(ti, argv[0], start, sc->stripe_width,
+ dm_table_get_mode(ti->table),
+ &sc->stripe[stripe].dev))
+ return -ENXIO;
+
+ sc->stripe[stripe].physical_start = start;
+ return 0;
+}
+
+/*
+ * Construct a striped mapping.
+ * <number of stripes> <chunk size (2^^n)> [<dev_path> <offset>]+
+ */
+static int stripe_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+ struct stripe_c *sc;
+ sector_t width;
+ uint32_t stripes;
+ uint32_t chunk_size;
+ char *end;
+ int r;
+ unsigned int i;
+
+ if (argc < 2) {
+ ti->error = "dm-stripe: Not enough arguments";
+ return -EINVAL;
+ }
+
+ stripes = simple_strtoul(argv[0], &end, 10);
+ if (*end) {
+ ti->error = "dm-stripe: Invalid stripe count";
+ return -EINVAL;
+ }
+
+ chunk_size = simple_strtoul(argv[1], &end, 10);
+ if (*end) {
+ ti->error = "dm-stripe: Invalid chunk_size";
+ return -EINVAL;
+ }
+
+ /*
+ * chunk_size is a power of two
+ */
+ if (!chunk_size || (chunk_size & (chunk_size - 1)) ||
+ (chunk_size < (PAGE_SIZE >> SECTOR_SHIFT))) {
+ ti->error = "dm-stripe: Invalid chunk size";
+ return -EINVAL;
+ }
+
+ width = ti->len;
+ if (sector_div(width, stripes)) {
+ ti->error = "dm-stripe: Target length not divisable by "
+ "number of stripes";
+ return -EINVAL;
+ }
+
+ /*
+ * Do we have enough arguments for that many stripes ?
+ */
+ if (argc != (2 + 2 * stripes)) {
+ ti->error = "dm-stripe: Not enough destinations "
+ "specified";
+ return -EINVAL;
+ }
+
+ sc = alloc_context(stripes);
+ if (!sc) {
+ ti->error = "dm-stripe: Memory allocation for striped context "
+ "failed";
+ return -ENOMEM;
+ }
+
+ sc->stripes = stripes;
+ sc->stripe_width = width;
+ ti->split_io = chunk_size;
+
+ sc->chunk_mask = ((sector_t) chunk_size) - 1;
+ for (sc->chunk_shift = 0; chunk_size; sc->chunk_shift++)
+ chunk_size >>= 1;
+ sc->chunk_shift--;
+
+ /*
+ * Get the stripe destinations.
+ */
+ for (i = 0; i < stripes; i++) {
+ argv += 2;
+
+ r = get_stripe(ti, sc, i, argv);
+ if (r < 0) {
+ ti->error = "dm-stripe: Couldn't parse stripe "
+ "destination";
+ while (i--)
+ dm_put_device(ti, sc->stripe[i].dev);
+ kfree(sc);
+ return r;
+ }
+ }
+
+ ti->private = sc;
+ return 0;
+}
+
+static void stripe_dtr(struct dm_target *ti)
+{
+ unsigned int i;
+ struct stripe_c *sc = (struct stripe_c *) ti->private;
+
+ for (i = 0; i < sc->stripes; i++)
+ dm_put_device(ti, sc->stripe[i].dev);
+
+ kfree(sc);
+}
+
+static int stripe_map(struct dm_target *ti, struct bio *bio,
+ union map_info *map_context)
+{
+ struct stripe_c *sc = (struct stripe_c *) ti->private;
+
+ sector_t offset = bio->bi_sector - ti->begin;
+ sector_t chunk = offset >> sc->chunk_shift;
+ uint32_t stripe = sector_div(chunk, sc->stripes);
+
+ bio->bi_bdev = sc->stripe[stripe].dev->bdev;
+ bio->bi_sector = sc->stripe[stripe].physical_start +
+ (chunk << sc->chunk_shift) + (offset & sc->chunk_mask);
+ return 1;
+}
+
+static int stripe_status(struct dm_target *ti,
+ status_type_t type, char *result, unsigned int maxlen)
+{
+ struct stripe_c *sc = (struct stripe_c *) ti->private;
+ unsigned int sz = 0;
+ unsigned int i;
+
+ switch (type) {
+ case STATUSTYPE_INFO:
+ result[0] = '\0';
+ break;
+
+ case STATUSTYPE_TABLE:
+ DMEMIT("%d " SECTOR_FORMAT, sc->stripes, sc->chunk_mask + 1);
+ for (i = 0; i < sc->stripes; i++)
+ DMEMIT(" %s " SECTOR_FORMAT, sc->stripe[i].dev->name,
+ sc->stripe[i].physical_start);
+ break;
+ }
+ return 0;
+}
+
+static struct target_type stripe_target = {
+ .name = "striped",
+ .version= {1, 0, 2},
+ .module = THIS_MODULE,
+ .ctr = stripe_ctr,
+ .dtr = stripe_dtr,
+ .map = stripe_map,
+ .status = stripe_status,
+};
+
+int __init dm_stripe_init(void)
+{
+ int r;
+
+ r = dm_register_target(&stripe_target);
+ if (r < 0)
+ DMWARN("striped target registration failed");
+
+ return r;
+}
+
+void dm_stripe_exit(void)
+{
+ if (dm_unregister_target(&stripe_target))
+ DMWARN("striped target unregistration failed");
+
+ return;
+}
diff --git a/drivers/md/dm-table.c b/drivers/md/dm-table.c
new file mode 100644
index 000000000000..ee175d4906c4
--- /dev/null
+++ b/drivers/md/dm-table.c
@@ -0,0 +1,950 @@
+/*
+ * Copyright (C) 2001 Sistina Software (UK) Limited.
+ * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+
+#include <linux/module.h>
+#include <linux/vmalloc.h>
+#include <linux/blkdev.h>
+#include <linux/namei.h>
+#include <linux/ctype.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <asm/atomic.h>
+
+#define MAX_DEPTH 16
+#define NODE_SIZE L1_CACHE_BYTES
+#define KEYS_PER_NODE (NODE_SIZE / sizeof(sector_t))
+#define CHILDREN_PER_NODE (KEYS_PER_NODE + 1)
+
+struct dm_table {
+ atomic_t holders;
+
+ /* btree table */
+ unsigned int depth;
+ unsigned int counts[MAX_DEPTH]; /* in nodes */
+ sector_t *index[MAX_DEPTH];
+
+ unsigned int num_targets;
+ unsigned int num_allocated;
+ sector_t *highs;
+ struct dm_target *targets;
+
+ /*
+ * Indicates the rw permissions for the new logical
+ * device. This should be a combination of FMODE_READ
+ * and FMODE_WRITE.
+ */
+ int mode;
+
+ /* a list of devices used by this table */
+ struct list_head devices;
+
+ /*
+ * These are optimistic limits taken from all the
+ * targets, some targets will need smaller limits.
+ */
+ struct io_restrictions limits;
+
+ /* events get handed up using this callback */
+ void (*event_fn)(void *);
+ void *event_context;
+};
+
+/*
+ * Similar to ceiling(log_size(n))
+ */
+static unsigned int int_log(unsigned int n, unsigned int base)
+{
+ int result = 0;
+
+ while (n > 1) {
+ n = dm_div_up(n, base);
+ result++;
+ }
+
+ return result;
+}
+
+/*
+ * Returns the minimum that is _not_ zero, unless both are zero.
+ */
+#define min_not_zero(l, r) (l == 0) ? r : ((r == 0) ? l : min(l, r))
+
+/*
+ * Combine two io_restrictions, always taking the lower value.
+ */
+static void combine_restrictions_low(struct io_restrictions *lhs,
+ struct io_restrictions *rhs)
+{
+ lhs->max_sectors =
+ min_not_zero(lhs->max_sectors, rhs->max_sectors);
+
+ lhs->max_phys_segments =
+ min_not_zero(lhs->max_phys_segments, rhs->max_phys_segments);
+
+ lhs->max_hw_segments =
+ min_not_zero(lhs->max_hw_segments, rhs->max_hw_segments);
+
+ lhs->hardsect_size = max(lhs->hardsect_size, rhs->hardsect_size);
+
+ lhs->max_segment_size =
+ min_not_zero(lhs->max_segment_size, rhs->max_segment_size);
+
+ lhs->seg_boundary_mask =
+ min_not_zero(lhs->seg_boundary_mask, rhs->seg_boundary_mask);
+}
+
+/*
+ * Calculate the index of the child node of the n'th node k'th key.
+ */
+static inline unsigned int get_child(unsigned int n, unsigned int k)
+{
+ return (n * CHILDREN_PER_NODE) + k;
+}
+
+/*
+ * Return the n'th node of level l from table t.
+ */
+static inline sector_t *get_node(struct dm_table *t,
+ unsigned int l, unsigned int n)
+{
+ return t->index[l] + (n * KEYS_PER_NODE);
+}
+
+/*
+ * Return the highest key that you could lookup from the n'th
+ * node on level l of the btree.
+ */
+static sector_t high(struct dm_table *t, unsigned int l, unsigned int n)
+{
+ for (; l < t->depth - 1; l++)
+ n = get_child(n, CHILDREN_PER_NODE - 1);
+
+ if (n >= t->counts[l])
+ return (sector_t) - 1;
+
+ return get_node(t, l, n)[KEYS_PER_NODE - 1];
+}
+
+/*
+ * Fills in a level of the btree based on the highs of the level
+ * below it.
+ */
+static int setup_btree_index(unsigned int l, struct dm_table *t)
+{
+ unsigned int n, k;
+ sector_t *node;
+
+ for (n = 0U; n < t->counts[l]; n++) {
+ node = get_node(t, l, n);
+
+ for (k = 0U; k < KEYS_PER_NODE; k++)
+ node[k] = high(t, l + 1, get_child(n, k));
+ }
+
+ return 0;
+}
+
+void *dm_vcalloc(unsigned long nmemb, unsigned long elem_size)
+{
+ unsigned long size;
+ void *addr;
+
+ /*
+ * Check that we're not going to overflow.
+ */
+ if (nmemb > (ULONG_MAX / elem_size))
+ return NULL;
+
+ size = nmemb * elem_size;
+ addr = vmalloc(size);
+ if (addr)
+ memset(addr, 0, size);
+
+ return addr;
+}
+
+/*
+ * highs, and targets are managed as dynamic arrays during a
+ * table load.
+ */
+static int alloc_targets(struct dm_table *t, unsigned int num)
+{
+ sector_t *n_highs;
+ struct dm_target *n_targets;
+ int n = t->num_targets;
+
+ /*
+ * Allocate both the target array and offset array at once.
+ */
+ n_highs = (sector_t *) dm_vcalloc(num, sizeof(struct dm_target) +
+ sizeof(sector_t));
+ if (!n_highs)
+ return -ENOMEM;
+
+ n_targets = (struct dm_target *) (n_highs + num);
+
+ if (n) {
+ memcpy(n_highs, t->highs, sizeof(*n_highs) * n);
+ memcpy(n_targets, t->targets, sizeof(*n_targets) * n);
+ }
+
+ memset(n_highs + n, -1, sizeof(*n_highs) * (num - n));
+ vfree(t->highs);
+
+ t->num_allocated = num;
+ t->highs = n_highs;
+ t->targets = n_targets;
+
+ return 0;
+}
+
+int dm_table_create(struct dm_table **result, int mode, unsigned num_targets)
+{
+ struct dm_table *t = kmalloc(sizeof(*t), GFP_KERNEL);
+
+ if (!t)
+ return -ENOMEM;
+
+ memset(t, 0, sizeof(*t));
+ INIT_LIST_HEAD(&t->devices);
+ atomic_set(&t->holders, 1);
+
+ if (!num_targets)
+ num_targets = KEYS_PER_NODE;
+
+ num_targets = dm_round_up(num_targets, KEYS_PER_NODE);
+
+ if (alloc_targets(t, num_targets)) {
+ kfree(t);
+ t = NULL;
+ return -ENOMEM;
+ }
+
+ t->mode = mode;
+ *result = t;
+ return 0;
+}
+
+static void free_devices(struct list_head *devices)
+{
+ struct list_head *tmp, *next;
+
+ for (tmp = devices->next; tmp != devices; tmp = next) {
+ struct dm_dev *dd = list_entry(tmp, struct dm_dev, list);
+ next = tmp->next;
+ kfree(dd);
+ }
+}
+
+void table_destroy(struct dm_table *t)
+{
+ unsigned int i;
+
+ /* free the indexes (see dm_table_complete) */
+ if (t->depth >= 2)
+ vfree(t->index[t->depth - 2]);
+
+ /* free the targets */
+ for (i = 0; i < t->num_targets; i++) {
+ struct dm_target *tgt = t->targets + i;
+
+ if (tgt->type->dtr)
+ tgt->type->dtr(tgt);
+
+ dm_put_target_type(tgt->type);
+ }
+
+ vfree(t->highs);
+
+ /* free the device list */
+ if (t->devices.next != &t->devices) {
+ DMWARN("devices still present during destroy: "
+ "dm_table_remove_device calls missing");
+
+ free_devices(&t->devices);
+ }
+
+ kfree(t);
+}
+
+void dm_table_get(struct dm_table *t)
+{
+ atomic_inc(&t->holders);
+}
+
+void dm_table_put(struct dm_table *t)
+{
+ if (!t)
+ return;
+
+ if (atomic_dec_and_test(&t->holders))
+ table_destroy(t);
+}
+
+/*
+ * Checks to see if we need to extend highs or targets.
+ */
+static inline int check_space(struct dm_table *t)
+{
+ if (t->num_targets >= t->num_allocated)
+ return alloc_targets(t, t->num_allocated * 2);
+
+ return 0;
+}
+
+/*
+ * Convert a device path to a dev_t.
+ */
+static int lookup_device(const char *path, dev_t *dev)
+{
+ int r;
+ struct nameidata nd;
+ struct inode *inode;
+
+ if ((r = path_lookup(path, LOOKUP_FOLLOW, &nd)))
+ return r;
+
+ inode = nd.dentry->d_inode;
+ if (!inode) {
+ r = -ENOENT;
+ goto out;
+ }
+
+ if (!S_ISBLK(inode->i_mode)) {
+ r = -ENOTBLK;
+ goto out;
+ }
+
+ *dev = inode->i_rdev;
+
+ out:
+ path_release(&nd);
+ return r;
+}
+
+/*
+ * See if we've already got a device in the list.
+ */
+static struct dm_dev *find_device(struct list_head *l, dev_t dev)
+{
+ struct dm_dev *dd;
+
+ list_for_each_entry (dd, l, list)
+ if (dd->bdev->bd_dev == dev)
+ return dd;
+
+ return NULL;
+}
+
+/*
+ * Open a device so we can use it as a map destination.
+ */
+static int open_dev(struct dm_dev *d, dev_t dev)
+{
+ static char *_claim_ptr = "I belong to device-mapper";
+ struct block_device *bdev;
+
+ int r;
+
+ if (d->bdev)
+ BUG();
+
+ bdev = open_by_devnum(dev, d->mode);
+ if (IS_ERR(bdev))
+ return PTR_ERR(bdev);
+ r = bd_claim(bdev, _claim_ptr);
+ if (r)
+ blkdev_put(bdev);
+ else
+ d->bdev = bdev;
+ return r;
+}
+
+/*
+ * Close a device that we've been using.
+ */
+static void close_dev(struct dm_dev *d)
+{
+ if (!d->bdev)
+ return;
+
+ bd_release(d->bdev);
+ blkdev_put(d->bdev);
+ d->bdev = NULL;
+}
+
+/*
+ * If possible (ie. blk_size[major] is set), this checks an area
+ * of a destination device is valid.
+ */
+static int check_device_area(struct dm_dev *dd, sector_t start, sector_t len)
+{
+ sector_t dev_size;
+ dev_size = dd->bdev->bd_inode->i_size >> SECTOR_SHIFT;
+ return ((start < dev_size) && (len <= (dev_size - start)));
+}
+
+/*
+ * This upgrades the mode on an already open dm_dev. Being
+ * careful to leave things as they were if we fail to reopen the
+ * device.
+ */
+static int upgrade_mode(struct dm_dev *dd, int new_mode)
+{
+ int r;
+ struct dm_dev dd_copy;
+ dev_t dev = dd->bdev->bd_dev;
+
+ dd_copy = *dd;
+
+ dd->mode |= new_mode;
+ dd->bdev = NULL;
+ r = open_dev(dd, dev);
+ if (!r)
+ close_dev(&dd_copy);
+ else
+ *dd = dd_copy;
+
+ return r;
+}
+
+/*
+ * Add a device to the list, or just increment the usage count if
+ * it's already present.
+ */
+static int __table_get_device(struct dm_table *t, struct dm_target *ti,
+ const char *path, sector_t start, sector_t len,
+ int mode, struct dm_dev **result)
+{
+ int r;
+ dev_t dev;
+ struct dm_dev *dd;
+ unsigned int major, minor;
+
+ if (!t)
+ BUG();
+
+ if (sscanf(path, "%u:%u", &major, &minor) == 2) {
+ /* Extract the major/minor numbers */
+ dev = MKDEV(major, minor);
+ if (MAJOR(dev) != major || MINOR(dev) != minor)
+ return -EOVERFLOW;
+ } else {
+ /* convert the path to a device */
+ if ((r = lookup_device(path, &dev)))
+ return r;
+ }
+
+ dd = find_device(&t->devices, dev);
+ if (!dd) {
+ dd = kmalloc(sizeof(*dd), GFP_KERNEL);
+ if (!dd)
+ return -ENOMEM;
+
+ dd->mode = mode;
+ dd->bdev = NULL;
+
+ if ((r = open_dev(dd, dev))) {
+ kfree(dd);
+ return r;
+ }
+
+ format_dev_t(dd->name, dev);
+
+ atomic_set(&dd->count, 0);
+ list_add(&dd->list, &t->devices);
+
+ } else if (dd->mode != (mode | dd->mode)) {
+ r = upgrade_mode(dd, mode);
+ if (r)
+ return r;
+ }
+ atomic_inc(&dd->count);
+
+ if (!check_device_area(dd, start, len)) {
+ DMWARN("device %s too small for target", path);
+ dm_put_device(ti, dd);
+ return -EINVAL;
+ }
+
+ *result = dd;
+
+ return 0;
+}
+
+
+int dm_get_device(struct dm_target *ti, const char *path, sector_t start,
+ sector_t len, int mode, struct dm_dev **result)
+{
+ int r = __table_get_device(ti->table, ti, path,
+ start, len, mode, result);
+ if (!r) {
+ request_queue_t *q = bdev_get_queue((*result)->bdev);
+ struct io_restrictions *rs = &ti->limits;
+
+ /*
+ * Combine the device limits low.
+ *
+ * FIXME: if we move an io_restriction struct
+ * into q this would just be a call to
+ * combine_restrictions_low()
+ */
+ rs->max_sectors =
+ min_not_zero(rs->max_sectors, q->max_sectors);
+
+ /* FIXME: Device-Mapper on top of RAID-0 breaks because DM
+ * currently doesn't honor MD's merge_bvec_fn routine.
+ * In this case, we'll force DM to use PAGE_SIZE or
+ * smaller I/O, just to be safe. A better fix is in the
+ * works, but add this for the time being so it will at
+ * least operate correctly.
+ */
+ if (q->merge_bvec_fn)
+ rs->max_sectors =
+ min_not_zero(rs->max_sectors,
+ (unsigned short)(PAGE_SIZE >> 9));
+
+ rs->max_phys_segments =
+ min_not_zero(rs->max_phys_segments,
+ q->max_phys_segments);
+
+ rs->max_hw_segments =
+ min_not_zero(rs->max_hw_segments, q->max_hw_segments);
+
+ rs->hardsect_size = max(rs->hardsect_size, q->hardsect_size);
+
+ rs->max_segment_size =
+ min_not_zero(rs->max_segment_size, q->max_segment_size);
+
+ rs->seg_boundary_mask =
+ min_not_zero(rs->seg_boundary_mask,
+ q->seg_boundary_mask);
+ }
+
+ return r;
+}
+
+/*
+ * Decrement a devices use count and remove it if necessary.
+ */
+void dm_put_device(struct dm_target *ti, struct dm_dev *dd)
+{
+ if (atomic_dec_and_test(&dd->count)) {
+ close_dev(dd);
+ list_del(&dd->list);
+ kfree(dd);
+ }
+}
+
+/*
+ * Checks to see if the target joins onto the end of the table.
+ */
+static int adjoin(struct dm_table *table, struct dm_target *ti)
+{
+ struct dm_target *prev;
+
+ if (!table->num_targets)
+ return !ti->begin;
+
+ prev = &table->targets[table->num_targets - 1];
+ return (ti->begin == (prev->begin + prev->len));
+}
+
+/*
+ * Used to dynamically allocate the arg array.
+ */
+static char **realloc_argv(unsigned *array_size, char **old_argv)
+{
+ char **argv;
+ unsigned new_size;
+
+ new_size = *array_size ? *array_size * 2 : 64;
+ argv = kmalloc(new_size * sizeof(*argv), GFP_KERNEL);
+ if (argv) {
+ memcpy(argv, old_argv, *array_size * sizeof(*argv));
+ *array_size = new_size;
+ }
+
+ kfree(old_argv);
+ return argv;
+}
+
+/*
+ * Destructively splits up the argument list to pass to ctr.
+ */
+int dm_split_args(int *argc, char ***argvp, char *input)
+{
+ char *start, *end = input, *out, **argv = NULL;
+ unsigned array_size = 0;
+
+ *argc = 0;
+ argv = realloc_argv(&array_size, argv);
+ if (!argv)
+ return -ENOMEM;
+
+ while (1) {
+ start = end;
+
+ /* Skip whitespace */
+ while (*start && isspace(*start))
+ start++;
+
+ if (!*start)
+ break; /* success, we hit the end */
+
+ /* 'out' is used to remove any back-quotes */
+ end = out = start;
+ while (*end) {
+ /* Everything apart from '\0' can be quoted */
+ if (*end == '\\' && *(end + 1)) {
+ *out++ = *(end + 1);
+ end += 2;
+ continue;
+ }
+
+ if (isspace(*end))
+ break; /* end of token */
+
+ *out++ = *end++;
+ }
+
+ /* have we already filled the array ? */
+ if ((*argc + 1) > array_size) {
+ argv = realloc_argv(&array_size, argv);
+ if (!argv)
+ return -ENOMEM;
+ }
+
+ /* we know this is whitespace */
+ if (*end)
+ end++;
+
+ /* terminate the string and put it in the array */
+ *out = '\0';
+ argv[*argc] = start;
+ (*argc)++;
+ }
+
+ *argvp = argv;
+ return 0;
+}
+
+static void check_for_valid_limits(struct io_restrictions *rs)
+{
+ if (!rs->max_sectors)
+ rs->max_sectors = MAX_SECTORS;
+ if (!rs->max_phys_segments)
+ rs->max_phys_segments = MAX_PHYS_SEGMENTS;
+ if (!rs->max_hw_segments)
+ rs->max_hw_segments = MAX_HW_SEGMENTS;
+ if (!rs->hardsect_size)
+ rs->hardsect_size = 1 << SECTOR_SHIFT;
+ if (!rs->max_segment_size)
+ rs->max_segment_size = MAX_SEGMENT_SIZE;
+ if (!rs->seg_boundary_mask)
+ rs->seg_boundary_mask = -1;
+}
+
+int dm_table_add_target(struct dm_table *t, const char *type,
+ sector_t start, sector_t len, char *params)
+{
+ int r = -EINVAL, argc;
+ char **argv;
+ struct dm_target *tgt;
+
+ if ((r = check_space(t)))
+ return r;
+
+ tgt = t->targets + t->num_targets;
+ memset(tgt, 0, sizeof(*tgt));
+
+ if (!len) {
+ tgt->error = "zero-length target";
+ DMERR("%s", tgt->error);
+ return -EINVAL;
+ }
+
+ tgt->type = dm_get_target_type(type);
+ if (!tgt->type) {
+ tgt->error = "unknown target type";
+ DMERR("%s", tgt->error);
+ return -EINVAL;
+ }
+
+ tgt->table = t;
+ tgt->begin = start;
+ tgt->len = len;
+ tgt->error = "Unknown error";
+
+ /*
+ * Does this target adjoin the previous one ?
+ */
+ if (!adjoin(t, tgt)) {
+ tgt->error = "Gap in table";
+ r = -EINVAL;
+ goto bad;
+ }
+
+ r = dm_split_args(&argc, &argv, params);
+ if (r) {
+ tgt->error = "couldn't split parameters (insufficient memory)";
+ goto bad;
+ }
+
+ r = tgt->type->ctr(tgt, argc, argv);
+ kfree(argv);
+ if (r)
+ goto bad;
+
+ t->highs[t->num_targets++] = tgt->begin + tgt->len - 1;
+
+ /* FIXME: the plan is to combine high here and then have
+ * the merge fn apply the target level restrictions. */
+ combine_restrictions_low(&t->limits, &tgt->limits);
+ return 0;
+
+ bad:
+ DMERR("%s", tgt->error);
+ dm_put_target_type(tgt->type);
+ return r;
+}
+
+static int setup_indexes(struct dm_table *t)
+{
+ int i;
+ unsigned int total = 0;
+ sector_t *indexes;
+
+ /* allocate the space for *all* the indexes */
+ for (i = t->depth - 2; i >= 0; i--) {
+ t->counts[i] = dm_div_up(t->counts[i + 1], CHILDREN_PER_NODE);
+ total += t->counts[i];
+ }
+
+ indexes = (sector_t *) dm_vcalloc(total, (unsigned long) NODE_SIZE);
+ if (!indexes)
+ return -ENOMEM;
+
+ /* set up internal nodes, bottom-up */
+ for (i = t->depth - 2, total = 0; i >= 0; i--) {
+ t->index[i] = indexes;
+ indexes += (KEYS_PER_NODE * t->counts[i]);
+ setup_btree_index(i, t);
+ }
+
+ return 0;
+}
+
+/*
+ * Builds the btree to index the map.
+ */
+int dm_table_complete(struct dm_table *t)
+{
+ int r = 0;
+ unsigned int leaf_nodes;
+
+ check_for_valid_limits(&t->limits);
+
+ /* how many indexes will the btree have ? */
+ leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE);
+ t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE);
+
+ /* leaf layer has already been set up */
+ t->counts[t->depth - 1] = leaf_nodes;
+ t->index[t->depth - 1] = t->highs;
+
+ if (t->depth >= 2)
+ r = setup_indexes(t);
+
+ return r;
+}
+
+static DECLARE_MUTEX(_event_lock);
+void dm_table_event_callback(struct dm_table *t,
+ void (*fn)(void *), void *context)
+{
+ down(&_event_lock);
+ t->event_fn = fn;
+ t->event_context = context;
+ up(&_event_lock);
+}
+
+void dm_table_event(struct dm_table *t)
+{
+ /*
+ * You can no longer call dm_table_event() from interrupt
+ * context, use a bottom half instead.
+ */
+ BUG_ON(in_interrupt());
+
+ down(&_event_lock);
+ if (t->event_fn)
+ t->event_fn(t->event_context);
+ up(&_event_lock);
+}
+
+sector_t dm_table_get_size(struct dm_table *t)
+{
+ return t->num_targets ? (t->highs[t->num_targets - 1] + 1) : 0;
+}
+
+struct dm_target *dm_table_get_target(struct dm_table *t, unsigned int index)
+{
+ if (index > t->num_targets)
+ return NULL;
+
+ return t->targets + index;
+}
+
+/*
+ * Search the btree for the correct target.
+ */
+struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector)
+{
+ unsigned int l, n = 0, k = 0;
+ sector_t *node;
+
+ for (l = 0; l < t->depth; l++) {
+ n = get_child(n, k);
+ node = get_node(t, l, n);
+
+ for (k = 0; k < KEYS_PER_NODE; k++)
+ if (node[k] >= sector)
+ break;
+ }
+
+ return &t->targets[(KEYS_PER_NODE * n) + k];
+}
+
+void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q)
+{
+ /*
+ * Make sure we obey the optimistic sub devices
+ * restrictions.
+ */
+ blk_queue_max_sectors(q, t->limits.max_sectors);
+ q->max_phys_segments = t->limits.max_phys_segments;
+ q->max_hw_segments = t->limits.max_hw_segments;
+ q->hardsect_size = t->limits.hardsect_size;
+ q->max_segment_size = t->limits.max_segment_size;
+ q->seg_boundary_mask = t->limits.seg_boundary_mask;
+}
+
+unsigned int dm_table_get_num_targets(struct dm_table *t)
+{
+ return t->num_targets;
+}
+
+struct list_head *dm_table_get_devices(struct dm_table *t)
+{
+ return &t->devices;
+}
+
+int dm_table_get_mode(struct dm_table *t)
+{
+ return t->mode;
+}
+
+static void suspend_targets(struct dm_table *t, unsigned postsuspend)
+{
+ int i = t->num_targets;
+ struct dm_target *ti = t->targets;
+
+ while (i--) {
+ if (postsuspend) {
+ if (ti->type->postsuspend)
+ ti->type->postsuspend(ti);
+ } else if (ti->type->presuspend)
+ ti->type->presuspend(ti);
+
+ ti++;
+ }
+}
+
+void dm_table_presuspend_targets(struct dm_table *t)
+{
+ return suspend_targets(t, 0);
+}
+
+void dm_table_postsuspend_targets(struct dm_table *t)
+{
+ return suspend_targets(t, 1);
+}
+
+void dm_table_resume_targets(struct dm_table *t)
+{
+ int i;
+
+ for (i = 0; i < t->num_targets; i++) {
+ struct dm_target *ti = t->targets + i;
+
+ if (ti->type->resume)
+ ti->type->resume(ti);
+ }
+}
+
+int dm_table_any_congested(struct dm_table *t, int bdi_bits)
+{
+ struct list_head *d, *devices;
+ int r = 0;
+
+ devices = dm_table_get_devices(t);
+ for (d = devices->next; d != devices; d = d->next) {
+ struct dm_dev *dd = list_entry(d, struct dm_dev, list);
+ request_queue_t *q = bdev_get_queue(dd->bdev);
+ r |= bdi_congested(&q->backing_dev_info, bdi_bits);
+ }
+
+ return r;
+}
+
+void dm_table_unplug_all(struct dm_table *t)
+{
+ struct list_head *d, *devices = dm_table_get_devices(t);
+
+ for (d = devices->next; d != devices; d = d->next) {
+ struct dm_dev *dd = list_entry(d, struct dm_dev, list);
+ request_queue_t *q = bdev_get_queue(dd->bdev);
+
+ if (q->unplug_fn)
+ q->unplug_fn(q);
+ }
+}
+
+int dm_table_flush_all(struct dm_table *t)
+{
+ struct list_head *d, *devices = dm_table_get_devices(t);
+ int ret = 0;
+
+ for (d = devices->next; d != devices; d = d->next) {
+ struct dm_dev *dd = list_entry(d, struct dm_dev, list);
+ request_queue_t *q = bdev_get_queue(dd->bdev);
+ int err;
+
+ if (!q->issue_flush_fn)
+ err = -EOPNOTSUPP;
+ else
+ err = q->issue_flush_fn(q, dd->bdev->bd_disk, NULL);
+
+ if (!ret)
+ ret = err;
+ }
+
+ return ret;
+}
+
+EXPORT_SYMBOL(dm_vcalloc);
+EXPORT_SYMBOL(dm_get_device);
+EXPORT_SYMBOL(dm_put_device);
+EXPORT_SYMBOL(dm_table_event);
+EXPORT_SYMBOL(dm_table_get_mode);
+EXPORT_SYMBOL(dm_table_put);
+EXPORT_SYMBOL(dm_table_get);
+EXPORT_SYMBOL(dm_table_unplug_all);
+EXPORT_SYMBOL(dm_table_flush_all);
diff --git a/drivers/md/dm-target.c b/drivers/md/dm-target.c
new file mode 100644
index 000000000000..aecd9e0c2616
--- /dev/null
+++ b/drivers/md/dm-target.c
@@ -0,0 +1,196 @@
+/*
+ * Copyright (C) 2001 Sistina Software (UK) Limited
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/kmod.h>
+#include <linux/bio.h>
+#include <linux/slab.h>
+
+struct tt_internal {
+ struct target_type tt;
+
+ struct list_head list;
+ long use;
+};
+
+static LIST_HEAD(_targets);
+static DECLARE_RWSEM(_lock);
+
+#define DM_MOD_NAME_SIZE 32
+
+static inline struct tt_internal *__find_target_type(const char *name)
+{
+ struct tt_internal *ti;
+
+ list_for_each_entry (ti, &_targets, list)
+ if (!strcmp(name, ti->tt.name))
+ return ti;
+
+ return NULL;
+}
+
+static struct tt_internal *get_target_type(const char *name)
+{
+ struct tt_internal *ti;
+
+ down_read(&_lock);
+
+ ti = __find_target_type(name);
+ if (ti) {
+ if ((ti->use == 0) && !try_module_get(ti->tt.module))
+ ti = NULL;
+ else
+ ti->use++;
+ }
+
+ up_read(&_lock);
+ return ti;
+}
+
+static void load_module(const char *name)
+{
+ request_module("dm-%s", name);
+}
+
+struct target_type *dm_get_target_type(const char *name)
+{
+ struct tt_internal *ti = get_target_type(name);
+
+ if (!ti) {
+ load_module(name);
+ ti = get_target_type(name);
+ }
+
+ return ti ? &ti->tt : NULL;
+}
+
+void dm_put_target_type(struct target_type *t)
+{
+ struct tt_internal *ti = (struct tt_internal *) t;
+
+ down_read(&_lock);
+ if (--ti->use == 0)
+ module_put(ti->tt.module);
+
+ if (ti->use < 0)
+ BUG();
+ up_read(&_lock);
+
+ return;
+}
+
+static struct tt_internal *alloc_target(struct target_type *t)
+{
+ struct tt_internal *ti = kmalloc(sizeof(*ti), GFP_KERNEL);
+
+ if (ti) {
+ memset(ti, 0, sizeof(*ti));
+ ti->tt = *t;
+ }
+
+ return ti;
+}
+
+
+int dm_target_iterate(void (*iter_func)(struct target_type *tt,
+ void *param), void *param)
+{
+ struct tt_internal *ti;
+
+ down_read(&_lock);
+ list_for_each_entry (ti, &_targets, list)
+ iter_func(&ti->tt, param);
+ up_read(&_lock);
+
+ return 0;
+}
+
+int dm_register_target(struct target_type *t)
+{
+ int rv = 0;
+ struct tt_internal *ti = alloc_target(t);
+
+ if (!ti)
+ return -ENOMEM;
+
+ down_write(&_lock);
+ if (__find_target_type(t->name))
+ rv = -EEXIST;
+ else
+ list_add(&ti->list, &_targets);
+
+ up_write(&_lock);
+ if (rv)
+ kfree(ti);
+ return rv;
+}
+
+int dm_unregister_target(struct target_type *t)
+{
+ struct tt_internal *ti;
+
+ down_write(&_lock);
+ if (!(ti = __find_target_type(t->name))) {
+ up_write(&_lock);
+ return -EINVAL;
+ }
+
+ if (ti->use) {
+ up_write(&_lock);
+ return -ETXTBSY;
+ }
+
+ list_del(&ti->list);
+ kfree(ti);
+
+ up_write(&_lock);
+ return 0;
+}
+
+/*
+ * io-err: always fails an io, useful for bringing
+ * up LVs that have holes in them.
+ */
+static int io_err_ctr(struct dm_target *ti, unsigned int argc, char **args)
+{
+ return 0;
+}
+
+static void io_err_dtr(struct dm_target *ti)
+{
+ /* empty */
+}
+
+static int io_err_map(struct dm_target *ti, struct bio *bio,
+ union map_info *map_context)
+{
+ return -EIO;
+}
+
+static struct target_type error_target = {
+ .name = "error",
+ .version = {1, 0, 1},
+ .ctr = io_err_ctr,
+ .dtr = io_err_dtr,
+ .map = io_err_map,
+};
+
+int __init dm_target_init(void)
+{
+ return dm_register_target(&error_target);
+}
+
+void dm_target_exit(void)
+{
+ if (dm_unregister_target(&error_target))
+ DMWARN("error target unregistration failed");
+}
+
+EXPORT_SYMBOL(dm_register_target);
+EXPORT_SYMBOL(dm_unregister_target);
diff --git a/drivers/md/dm-zero.c b/drivers/md/dm-zero.c
new file mode 100644
index 000000000000..7febc2cac73d
--- /dev/null
+++ b/drivers/md/dm-zero.c
@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) 2003 Christophe Saout <christophe@saout.de>
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/bio.h>
+
+/*
+ * Construct a dummy mapping that only returns zeros
+ */
+static int zero_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+ if (argc != 0) {
+ ti->error = "dm-zero: No arguments required";
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/*
+ * Return zeros only on reads
+ */
+static int zero_map(struct dm_target *ti, struct bio *bio,
+ union map_info *map_context)
+{
+ switch(bio_rw(bio)) {
+ case READ:
+ zero_fill_bio(bio);
+ break;
+ case READA:
+ /* readahead of null bytes only wastes buffer cache */
+ return -EIO;
+ case WRITE:
+ /* writes get silently dropped */
+ break;
+ }
+
+ bio_endio(bio, bio->bi_size, 0);
+
+ /* accepted bio, don't make new request */
+ return 0;
+}
+
+static struct target_type zero_target = {
+ .name = "zero",
+ .version = {1, 0, 0},
+ .module = THIS_MODULE,
+ .ctr = zero_ctr,
+ .map = zero_map,
+};
+
+int __init dm_zero_init(void)
+{
+ int r = dm_register_target(&zero_target);
+
+ if (r < 0)
+ DMERR("zero: register failed %d", r);
+
+ return r;
+}
+
+void __exit dm_zero_exit(void)
+{
+ int r = dm_unregister_target(&zero_target);
+
+ if (r < 0)
+ DMERR("zero: unregister failed %d", r);
+}
+
+module_init(dm_zero_init)
+module_exit(dm_zero_exit)
+
+MODULE_AUTHOR("Christophe Saout <christophe@saout.de>");
+MODULE_DESCRIPTION(DM_NAME " dummy target returning zeros");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
new file mode 100644
index 000000000000..243ff6884e83
--- /dev/null
+++ b/drivers/md/dm.c
@@ -0,0 +1,1194 @@
+/*
+ * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
+ * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the GPL.
+ */
+
+#include "dm.h"
+#include "dm-bio-list.h"
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/blkpg.h>
+#include <linux/bio.h>
+#include <linux/buffer_head.h>
+#include <linux/mempool.h>
+#include <linux/slab.h>
+#include <linux/idr.h>
+
+static const char *_name = DM_NAME;
+
+static unsigned int major = 0;
+static unsigned int _major = 0;
+
+/*
+ * One of these is allocated per bio.
+ */
+struct dm_io {
+ struct mapped_device *md;
+ int error;
+ struct bio *bio;
+ atomic_t io_count;
+};
+
+/*
+ * One of these is allocated per target within a bio. Hopefully
+ * this will be simplified out one day.
+ */
+struct target_io {
+ struct dm_io *io;
+ struct dm_target *ti;
+ union map_info info;
+};
+
+union map_info *dm_get_mapinfo(struct bio *bio)
+{
+ if (bio && bio->bi_private)
+ return &((struct target_io *)bio->bi_private)->info;
+ return NULL;
+}
+
+/*
+ * Bits for the md->flags field.
+ */
+#define DMF_BLOCK_IO 0
+#define DMF_SUSPENDED 1
+#define DMF_FS_LOCKED 2
+
+struct mapped_device {
+ struct rw_semaphore lock;
+ rwlock_t map_lock;
+ atomic_t holders;
+
+ unsigned long flags;
+
+ request_queue_t *queue;
+ struct gendisk *disk;
+
+ void *interface_ptr;
+
+ /*
+ * A list of ios that arrived while we were suspended.
+ */
+ atomic_t pending;
+ wait_queue_head_t wait;
+ struct bio_list deferred;
+
+ /*
+ * The current mapping.
+ */
+ struct dm_table *map;
+
+ /*
+ * io objects are allocated from here.
+ */
+ mempool_t *io_pool;
+ mempool_t *tio_pool;
+
+ /*
+ * Event handling.
+ */
+ atomic_t event_nr;
+ wait_queue_head_t eventq;
+
+ /*
+ * freeze/thaw support require holding onto a super block
+ */
+ struct super_block *frozen_sb;
+};
+
+#define MIN_IOS 256
+static kmem_cache_t *_io_cache;
+static kmem_cache_t *_tio_cache;
+
+static struct bio_set *dm_set;
+
+static int __init local_init(void)
+{
+ int r;
+
+ dm_set = bioset_create(16, 16, 4);
+ if (!dm_set)
+ return -ENOMEM;
+
+ /* allocate a slab for the dm_ios */
+ _io_cache = kmem_cache_create("dm_io",
+ sizeof(struct dm_io), 0, 0, NULL, NULL);
+ if (!_io_cache)
+ return -ENOMEM;
+
+ /* allocate a slab for the target ios */
+ _tio_cache = kmem_cache_create("dm_tio", sizeof(struct target_io),
+ 0, 0, NULL, NULL);
+ if (!_tio_cache) {
+ kmem_cache_destroy(_io_cache);
+ return -ENOMEM;
+ }
+
+ _major = major;
+ r = register_blkdev(_major, _name);
+ if (r < 0) {
+ kmem_cache_destroy(_tio_cache);
+ kmem_cache_destroy(_io_cache);
+ return r;
+ }
+
+ if (!_major)
+ _major = r;
+
+ return 0;
+}
+
+static void local_exit(void)
+{
+ kmem_cache_destroy(_tio_cache);
+ kmem_cache_destroy(_io_cache);
+
+ bioset_free(dm_set);
+
+ if (unregister_blkdev(_major, _name) < 0)
+ DMERR("devfs_unregister_blkdev failed");
+
+ _major = 0;
+
+ DMINFO("cleaned up");
+}
+
+int (*_inits[])(void) __initdata = {
+ local_init,
+ dm_target_init,
+ dm_linear_init,
+ dm_stripe_init,
+ dm_interface_init,
+};
+
+void (*_exits[])(void) = {
+ local_exit,
+ dm_target_exit,
+ dm_linear_exit,
+ dm_stripe_exit,
+ dm_interface_exit,
+};
+
+static int __init dm_init(void)
+{
+ const int count = ARRAY_SIZE(_inits);
+
+ int r, i;
+
+ for (i = 0; i < count; i++) {
+ r = _inits[i]();
+ if (r)
+ goto bad;
+ }
+
+ return 0;
+
+ bad:
+ while (i--)
+ _exits[i]();
+
+ return r;
+}
+
+static void __exit dm_exit(void)
+{
+ int i = ARRAY_SIZE(_exits);
+
+ while (i--)
+ _exits[i]();
+}
+
+/*
+ * Block device functions
+ */
+static int dm_blk_open(struct inode *inode, struct file *file)
+{
+ struct mapped_device *md;
+
+ md = inode->i_bdev->bd_disk->private_data;
+ dm_get(md);
+ return 0;
+}
+
+static int dm_blk_close(struct inode *inode, struct file *file)
+{
+ struct mapped_device *md;
+
+ md = inode->i_bdev->bd_disk->private_data;
+ dm_put(md);
+ return 0;
+}
+
+static inline struct dm_io *alloc_io(struct mapped_device *md)
+{
+ return mempool_alloc(md->io_pool, GFP_NOIO);
+}
+
+static inline void free_io(struct mapped_device *md, struct dm_io *io)
+{
+ mempool_free(io, md->io_pool);
+}
+
+static inline struct target_io *alloc_tio(struct mapped_device *md)
+{
+ return mempool_alloc(md->tio_pool, GFP_NOIO);
+}
+
+static inline void free_tio(struct mapped_device *md, struct target_io *tio)
+{
+ mempool_free(tio, md->tio_pool);
+}
+
+/*
+ * Add the bio to the list of deferred io.
+ */
+static int queue_io(struct mapped_device *md, struct bio *bio)
+{
+ down_write(&md->lock);
+
+ if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
+ up_write(&md->lock);
+ return 1;
+ }
+
+ bio_list_add(&md->deferred, bio);
+
+ up_write(&md->lock);
+ return 0; /* deferred successfully */
+}
+
+/*
+ * Everyone (including functions in this file), should use this
+ * function to access the md->map field, and make sure they call
+ * dm_table_put() when finished.
+ */
+struct dm_table *dm_get_table(struct mapped_device *md)
+{
+ struct dm_table *t;
+
+ read_lock(&md->map_lock);
+ t = md->map;
+ if (t)
+ dm_table_get(t);
+ read_unlock(&md->map_lock);
+
+ return t;
+}
+
+/*-----------------------------------------------------------------
+ * CRUD START:
+ * A more elegant soln is in the works that uses the queue
+ * merge fn, unfortunately there are a couple of changes to
+ * the block layer that I want to make for this. So in the
+ * interests of getting something for people to use I give
+ * you this clearly demarcated crap.
+ *---------------------------------------------------------------*/
+
+/*
+ * Decrements the number of outstanding ios that a bio has been
+ * cloned into, completing the original io if necc.
+ */
+static inline void dec_pending(struct dm_io *io, int error)
+{
+ if (error)
+ io->error = error;
+
+ if (atomic_dec_and_test(&io->io_count)) {
+ if (atomic_dec_and_test(&io->md->pending))
+ /* nudge anyone waiting on suspend queue */
+ wake_up(&io->md->wait);
+
+ bio_endio(io->bio, io->bio->bi_size, io->error);
+ free_io(io->md, io);
+ }
+}
+
+static int clone_endio(struct bio *bio, unsigned int done, int error)
+{
+ int r = 0;
+ struct target_io *tio = bio->bi_private;
+ struct dm_io *io = tio->io;
+ dm_endio_fn endio = tio->ti->type->end_io;
+
+ if (bio->bi_size)
+ return 1;
+
+ if (!bio_flagged(bio, BIO_UPTODATE) && !error)
+ error = -EIO;
+
+ if (endio) {
+ r = endio(tio->ti, bio, error, &tio->info);
+ if (r < 0)
+ error = r;
+
+ else if (r > 0)
+ /* the target wants another shot at the io */
+ return 1;
+ }
+
+ free_tio(io->md, tio);
+ dec_pending(io, error);
+ bio_put(bio);
+ return r;
+}
+
+static sector_t max_io_len(struct mapped_device *md,
+ sector_t sector, struct dm_target *ti)
+{
+ sector_t offset = sector - ti->begin;
+ sector_t len = ti->len - offset;
+
+ /*
+ * Does the target need to split even further ?
+ */
+ if (ti->split_io) {
+ sector_t boundary;
+ boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
+ - offset;
+ if (len > boundary)
+ len = boundary;
+ }
+
+ return len;
+}
+
+static void __map_bio(struct dm_target *ti, struct bio *clone,
+ struct target_io *tio)
+{
+ int r;
+
+ /*
+ * Sanity checks.
+ */
+ BUG_ON(!clone->bi_size);
+
+ clone->bi_end_io = clone_endio;
+ clone->bi_private = tio;
+
+ /*
+ * Map the clone. If r == 0 we don't need to do
+ * anything, the target has assumed ownership of
+ * this io.
+ */
+ atomic_inc(&tio->io->io_count);
+ r = ti->type->map(ti, clone, &tio->info);
+ if (r > 0)
+ /* the bio has been remapped so dispatch it */
+ generic_make_request(clone);
+
+ else if (r < 0) {
+ /* error the io and bail out */
+ struct dm_io *io = tio->io;
+ free_tio(tio->io->md, tio);
+ dec_pending(io, -EIO);
+ bio_put(clone);
+ }
+}
+
+struct clone_info {
+ struct mapped_device *md;
+ struct dm_table *map;
+ struct bio *bio;
+ struct dm_io *io;
+ sector_t sector;
+ sector_t sector_count;
+ unsigned short idx;
+};
+
+/*
+ * Creates a little bio that is just does part of a bvec.
+ */
+static struct bio *split_bvec(struct bio *bio, sector_t sector,
+ unsigned short idx, unsigned int offset,
+ unsigned int len)
+{
+ struct bio *clone;
+ struct bio_vec *bv = bio->bi_io_vec + idx;
+
+ clone = bio_alloc_bioset(GFP_NOIO, 1, dm_set);
+ *clone->bi_io_vec = *bv;
+
+ clone->bi_sector = sector;
+ clone->bi_bdev = bio->bi_bdev;
+ clone->bi_rw = bio->bi_rw;
+ clone->bi_vcnt = 1;
+ clone->bi_size = to_bytes(len);
+ clone->bi_io_vec->bv_offset = offset;
+ clone->bi_io_vec->bv_len = clone->bi_size;
+
+ return clone;
+}
+
+/*
+ * Creates a bio that consists of range of complete bvecs.
+ */
+static struct bio *clone_bio(struct bio *bio, sector_t sector,
+ unsigned short idx, unsigned short bv_count,
+ unsigned int len)
+{
+ struct bio *clone;
+
+ clone = bio_clone(bio, GFP_NOIO);
+ clone->bi_sector = sector;
+ clone->bi_idx = idx;
+ clone->bi_vcnt = idx + bv_count;
+ clone->bi_size = to_bytes(len);
+ clone->bi_flags &= ~(1 << BIO_SEG_VALID);
+
+ return clone;
+}
+
+static void __clone_and_map(struct clone_info *ci)
+{
+ struct bio *clone, *bio = ci->bio;
+ struct dm_target *ti = dm_table_find_target(ci->map, ci->sector);
+ sector_t len = 0, max = max_io_len(ci->md, ci->sector, ti);
+ struct target_io *tio;
+
+ /*
+ * Allocate a target io object.
+ */
+ tio = alloc_tio(ci->md);
+ tio->io = ci->io;
+ tio->ti = ti;
+ memset(&tio->info, 0, sizeof(tio->info));
+
+ if (ci->sector_count <= max) {
+ /*
+ * Optimise for the simple case where we can do all of
+ * the remaining io with a single clone.
+ */
+ clone = clone_bio(bio, ci->sector, ci->idx,
+ bio->bi_vcnt - ci->idx, ci->sector_count);
+ __map_bio(ti, clone, tio);
+ ci->sector_count = 0;
+
+ } else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
+ /*
+ * There are some bvecs that don't span targets.
+ * Do as many of these as possible.
+ */
+ int i;
+ sector_t remaining = max;
+ sector_t bv_len;
+
+ for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
+ bv_len = to_sector(bio->bi_io_vec[i].bv_len);
+
+ if (bv_len > remaining)
+ break;
+
+ remaining -= bv_len;
+ len += bv_len;
+ }
+
+ clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len);
+ __map_bio(ti, clone, tio);
+
+ ci->sector += len;
+ ci->sector_count -= len;
+ ci->idx = i;
+
+ } else {
+ /*
+ * Create two copy bios to deal with io that has
+ * been split across a target.
+ */
+ struct bio_vec *bv = bio->bi_io_vec + ci->idx;
+
+ clone = split_bvec(bio, ci->sector, ci->idx,
+ bv->bv_offset, max);
+ __map_bio(ti, clone, tio);
+
+ ci->sector += max;
+ ci->sector_count -= max;
+ ti = dm_table_find_target(ci->map, ci->sector);
+
+ len = to_sector(bv->bv_len) - max;
+ clone = split_bvec(bio, ci->sector, ci->idx,
+ bv->bv_offset + to_bytes(max), len);
+ tio = alloc_tio(ci->md);
+ tio->io = ci->io;
+ tio->ti = ti;
+ memset(&tio->info, 0, sizeof(tio->info));
+ __map_bio(ti, clone, tio);
+
+ ci->sector += len;
+ ci->sector_count -= len;
+ ci->idx++;
+ }
+}
+
+/*
+ * Split the bio into several clones.
+ */
+static void __split_bio(struct mapped_device *md, struct bio *bio)
+{
+ struct clone_info ci;
+
+ ci.map = dm_get_table(md);
+ if (!ci.map) {
+ bio_io_error(bio, bio->bi_size);
+ return;
+ }
+
+ ci.md = md;
+ ci.bio = bio;
+ ci.io = alloc_io(md);
+ ci.io->error = 0;
+ atomic_set(&ci.io->io_count, 1);
+ ci.io->bio = bio;
+ ci.io->md = md;
+ ci.sector = bio->bi_sector;
+ ci.sector_count = bio_sectors(bio);
+ ci.idx = bio->bi_idx;
+
+ atomic_inc(&md->pending);
+ while (ci.sector_count)
+ __clone_and_map(&ci);
+
+ /* drop the extra reference count */
+ dec_pending(ci.io, 0);
+ dm_table_put(ci.map);
+}
+/*-----------------------------------------------------------------
+ * CRUD END
+ *---------------------------------------------------------------*/
+
+/*
+ * The request function that just remaps the bio built up by
+ * dm_merge_bvec.
+ */
+static int dm_request(request_queue_t *q, struct bio *bio)
+{
+ int r;
+ struct mapped_device *md = q->queuedata;
+
+ down_read(&md->lock);
+
+ /*
+ * If we're suspended we have to queue
+ * this io for later.
+ */
+ while (test_bit(DMF_BLOCK_IO, &md->flags)) {
+ up_read(&md->lock);
+
+ if (bio_rw(bio) == READA) {
+ bio_io_error(bio, bio->bi_size);
+ return 0;
+ }
+
+ r = queue_io(md, bio);
+ if (r < 0) {
+ bio_io_error(bio, bio->bi_size);
+ return 0;
+
+ } else if (r == 0)
+ return 0; /* deferred successfully */
+
+ /*
+ * We're in a while loop, because someone could suspend
+ * before we get to the following read lock.
+ */
+ down_read(&md->lock);
+ }
+
+ __split_bio(md, bio);
+ up_read(&md->lock);
+ return 0;
+}
+
+static int dm_flush_all(request_queue_t *q, struct gendisk *disk,
+ sector_t *error_sector)
+{
+ struct mapped_device *md = q->queuedata;
+ struct dm_table *map = dm_get_table(md);
+ int ret = -ENXIO;
+
+ if (map) {
+ ret = dm_table_flush_all(md->map);
+ dm_table_put(map);
+ }
+
+ return ret;
+}
+
+static void dm_unplug_all(request_queue_t *q)
+{
+ struct mapped_device *md = q->queuedata;
+ struct dm_table *map = dm_get_table(md);
+
+ if (map) {
+ dm_table_unplug_all(map);
+ dm_table_put(map);
+ }
+}
+
+static int dm_any_congested(void *congested_data, int bdi_bits)
+{
+ int r;
+ struct mapped_device *md = (struct mapped_device *) congested_data;
+ struct dm_table *map = dm_get_table(md);
+
+ if (!map || test_bit(DMF_BLOCK_IO, &md->flags))
+ r = bdi_bits;
+ else
+ r = dm_table_any_congested(map, bdi_bits);
+
+ dm_table_put(map);
+ return r;
+}
+
+/*-----------------------------------------------------------------
+ * An IDR is used to keep track of allocated minor numbers.
+ *---------------------------------------------------------------*/
+static DECLARE_MUTEX(_minor_lock);
+static DEFINE_IDR(_minor_idr);
+
+static void free_minor(unsigned int minor)
+{
+ down(&_minor_lock);
+ idr_remove(&_minor_idr, minor);
+ up(&_minor_lock);
+}
+
+/*
+ * See if the device with a specific minor # is free.
+ */
+static int specific_minor(struct mapped_device *md, unsigned int minor)
+{
+ int r, m;
+
+ if (minor >= (1 << MINORBITS))
+ return -EINVAL;
+
+ down(&_minor_lock);
+
+ if (idr_find(&_minor_idr, minor)) {
+ r = -EBUSY;
+ goto out;
+ }
+
+ r = idr_pre_get(&_minor_idr, GFP_KERNEL);
+ if (!r) {
+ r = -ENOMEM;
+ goto out;
+ }
+
+ r = idr_get_new_above(&_minor_idr, md, minor, &m);
+ if (r) {
+ goto out;
+ }
+
+ if (m != minor) {
+ idr_remove(&_minor_idr, m);
+ r = -EBUSY;
+ goto out;
+ }
+
+out:
+ up(&_minor_lock);
+ return r;
+}
+
+static int next_free_minor(struct mapped_device *md, unsigned int *minor)
+{
+ int r;
+ unsigned int m;
+
+ down(&_minor_lock);
+
+ r = idr_pre_get(&_minor_idr, GFP_KERNEL);
+ if (!r) {
+ r = -ENOMEM;
+ goto out;
+ }
+
+ r = idr_get_new(&_minor_idr, md, &m);
+ if (r) {
+ goto out;
+ }
+
+ if (m >= (1 << MINORBITS)) {
+ idr_remove(&_minor_idr, m);
+ r = -ENOSPC;
+ goto out;
+ }
+
+ *minor = m;
+
+out:
+ up(&_minor_lock);
+ return r;
+}
+
+static struct block_device_operations dm_blk_dops;
+
+/*
+ * Allocate and initialise a blank device with a given minor.
+ */
+static struct mapped_device *alloc_dev(unsigned int minor, int persistent)
+{
+ int r;
+ struct mapped_device *md = kmalloc(sizeof(*md), GFP_KERNEL);
+
+ if (!md) {
+ DMWARN("unable to allocate device, out of memory.");
+ return NULL;
+ }
+
+ /* get a minor number for the dev */
+ r = persistent ? specific_minor(md, minor) : next_free_minor(md, &minor);
+ if (r < 0)
+ goto bad1;
+
+ memset(md, 0, sizeof(*md));
+ init_rwsem(&md->lock);
+ rwlock_init(&md->map_lock);
+ atomic_set(&md->holders, 1);
+ atomic_set(&md->event_nr, 0);
+
+ md->queue = blk_alloc_queue(GFP_KERNEL);
+ if (!md->queue)
+ goto bad1;
+
+ md->queue->queuedata = md;
+ md->queue->backing_dev_info.congested_fn = dm_any_congested;
+ md->queue->backing_dev_info.congested_data = md;
+ blk_queue_make_request(md->queue, dm_request);
+ md->queue->unplug_fn = dm_unplug_all;
+ md->queue->issue_flush_fn = dm_flush_all;
+
+ md->io_pool = mempool_create(MIN_IOS, mempool_alloc_slab,
+ mempool_free_slab, _io_cache);
+ if (!md->io_pool)
+ goto bad2;
+
+ md->tio_pool = mempool_create(MIN_IOS, mempool_alloc_slab,
+ mempool_free_slab, _tio_cache);
+ if (!md->tio_pool)
+ goto bad3;
+
+ md->disk = alloc_disk(1);
+ if (!md->disk)
+ goto bad4;
+
+ md->disk->major = _major;
+ md->disk->first_minor = minor;
+ md->disk->fops = &dm_blk_dops;
+ md->disk->queue = md->queue;
+ md->disk->private_data = md;
+ sprintf(md->disk->disk_name, "dm-%d", minor);
+ add_disk(md->disk);
+
+ atomic_set(&md->pending, 0);
+ init_waitqueue_head(&md->wait);
+ init_waitqueue_head(&md->eventq);
+
+ return md;
+
+ bad4:
+ mempool_destroy(md->tio_pool);
+ bad3:
+ mempool_destroy(md->io_pool);
+ bad2:
+ blk_put_queue(md->queue);
+ free_minor(minor);
+ bad1:
+ kfree(md);
+ return NULL;
+}
+
+static void free_dev(struct mapped_device *md)
+{
+ free_minor(md->disk->first_minor);
+ mempool_destroy(md->tio_pool);
+ mempool_destroy(md->io_pool);
+ del_gendisk(md->disk);
+ put_disk(md->disk);
+ blk_put_queue(md->queue);
+ kfree(md);
+}
+
+/*
+ * Bind a table to the device.
+ */
+static void event_callback(void *context)
+{
+ struct mapped_device *md = (struct mapped_device *) context;
+
+ atomic_inc(&md->event_nr);
+ wake_up(&md->eventq);
+}
+
+static void __set_size(struct gendisk *disk, sector_t size)
+{
+ struct block_device *bdev;
+
+ set_capacity(disk, size);
+ bdev = bdget_disk(disk, 0);
+ if (bdev) {
+ down(&bdev->bd_inode->i_sem);
+ i_size_write(bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
+ up(&bdev->bd_inode->i_sem);
+ bdput(bdev);
+ }
+}
+
+static int __bind(struct mapped_device *md, struct dm_table *t)
+{
+ request_queue_t *q = md->queue;
+ sector_t size;
+
+ size = dm_table_get_size(t);
+ __set_size(md->disk, size);
+ if (size == 0)
+ return 0;
+
+ write_lock(&md->map_lock);
+ md->map = t;
+ write_unlock(&md->map_lock);
+
+ dm_table_get(t);
+ dm_table_event_callback(md->map, event_callback, md);
+ dm_table_set_restrictions(t, q);
+ return 0;
+}
+
+static void __unbind(struct mapped_device *md)
+{
+ struct dm_table *map = md->map;
+
+ if (!map)
+ return;
+
+ dm_table_event_callback(map, NULL, NULL);
+ write_lock(&md->map_lock);
+ md->map = NULL;
+ write_unlock(&md->map_lock);
+ dm_table_put(map);
+}
+
+/*
+ * Constructor for a new device.
+ */
+static int create_aux(unsigned int minor, int persistent,
+ struct mapped_device **result)
+{
+ struct mapped_device *md;
+
+ md = alloc_dev(minor, persistent);
+ if (!md)
+ return -ENXIO;
+
+ *result = md;
+ return 0;
+}
+
+int dm_create(struct mapped_device **result)
+{
+ return create_aux(0, 0, result);
+}
+
+int dm_create_with_minor(unsigned int minor, struct mapped_device **result)
+{
+ return create_aux(minor, 1, result);
+}
+
+void *dm_get_mdptr(dev_t dev)
+{
+ struct mapped_device *md;
+ void *mdptr = NULL;
+ unsigned minor = MINOR(dev);
+
+ if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
+ return NULL;
+
+ down(&_minor_lock);
+
+ md = idr_find(&_minor_idr, minor);
+
+ if (md && (dm_disk(md)->first_minor == minor))
+ mdptr = md->interface_ptr;
+
+ up(&_minor_lock);
+
+ return mdptr;
+}
+
+void dm_set_mdptr(struct mapped_device *md, void *ptr)
+{
+ md->interface_ptr = ptr;
+}
+
+void dm_get(struct mapped_device *md)
+{
+ atomic_inc(&md->holders);
+}
+
+void dm_put(struct mapped_device *md)
+{
+ struct dm_table *map = dm_get_table(md);
+
+ if (atomic_dec_and_test(&md->holders)) {
+ if (!test_bit(DMF_SUSPENDED, &md->flags) && map) {
+ dm_table_presuspend_targets(map);
+ dm_table_postsuspend_targets(map);
+ }
+ __unbind(md);
+ free_dev(md);
+ }
+
+ dm_table_put(map);
+}
+
+/*
+ * Process the deferred bios
+ */
+static void __flush_deferred_io(struct mapped_device *md, struct bio *c)
+{
+ struct bio *n;
+
+ while (c) {
+ n = c->bi_next;
+ c->bi_next = NULL;
+ __split_bio(md, c);
+ c = n;
+ }
+}
+
+/*
+ * Swap in a new table (destroying old one).
+ */
+int dm_swap_table(struct mapped_device *md, struct dm_table *table)
+{
+ int r;
+
+ down_write(&md->lock);
+
+ /* device must be suspended */
+ if (!test_bit(DMF_SUSPENDED, &md->flags)) {
+ up_write(&md->lock);
+ return -EPERM;
+ }
+
+ __unbind(md);
+ r = __bind(md, table);
+ if (r)
+ return r;
+
+ up_write(&md->lock);
+ return 0;
+}
+
+/*
+ * Functions to lock and unlock any filesystem running on the
+ * device.
+ */
+static int __lock_fs(struct mapped_device *md)
+{
+ struct block_device *bdev;
+
+ if (test_and_set_bit(DMF_FS_LOCKED, &md->flags))
+ return 0;
+
+ bdev = bdget_disk(md->disk, 0);
+ if (!bdev) {
+ DMWARN("bdget failed in __lock_fs");
+ return -ENOMEM;
+ }
+
+ WARN_ON(md->frozen_sb);
+ md->frozen_sb = freeze_bdev(bdev);
+ /* don't bdput right now, we don't want the bdev
+ * to go away while it is locked. We'll bdput
+ * in __unlock_fs
+ */
+ return 0;
+}
+
+static int __unlock_fs(struct mapped_device *md)
+{
+ struct block_device *bdev;
+
+ if (!test_and_clear_bit(DMF_FS_LOCKED, &md->flags))
+ return 0;
+
+ bdev = bdget_disk(md->disk, 0);
+ if (!bdev) {
+ DMWARN("bdget failed in __unlock_fs");
+ return -ENOMEM;
+ }
+
+ thaw_bdev(bdev, md->frozen_sb);
+ md->frozen_sb = NULL;
+ bdput(bdev);
+ bdput(bdev);
+ return 0;
+}
+
+/*
+ * We need to be able to change a mapping table under a mounted
+ * filesystem. For example we might want to move some data in
+ * the background. Before the table can be swapped with
+ * dm_bind_table, dm_suspend must be called to flush any in
+ * flight bios and ensure that any further io gets deferred.
+ */
+int dm_suspend(struct mapped_device *md)
+{
+ struct dm_table *map;
+ DECLARE_WAITQUEUE(wait, current);
+
+ /* Flush I/O to the device. */
+ down_read(&md->lock);
+ if (test_bit(DMF_BLOCK_IO, &md->flags)) {
+ up_read(&md->lock);
+ return -EINVAL;
+ }
+
+ map = dm_get_table(md);
+ if (map)
+ dm_table_presuspend_targets(map);
+ __lock_fs(md);
+
+ up_read(&md->lock);
+
+ /*
+ * First we set the BLOCK_IO flag so no more ios will be
+ * mapped.
+ */
+ down_write(&md->lock);
+ if (test_bit(DMF_BLOCK_IO, &md->flags)) {
+ /*
+ * If we get here we know another thread is
+ * trying to suspend as well, so we leave the fs
+ * locked for this thread.
+ */
+ up_write(&md->lock);
+ return -EINVAL;
+ }
+
+ set_bit(DMF_BLOCK_IO, &md->flags);
+ add_wait_queue(&md->wait, &wait);
+ up_write(&md->lock);
+
+ /* unplug */
+ if (map) {
+ dm_table_unplug_all(map);
+ dm_table_put(map);
+ }
+
+ /*
+ * Then we wait for the already mapped ios to
+ * complete.
+ */
+ while (1) {
+ set_current_state(TASK_INTERRUPTIBLE);
+
+ if (!atomic_read(&md->pending) || signal_pending(current))
+ break;
+
+ io_schedule();
+ }
+ set_current_state(TASK_RUNNING);
+
+ down_write(&md->lock);
+ remove_wait_queue(&md->wait, &wait);
+
+ /* were we interrupted ? */
+ if (atomic_read(&md->pending)) {
+ __unlock_fs(md);
+ clear_bit(DMF_BLOCK_IO, &md->flags);
+ up_write(&md->lock);
+ return -EINTR;
+ }
+
+ set_bit(DMF_SUSPENDED, &md->flags);
+
+ map = dm_get_table(md);
+ if (map)
+ dm_table_postsuspend_targets(map);
+ dm_table_put(map);
+ up_write(&md->lock);
+
+ return 0;
+}
+
+int dm_resume(struct mapped_device *md)
+{
+ struct bio *def;
+ struct dm_table *map = dm_get_table(md);
+
+ down_write(&md->lock);
+ if (!map ||
+ !test_bit(DMF_SUSPENDED, &md->flags) ||
+ !dm_table_get_size(map)) {
+ up_write(&md->lock);
+ dm_table_put(map);
+ return -EINVAL;
+ }
+
+ dm_table_resume_targets(map);
+ clear_bit(DMF_SUSPENDED, &md->flags);
+ clear_bit(DMF_BLOCK_IO, &md->flags);
+
+ def = bio_list_get(&md->deferred);
+ __flush_deferred_io(md, def);
+ up_write(&md->lock);
+ __unlock_fs(md);
+ dm_table_unplug_all(map);
+ dm_table_put(map);
+
+ return 0;
+}
+
+/*-----------------------------------------------------------------
+ * Event notification.
+ *---------------------------------------------------------------*/
+uint32_t dm_get_event_nr(struct mapped_device *md)
+{
+ return atomic_read(&md->event_nr);
+}
+
+int dm_wait_event(struct mapped_device *md, int event_nr)
+{
+ return wait_event_interruptible(md->eventq,
+ (event_nr != atomic_read(&md->event_nr)));
+}
+
+/*
+ * The gendisk is only valid as long as you have a reference
+ * count on 'md'.
+ */
+struct gendisk *dm_disk(struct mapped_device *md)
+{
+ return md->disk;
+}
+
+int dm_suspended(struct mapped_device *md)
+{
+ return test_bit(DMF_SUSPENDED, &md->flags);
+}
+
+static struct block_device_operations dm_blk_dops = {
+ .open = dm_blk_open,
+ .release = dm_blk_close,
+ .owner = THIS_MODULE
+};
+
+EXPORT_SYMBOL(dm_get_mapinfo);
+
+/*
+ * module hooks
+ */
+module_init(dm_init);
+module_exit(dm_exit);
+
+module_param(major, uint, 0);
+MODULE_PARM_DESC(major, "The major number of the device mapper");
+MODULE_DESCRIPTION(DM_NAME " driver");
+MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm.h b/drivers/md/dm.h
new file mode 100644
index 000000000000..e38c3fc1a1db
--- /dev/null
+++ b/drivers/md/dm.h
@@ -0,0 +1,195 @@
+/*
+ * Internal header file for device mapper
+ *
+ * Copyright (C) 2001, 2002 Sistina Software
+ * Copyright (C) 2004 Red Hat, Inc. All rights reserved.
+ *
+ * This file is released under the LGPL.
+ */
+
+#ifndef DM_INTERNAL_H
+#define DM_INTERNAL_H
+
+#include <linux/fs.h>
+#include <linux/device-mapper.h>
+#include <linux/list.h>
+#include <linux/blkdev.h>
+
+#define DM_NAME "device-mapper"
+#define DMWARN(f, x...) printk(KERN_WARNING DM_NAME ": " f "\n" , ## x)
+#define DMERR(f, x...) printk(KERN_ERR DM_NAME ": " f "\n" , ## x)
+#define DMINFO(f, x...) printk(KERN_INFO DM_NAME ": " f "\n" , ## x)
+
+#define DMEMIT(x...) sz += ((sz >= maxlen) ? \
+ 0 : scnprintf(result + sz, maxlen - sz, x))
+
+/*
+ * FIXME: I think this should be with the definition of sector_t
+ * in types.h.
+ */
+#ifdef CONFIG_LBD
+#define SECTOR_FORMAT "%Lu"
+#else
+#define SECTOR_FORMAT "%lu"
+#endif
+
+#define SECTOR_SHIFT 9
+
+/*
+ * List of devices that a metadevice uses and should open/close.
+ */
+struct dm_dev {
+ struct list_head list;
+
+ atomic_t count;
+ int mode;
+ struct block_device *bdev;
+ char name[16];
+};
+
+struct dm_table;
+struct mapped_device;
+
+/*-----------------------------------------------------------------
+ * Functions for manipulating a struct mapped_device.
+ * Drop the reference with dm_put when you finish with the object.
+ *---------------------------------------------------------------*/
+int dm_create(struct mapped_device **md);
+int dm_create_with_minor(unsigned int minor, struct mapped_device **md);
+void dm_set_mdptr(struct mapped_device *md, void *ptr);
+void *dm_get_mdptr(dev_t dev);
+
+/*
+ * Reference counting for md.
+ */
+void dm_get(struct mapped_device *md);
+void dm_put(struct mapped_device *md);
+
+/*
+ * A device can still be used while suspended, but I/O is deferred.
+ */
+int dm_suspend(struct mapped_device *md);
+int dm_resume(struct mapped_device *md);
+
+/*
+ * The device must be suspended before calling this method.
+ */
+int dm_swap_table(struct mapped_device *md, struct dm_table *t);
+
+/*
+ * Drop a reference on the table when you've finished with the
+ * result.
+ */
+struct dm_table *dm_get_table(struct mapped_device *md);
+
+/*
+ * Event functions.
+ */
+uint32_t dm_get_event_nr(struct mapped_device *md);
+int dm_wait_event(struct mapped_device *md, int event_nr);
+
+/*
+ * Info functions.
+ */
+struct gendisk *dm_disk(struct mapped_device *md);
+int dm_suspended(struct mapped_device *md);
+
+/*-----------------------------------------------------------------
+ * Functions for manipulating a table. Tables are also reference
+ * counted.
+ *---------------------------------------------------------------*/
+int dm_table_create(struct dm_table **result, int mode, unsigned num_targets);
+
+void dm_table_get(struct dm_table *t);
+void dm_table_put(struct dm_table *t);
+
+int dm_table_add_target(struct dm_table *t, const char *type,
+ sector_t start, sector_t len, char *params);
+int dm_table_complete(struct dm_table *t);
+void dm_table_event_callback(struct dm_table *t,
+ void (*fn)(void *), void *context);
+void dm_table_event(struct dm_table *t);
+sector_t dm_table_get_size(struct dm_table *t);
+struct dm_target *dm_table_get_target(struct dm_table *t, unsigned int index);
+struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector);
+void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q);
+unsigned int dm_table_get_num_targets(struct dm_table *t);
+struct list_head *dm_table_get_devices(struct dm_table *t);
+int dm_table_get_mode(struct dm_table *t);
+void dm_table_presuspend_targets(struct dm_table *t);
+void dm_table_postsuspend_targets(struct dm_table *t);
+void dm_table_resume_targets(struct dm_table *t);
+int dm_table_any_congested(struct dm_table *t, int bdi_bits);
+void dm_table_unplug_all(struct dm_table *t);
+int dm_table_flush_all(struct dm_table *t);
+
+/*-----------------------------------------------------------------
+ * A registry of target types.
+ *---------------------------------------------------------------*/
+int dm_target_init(void);
+void dm_target_exit(void);
+struct target_type *dm_get_target_type(const char *name);
+void dm_put_target_type(struct target_type *t);
+int dm_target_iterate(void (*iter_func)(struct target_type *tt,
+ void *param), void *param);
+
+
+/*-----------------------------------------------------------------
+ * Useful inlines.
+ *---------------------------------------------------------------*/
+static inline int array_too_big(unsigned long fixed, unsigned long obj,
+ unsigned long num)
+{
+ return (num > (ULONG_MAX - fixed) / obj);
+}
+
+/*
+ * Ceiling(n / sz)
+ */
+#define dm_div_up(n, sz) (((n) + (sz) - 1) / (sz))
+
+#define dm_sector_div_up(n, sz) ( \
+{ \
+ sector_t _r = ((n) + (sz) - 1); \
+ sector_div(_r, (sz)); \
+ _r; \
+} \
+)
+
+/*
+ * ceiling(n / size) * size
+ */
+#define dm_round_up(n, sz) (dm_div_up((n), (sz)) * (sz))
+
+static inline sector_t to_sector(unsigned long n)
+{
+ return (n >> 9);
+}
+
+static inline unsigned long to_bytes(sector_t n)
+{
+ return (n << 9);
+}
+
+int dm_split_args(int *argc, char ***argvp, char *input);
+
+/*
+ * The device-mapper can be driven through one of two interfaces;
+ * ioctl or filesystem, depending which patch you have applied.
+ */
+int dm_interface_init(void);
+void dm_interface_exit(void);
+
+/*
+ * Targets for linear and striped mappings
+ */
+int dm_linear_init(void);
+void dm_linear_exit(void);
+
+int dm_stripe_init(void);
+void dm_stripe_exit(void);
+
+void *dm_vcalloc(unsigned long nmemb, unsigned long elem_size);
+union map_info *dm_get_mapinfo(struct bio *bio);
+
+#endif
diff --git a/drivers/md/faulty.c b/drivers/md/faulty.c
new file mode 100644
index 000000000000..0248f8e7eac0
--- /dev/null
+++ b/drivers/md/faulty.c
@@ -0,0 +1,343 @@
+/*
+ * faulty.c : Multiple Devices driver for Linux
+ *
+ * Copyright (C) 2004 Neil Brown
+ *
+ * fautly-device-simulator personality for md
+ *
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * (for example /usr/src/linux/COPYING); if not, write to the Free
+ * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+
+/*
+ * The "faulty" personality causes some requests to fail.
+ *
+ * Possible failure modes are:
+ * reads fail "randomly" but succeed on retry
+ * writes fail "randomly" but succeed on retry
+ * reads for some address fail and then persist until a write
+ * reads for some address fail and then persist irrespective of write
+ * writes for some address fail and persist
+ * all writes fail
+ *
+ * Different modes can be active at a time, but only
+ * one can be set at array creation. Others can be added later.
+ * A mode can be one-shot or recurrent with the recurrance being
+ * once in every N requests.
+ * The bottom 5 bits of the "layout" indicate the mode. The
+ * remainder indicate a period, or 0 for one-shot.
+ *
+ * There is an implementation limit on the number of concurrently
+ * persisting-faulty blocks. When a new fault is requested that would
+ * exceed the limit, it is ignored.
+ * All current faults can be clear using a layout of "0".
+ *
+ * Requests are always sent to the device. If they are to fail,
+ * we clone the bio and insert a new b_end_io into the chain.
+ */
+
+#define WriteTransient 0
+#define ReadTransient 1
+#define WritePersistent 2
+#define ReadPersistent 3
+#define WriteAll 4 /* doesn't go to device */
+#define ReadFixable 5
+#define Modes 6
+
+#define ClearErrors 31
+#define ClearFaults 30
+
+#define AllPersist 100 /* internal use only */
+#define NoPersist 101
+
+#define ModeMask 0x1f
+#define ModeShift 5
+
+#define MaxFault 50
+#include <linux/raid/md.h>
+
+
+static int faulty_fail(struct bio *bio, unsigned int bytes_done, int error)
+{
+ struct bio *b = bio->bi_private;
+
+ b->bi_size = bio->bi_size;
+ b->bi_sector = bio->bi_sector;
+
+ if (bio->bi_size == 0)
+ bio_put(bio);
+
+ clear_bit(BIO_UPTODATE, &b->bi_flags);
+ return (b->bi_end_io)(b, bytes_done, -EIO);
+}
+
+typedef struct faulty_conf {
+ int period[Modes];
+ atomic_t counters[Modes];
+ sector_t faults[MaxFault];
+ int modes[MaxFault];
+ int nfaults;
+ mdk_rdev_t *rdev;
+} conf_t;
+
+static int check_mode(conf_t *conf, int mode)
+{
+ if (conf->period[mode] == 0 &&
+ atomic_read(&conf->counters[mode]) <= 0)
+ return 0; /* no failure, no decrement */
+
+
+ if (atomic_dec_and_test(&conf->counters[mode])) {
+ if (conf->period[mode])
+ atomic_set(&conf->counters[mode], conf->period[mode]);
+ return 1;
+ }
+ return 0;
+}
+
+static int check_sector(conf_t *conf, sector_t start, sector_t end, int dir)
+{
+ /* If we find a ReadFixable sector, we fix it ... */
+ int i;
+ for (i=0; i<conf->nfaults; i++)
+ if (conf->faults[i] >= start &&
+ conf->faults[i] < end) {
+ /* found it ... */
+ switch (conf->modes[i] * 2 + dir) {
+ case WritePersistent*2+WRITE: return 1;
+ case ReadPersistent*2+READ: return 1;
+ case ReadFixable*2+READ: return 1;
+ case ReadFixable*2+WRITE:
+ conf->modes[i] = NoPersist;
+ return 0;
+ case AllPersist*2+READ:
+ case AllPersist*2+WRITE: return 1;
+ default:
+ return 0;
+ }
+ }
+ return 0;
+}
+
+static void add_sector(conf_t *conf, sector_t start, int mode)
+{
+ int i;
+ int n = conf->nfaults;
+ for (i=0; i<conf->nfaults; i++)
+ if (conf->faults[i] == start) {
+ switch(mode) {
+ case NoPersist: conf->modes[i] = mode; return;
+ case WritePersistent:
+ if (conf->modes[i] == ReadPersistent ||
+ conf->modes[i] == ReadFixable)
+ conf->modes[i] = AllPersist;
+ else
+ conf->modes[i] = WritePersistent;
+ return;
+ case ReadPersistent:
+ if (conf->modes[i] == WritePersistent)
+ conf->modes[i] = AllPersist;
+ else
+ conf->modes[i] = ReadPersistent;
+ return;
+ case ReadFixable:
+ if (conf->modes[i] == WritePersistent ||
+ conf->modes[i] == ReadPersistent)
+ conf->modes[i] = AllPersist;
+ else
+ conf->modes[i] = ReadFixable;
+ return;
+ }
+ } else if (conf->modes[i] == NoPersist)
+ n = i;
+
+ if (n >= MaxFault)
+ return;
+ conf->faults[n] = start;
+ conf->modes[n] = mode;
+ if (conf->nfaults == n)
+ conf->nfaults = n+1;
+}
+
+static int make_request(request_queue_t *q, struct bio *bio)
+{
+ mddev_t *mddev = q->queuedata;
+ conf_t *conf = (conf_t*)mddev->private;
+ int failit = 0;
+
+ if (bio->bi_rw & 1) {
+ /* write request */
+ if (atomic_read(&conf->counters[WriteAll])) {
+ /* special case - don't decrement, don't generic_make_request,
+ * just fail immediately
+ */
+ bio_endio(bio, bio->bi_size, -EIO);
+ return 0;
+ }
+
+ if (check_sector(conf, bio->bi_sector, bio->bi_sector+(bio->bi_size>>9),
+ WRITE))
+ failit = 1;
+ if (check_mode(conf, WritePersistent)) {
+ add_sector(conf, bio->bi_sector, WritePersistent);
+ failit = 1;
+ }
+ if (check_mode(conf, WriteTransient))
+ failit = 1;
+ } else {
+ /* read request */
+ if (check_sector(conf, bio->bi_sector, bio->bi_sector + (bio->bi_size>>9),
+ READ))
+ failit = 1;
+ if (check_mode(conf, ReadTransient))
+ failit = 1;
+ if (check_mode(conf, ReadPersistent)) {
+ add_sector(conf, bio->bi_sector, ReadPersistent);
+ failit = 1;
+ }
+ if (check_mode(conf, ReadFixable)) {
+ add_sector(conf, bio->bi_sector, ReadFixable);
+ failit = 1;
+ }
+ }
+ if (failit) {
+ struct bio *b = bio_clone(bio, GFP_NOIO);
+ b->bi_bdev = conf->rdev->bdev;
+ b->bi_private = bio;
+ b->bi_end_io = faulty_fail;
+ generic_make_request(b);
+ return 0;
+ } else {
+ bio->bi_bdev = conf->rdev->bdev;
+ return 1;
+ }
+}
+
+static void status(struct seq_file *seq, mddev_t *mddev)
+{
+ conf_t *conf = (conf_t*)mddev->private;
+ int n;
+
+ if ((n=atomic_read(&conf->counters[WriteTransient])) != 0)
+ seq_printf(seq, " WriteTransient=%d(%d)",
+ n, conf->period[WriteTransient]);
+
+ if ((n=atomic_read(&conf->counters[ReadTransient])) != 0)
+ seq_printf(seq, " ReadTransient=%d(%d)",
+ n, conf->period[ReadTransient]);
+
+ if ((n=atomic_read(&conf->counters[WritePersistent])) != 0)
+ seq_printf(seq, " WritePersistent=%d(%d)",
+ n, conf->period[WritePersistent]);
+
+ if ((n=atomic_read(&conf->counters[ReadPersistent])) != 0)
+ seq_printf(seq, " ReadPersistent=%d(%d)",
+ n, conf->period[ReadPersistent]);
+
+
+ if ((n=atomic_read(&conf->counters[ReadFixable])) != 0)
+ seq_printf(seq, " ReadFixable=%d(%d)",
+ n, conf->period[ReadFixable]);
+
+ if ((n=atomic_read(&conf->counters[WriteAll])) != 0)
+ seq_printf(seq, " WriteAll");
+
+ seq_printf(seq, " nfaults=%d", conf->nfaults);
+}
+
+
+static int reconfig(mddev_t *mddev, int layout, int chunk_size)
+{
+ int mode = layout & ModeMask;
+ int count = layout >> ModeShift;
+ conf_t *conf = mddev->private;
+
+ if (chunk_size != -1)
+ return -EINVAL;
+
+ /* new layout */
+ if (mode == ClearFaults)
+ conf->nfaults = 0;
+ else if (mode == ClearErrors) {
+ int i;
+ for (i=0 ; i < Modes ; i++) {
+ conf->period[i] = 0;
+ atomic_set(&conf->counters[i], 0);
+ }
+ } else if (mode < Modes) {
+ conf->period[mode] = count;
+ if (!count) count++;
+ atomic_set(&conf->counters[mode], count);
+ } else
+ return -EINVAL;
+ mddev->layout = -1; /* makes sure further changes come through */
+ return 0;
+}
+
+static int run(mddev_t *mddev)
+{
+ mdk_rdev_t *rdev;
+ struct list_head *tmp;
+ int i;
+
+ conf_t *conf = kmalloc(sizeof(*conf), GFP_KERNEL);
+
+ for (i=0; i<Modes; i++) {
+ atomic_set(&conf->counters[i], 0);
+ conf->period[i] = 0;
+ }
+ conf->nfaults = 0;
+
+ ITERATE_RDEV(mddev, rdev, tmp)
+ conf->rdev = rdev;
+
+ mddev->array_size = mddev->size;
+ mddev->private = conf;
+
+ reconfig(mddev, mddev->layout, -1);
+
+ return 0;
+}
+
+static int stop(mddev_t *mddev)
+{
+ conf_t *conf = (conf_t *)mddev->private;
+
+ kfree(conf);
+ mddev->private = NULL;
+ return 0;
+}
+
+static mdk_personality_t faulty_personality =
+{
+ .name = "faulty",
+ .owner = THIS_MODULE,
+ .make_request = make_request,
+ .run = run,
+ .stop = stop,
+ .status = status,
+ .reconfig = reconfig,
+};
+
+static int __init raid_init(void)
+{
+ return register_md_personality(FAULTY, &faulty_personality);
+}
+
+static void raid_exit(void)
+{
+ unregister_md_personality(FAULTY);
+}
+
+module_init(raid_init);
+module_exit(raid_exit);
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("md-personality-10"); /* faulty */
diff --git a/drivers/md/kcopyd.c b/drivers/md/kcopyd.c
new file mode 100644
index 000000000000..eb7036485975
--- /dev/null
+++ b/drivers/md/kcopyd.c
@@ -0,0 +1,687 @@
+/*
+ * Copyright (C) 2002 Sistina Software (UK) Limited.
+ *
+ * This file is released under the GPL.
+ *
+ * Kcopyd provides a simple interface for copying an area of one
+ * block-device to one or more other block-devices, with an asynchronous
+ * completion notification.
+ */
+
+#include <asm/atomic.h>
+
+#include <linux/blkdev.h>
+#include <linux/config.h>
+#include <linux/fs.h>
+#include <linux/init.h>
+#include <linux/list.h>
+#include <linux/mempool.h>
+#include <linux/module.h>
+#include <linux/pagemap.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/workqueue.h>
+
+#include "kcopyd.h"
+
+static struct workqueue_struct *_kcopyd_wq;
+static struct work_struct _kcopyd_work;
+
+static inline void wake(void)
+{
+ queue_work(_kcopyd_wq, &_kcopyd_work);
+}
+
+/*-----------------------------------------------------------------
+ * Each kcopyd client has its own little pool of preallocated
+ * pages for kcopyd io.
+ *---------------------------------------------------------------*/
+struct kcopyd_client {
+ struct list_head list;
+
+ spinlock_t lock;
+ struct page_list *pages;
+ unsigned int nr_pages;
+ unsigned int nr_free_pages;
+};
+
+static struct page_list *alloc_pl(void)
+{
+ struct page_list *pl;
+
+ pl = kmalloc(sizeof(*pl), GFP_KERNEL);
+ if (!pl)
+ return NULL;
+
+ pl->page = alloc_page(GFP_KERNEL);
+ if (!pl->page) {
+ kfree(pl);
+ return NULL;
+ }
+
+ return pl;
+}
+
+static void free_pl(struct page_list *pl)
+{
+ __free_page(pl->page);
+ kfree(pl);
+}
+
+static int kcopyd_get_pages(struct kcopyd_client *kc,
+ unsigned int nr, struct page_list **pages)
+{
+ struct page_list *pl;
+
+ spin_lock(&kc->lock);
+ if (kc->nr_free_pages < nr) {
+ spin_unlock(&kc->lock);
+ return -ENOMEM;
+ }
+
+ kc->nr_free_pages -= nr;
+ for (*pages = pl = kc->pages; --nr; pl = pl->next)
+ ;
+
+ kc->pages = pl->next;
+ pl->next = NULL;
+
+ spin_unlock(&kc->lock);
+
+ return 0;
+}
+
+static void kcopyd_put_pages(struct kcopyd_client *kc, struct page_list *pl)
+{
+ struct page_list *cursor;
+
+ spin_lock(&kc->lock);
+ for (cursor = pl; cursor->next; cursor = cursor->next)
+ kc->nr_free_pages++;
+
+ kc->nr_free_pages++;
+ cursor->next = kc->pages;
+ kc->pages = pl;
+ spin_unlock(&kc->lock);
+}
+
+/*
+ * These three functions resize the page pool.
+ */
+static void drop_pages(struct page_list *pl)
+{
+ struct page_list *next;
+
+ while (pl) {
+ next = pl->next;
+ free_pl(pl);
+ pl = next;
+ }
+}
+
+static int client_alloc_pages(struct kcopyd_client *kc, unsigned int nr)
+{
+ unsigned int i;
+ struct page_list *pl = NULL, *next;
+
+ for (i = 0; i < nr; i++) {
+ next = alloc_pl();
+ if (!next) {
+ if (pl)
+ drop_pages(pl);
+ return -ENOMEM;
+ }
+ next->next = pl;
+ pl = next;
+ }
+
+ kcopyd_put_pages(kc, pl);
+ kc->nr_pages += nr;
+ return 0;
+}
+
+static void client_free_pages(struct kcopyd_client *kc)
+{
+ BUG_ON(kc->nr_free_pages != kc->nr_pages);
+ drop_pages(kc->pages);
+ kc->pages = NULL;
+ kc->nr_free_pages = kc->nr_pages = 0;
+}
+
+/*-----------------------------------------------------------------
+ * kcopyd_jobs need to be allocated by the *clients* of kcopyd,
+ * for this reason we use a mempool to prevent the client from
+ * ever having to do io (which could cause a deadlock).
+ *---------------------------------------------------------------*/
+struct kcopyd_job {
+ struct kcopyd_client *kc;
+ struct list_head list;
+ unsigned long flags;
+
+ /*
+ * Error state of the job.
+ */
+ int read_err;
+ unsigned int write_err;
+
+ /*
+ * Either READ or WRITE
+ */
+ int rw;
+ struct io_region source;
+
+ /*
+ * The destinations for the transfer.
+ */
+ unsigned int num_dests;
+ struct io_region dests[KCOPYD_MAX_REGIONS];
+
+ sector_t offset;
+ unsigned int nr_pages;
+ struct page_list *pages;
+
+ /*
+ * Set this to ensure you are notified when the job has
+ * completed. 'context' is for callback to use.
+ */
+ kcopyd_notify_fn fn;
+ void *context;
+
+ /*
+ * These fields are only used if the job has been split
+ * into more manageable parts.
+ */
+ struct semaphore lock;
+ atomic_t sub_jobs;
+ sector_t progress;
+};
+
+/* FIXME: this should scale with the number of pages */
+#define MIN_JOBS 512
+
+static kmem_cache_t *_job_cache;
+static mempool_t *_job_pool;
+
+/*
+ * We maintain three lists of jobs:
+ *
+ * i) jobs waiting for pages
+ * ii) jobs that have pages, and are waiting for the io to be issued.
+ * iii) jobs that have completed.
+ *
+ * All three of these are protected by job_lock.
+ */
+static DEFINE_SPINLOCK(_job_lock);
+
+static LIST_HEAD(_complete_jobs);
+static LIST_HEAD(_io_jobs);
+static LIST_HEAD(_pages_jobs);
+
+static int jobs_init(void)
+{
+ _job_cache = kmem_cache_create("kcopyd-jobs",
+ sizeof(struct kcopyd_job),
+ __alignof__(struct kcopyd_job),
+ 0, NULL, NULL);
+ if (!_job_cache)
+ return -ENOMEM;
+
+ _job_pool = mempool_create(MIN_JOBS, mempool_alloc_slab,
+ mempool_free_slab, _job_cache);
+ if (!_job_pool) {
+ kmem_cache_destroy(_job_cache);
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void jobs_exit(void)
+{
+ BUG_ON(!list_empty(&_complete_jobs));
+ BUG_ON(!list_empty(&_io_jobs));
+ BUG_ON(!list_empty(&_pages_jobs));
+
+ mempool_destroy(_job_pool);
+ kmem_cache_destroy(_job_cache);
+ _job_pool = NULL;
+ _job_cache = NULL;
+}
+
+/*
+ * Functions to push and pop a job onto the head of a given job
+ * list.
+ */
+static inline struct kcopyd_job *pop(struct list_head *jobs)
+{
+ struct kcopyd_job *job = NULL;
+ unsigned long flags;
+
+ spin_lock_irqsave(&_job_lock, flags);
+
+ if (!list_empty(jobs)) {
+ job = list_entry(jobs->next, struct kcopyd_job, list);
+ list_del(&job->list);
+ }
+ spin_unlock_irqrestore(&_job_lock, flags);
+
+ return job;
+}
+
+static inline void push(struct list_head *jobs, struct kcopyd_job *job)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&_job_lock, flags);
+ list_add_tail(&job->list, jobs);
+ spin_unlock_irqrestore(&_job_lock, flags);
+}
+
+/*
+ * These three functions process 1 item from the corresponding
+ * job list.
+ *
+ * They return:
+ * < 0: error
+ * 0: success
+ * > 0: can't process yet.
+ */
+static int run_complete_job(struct kcopyd_job *job)
+{
+ void *context = job->context;
+ int read_err = job->read_err;
+ unsigned int write_err = job->write_err;
+ kcopyd_notify_fn fn = job->fn;
+
+ kcopyd_put_pages(job->kc, job->pages);
+ mempool_free(job, _job_pool);
+ fn(read_err, write_err, context);
+ return 0;
+}
+
+static void complete_io(unsigned long error, void *context)
+{
+ struct kcopyd_job *job = (struct kcopyd_job *) context;
+
+ if (error) {
+ if (job->rw == WRITE)
+ job->write_err &= error;
+ else
+ job->read_err = 1;
+
+ if (!test_bit(KCOPYD_IGNORE_ERROR, &job->flags)) {
+ push(&_complete_jobs, job);
+ wake();
+ return;
+ }
+ }
+
+ if (job->rw == WRITE)
+ push(&_complete_jobs, job);
+
+ else {
+ job->rw = WRITE;
+ push(&_io_jobs, job);
+ }
+
+ wake();
+}
+
+/*
+ * Request io on as many buffer heads as we can currently get for
+ * a particular job.
+ */
+static int run_io_job(struct kcopyd_job *job)
+{
+ int r;
+
+ if (job->rw == READ)
+ r = dm_io_async(1, &job->source, job->rw,
+ job->pages,
+ job->offset, complete_io, job);
+
+ else
+ r = dm_io_async(job->num_dests, job->dests, job->rw,
+ job->pages,
+ job->offset, complete_io, job);
+
+ return r;
+}
+
+static int run_pages_job(struct kcopyd_job *job)
+{
+ int r;
+
+ job->nr_pages = dm_div_up(job->dests[0].count + job->offset,
+ PAGE_SIZE >> 9);
+ r = kcopyd_get_pages(job->kc, job->nr_pages, &job->pages);
+ if (!r) {
+ /* this job is ready for io */
+ push(&_io_jobs, job);
+ return 0;
+ }
+
+ if (r == -ENOMEM)
+ /* can't complete now */
+ return 1;
+
+ return r;
+}
+
+/*
+ * Run through a list for as long as possible. Returns the count
+ * of successful jobs.
+ */
+static int process_jobs(struct list_head *jobs, int (*fn) (struct kcopyd_job *))
+{
+ struct kcopyd_job *job;
+ int r, count = 0;
+
+ while ((job = pop(jobs))) {
+
+ r = fn(job);
+
+ if (r < 0) {
+ /* error this rogue job */
+ if (job->rw == WRITE)
+ job->write_err = (unsigned int) -1;
+ else
+ job->read_err = 1;
+ push(&_complete_jobs, job);
+ break;
+ }
+
+ if (r > 0) {
+ /*
+ * We couldn't service this job ATM, so
+ * push this job back onto the list.
+ */
+ push(jobs, job);
+ break;
+ }
+
+ count++;
+ }
+
+ return count;
+}
+
+/*
+ * kcopyd does this every time it's woken up.
+ */
+static void do_work(void *ignored)
+{
+ /*
+ * The order that these are called is *very* important.
+ * complete jobs can free some pages for pages jobs.
+ * Pages jobs when successful will jump onto the io jobs
+ * list. io jobs call wake when they complete and it all
+ * starts again.
+ */
+ process_jobs(&_complete_jobs, run_complete_job);
+ process_jobs(&_pages_jobs, run_pages_job);
+ process_jobs(&_io_jobs, run_io_job);
+}
+
+/*
+ * If we are copying a small region we just dispatch a single job
+ * to do the copy, otherwise the io has to be split up into many
+ * jobs.
+ */
+static void dispatch_job(struct kcopyd_job *job)
+{
+ push(&_pages_jobs, job);
+ wake();
+}
+
+#define SUB_JOB_SIZE 128
+static void segment_complete(int read_err,
+ unsigned int write_err, void *context)
+{
+ /* FIXME: tidy this function */
+ sector_t progress = 0;
+ sector_t count = 0;
+ struct kcopyd_job *job = (struct kcopyd_job *) context;
+
+ down(&job->lock);
+
+ /* update the error */
+ if (read_err)
+ job->read_err = 1;
+
+ if (write_err)
+ job->write_err &= write_err;
+
+ /*
+ * Only dispatch more work if there hasn't been an error.
+ */
+ if ((!job->read_err && !job->write_err) ||
+ test_bit(KCOPYD_IGNORE_ERROR, &job->flags)) {
+ /* get the next chunk of work */
+ progress = job->progress;
+ count = job->source.count - progress;
+ if (count) {
+ if (count > SUB_JOB_SIZE)
+ count = SUB_JOB_SIZE;
+
+ job->progress += count;
+ }
+ }
+ up(&job->lock);
+
+ if (count) {
+ int i;
+ struct kcopyd_job *sub_job = mempool_alloc(_job_pool, GFP_NOIO);
+
+ *sub_job = *job;
+ sub_job->source.sector += progress;
+ sub_job->source.count = count;
+
+ for (i = 0; i < job->num_dests; i++) {
+ sub_job->dests[i].sector += progress;
+ sub_job->dests[i].count = count;
+ }
+
+ sub_job->fn = segment_complete;
+ sub_job->context = job;
+ dispatch_job(sub_job);
+
+ } else if (atomic_dec_and_test(&job->sub_jobs)) {
+
+ /*
+ * To avoid a race we must keep the job around
+ * until after the notify function has completed.
+ * Otherwise the client may try and stop the job
+ * after we've completed.
+ */
+ job->fn(read_err, write_err, job->context);
+ mempool_free(job, _job_pool);
+ }
+}
+
+/*
+ * Create some little jobs that will do the move between
+ * them.
+ */
+#define SPLIT_COUNT 8
+static void split_job(struct kcopyd_job *job)
+{
+ int i;
+
+ atomic_set(&job->sub_jobs, SPLIT_COUNT);
+ for (i = 0; i < SPLIT_COUNT; i++)
+ segment_complete(0, 0u, job);
+}
+
+int kcopyd_copy(struct kcopyd_client *kc, struct io_region *from,
+ unsigned int num_dests, struct io_region *dests,
+ unsigned int flags, kcopyd_notify_fn fn, void *context)
+{
+ struct kcopyd_job *job;
+
+ /*
+ * Allocate a new job.
+ */
+ job = mempool_alloc(_job_pool, GFP_NOIO);
+
+ /*
+ * set up for the read.
+ */
+ job->kc = kc;
+ job->flags = flags;
+ job->read_err = 0;
+ job->write_err = 0;
+ job->rw = READ;
+
+ job->source = *from;
+
+ job->num_dests = num_dests;
+ memcpy(&job->dests, dests, sizeof(*dests) * num_dests);
+
+ job->offset = 0;
+ job->nr_pages = 0;
+ job->pages = NULL;
+
+ job->fn = fn;
+ job->context = context;
+
+ if (job->source.count < SUB_JOB_SIZE)
+ dispatch_job(job);
+
+ else {
+ init_MUTEX(&job->lock);
+ job->progress = 0;
+ split_job(job);
+ }
+
+ return 0;
+}
+
+/*
+ * Cancels a kcopyd job, eg. someone might be deactivating a
+ * mirror.
+ */
+int kcopyd_cancel(struct kcopyd_job *job, int block)
+{
+ /* FIXME: finish */
+ return -1;
+}
+
+/*-----------------------------------------------------------------
+ * Unit setup
+ *---------------------------------------------------------------*/
+static DECLARE_MUTEX(_client_lock);
+static LIST_HEAD(_clients);
+
+static void client_add(struct kcopyd_client *kc)
+{
+ down(&_client_lock);
+ list_add(&kc->list, &_clients);
+ up(&_client_lock);
+}
+
+static void client_del(struct kcopyd_client *kc)
+{
+ down(&_client_lock);
+ list_del(&kc->list);
+ up(&_client_lock);
+}
+
+static DECLARE_MUTEX(kcopyd_init_lock);
+static int kcopyd_clients = 0;
+
+static int kcopyd_init(void)
+{
+ int r;
+
+ down(&kcopyd_init_lock);
+
+ if (kcopyd_clients) {
+ /* Already initialized. */
+ kcopyd_clients++;
+ up(&kcopyd_init_lock);
+ return 0;
+ }
+
+ r = jobs_init();
+ if (r) {
+ up(&kcopyd_init_lock);
+ return r;
+ }
+
+ _kcopyd_wq = create_singlethread_workqueue("kcopyd");
+ if (!_kcopyd_wq) {
+ jobs_exit();
+ up(&kcopyd_init_lock);
+ return -ENOMEM;
+ }
+
+ kcopyd_clients++;
+ INIT_WORK(&_kcopyd_work, do_work, NULL);
+ up(&kcopyd_init_lock);
+ return 0;
+}
+
+static void kcopyd_exit(void)
+{
+ down(&kcopyd_init_lock);
+ kcopyd_clients--;
+ if (!kcopyd_clients) {
+ jobs_exit();
+ destroy_workqueue(_kcopyd_wq);
+ _kcopyd_wq = NULL;
+ }
+ up(&kcopyd_init_lock);
+}
+
+int kcopyd_client_create(unsigned int nr_pages, struct kcopyd_client **result)
+{
+ int r = 0;
+ struct kcopyd_client *kc;
+
+ r = kcopyd_init();
+ if (r)
+ return r;
+
+ kc = kmalloc(sizeof(*kc), GFP_KERNEL);
+ if (!kc) {
+ kcopyd_exit();
+ return -ENOMEM;
+ }
+
+ spin_lock_init(&kc->lock);
+ kc->pages = NULL;
+ kc->nr_pages = kc->nr_free_pages = 0;
+ r = client_alloc_pages(kc, nr_pages);
+ if (r) {
+ kfree(kc);
+ kcopyd_exit();
+ return r;
+ }
+
+ r = dm_io_get(nr_pages);
+ if (r) {
+ client_free_pages(kc);
+ kfree(kc);
+ kcopyd_exit();
+ return r;
+ }
+
+ client_add(kc);
+ *result = kc;
+ return 0;
+}
+
+void kcopyd_client_destroy(struct kcopyd_client *kc)
+{
+ dm_io_put(kc->nr_pages);
+ client_free_pages(kc);
+ client_del(kc);
+ kfree(kc);
+ kcopyd_exit();
+}
+
+EXPORT_SYMBOL(kcopyd_client_create);
+EXPORT_SYMBOL(kcopyd_client_destroy);
+EXPORT_SYMBOL(kcopyd_copy);
+EXPORT_SYMBOL(kcopyd_cancel);
diff --git a/drivers/md/kcopyd.h b/drivers/md/kcopyd.h
new file mode 100644
index 000000000000..4621ea055c0e
--- /dev/null
+++ b/drivers/md/kcopyd.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2001 Sistina Software
+ *
+ * This file is released under the GPL.
+ *
+ * Kcopyd provides a simple interface for copying an area of one
+ * block-device to one or more other block-devices, with an asynchronous
+ * completion notification.
+ */
+
+#ifndef DM_KCOPYD_H
+#define DM_KCOPYD_H
+
+#include "dm-io.h"
+
+/* FIXME: make this configurable */
+#define KCOPYD_MAX_REGIONS 8
+
+#define KCOPYD_IGNORE_ERROR 1
+
+/*
+ * To use kcopyd you must first create a kcopyd client object.
+ */
+struct kcopyd_client;
+int kcopyd_client_create(unsigned int num_pages, struct kcopyd_client **result);
+void kcopyd_client_destroy(struct kcopyd_client *kc);
+
+/*
+ * Submit a copy job to kcopyd. This is built on top of the
+ * previous three fns.
+ *
+ * read_err is a boolean,
+ * write_err is a bitset, with 1 bit for each destination region
+ */
+typedef void (*kcopyd_notify_fn)(int read_err,
+ unsigned int write_err, void *context);
+
+int kcopyd_copy(struct kcopyd_client *kc, struct io_region *from,
+ unsigned int num_dests, struct io_region *dests,
+ unsigned int flags, kcopyd_notify_fn fn, void *context);
+
+#endif
diff --git a/drivers/md/linear.c b/drivers/md/linear.c
new file mode 100644
index 000000000000..161e9aa87291
--- /dev/null
+++ b/drivers/md/linear.c
@@ -0,0 +1,343 @@
+/*
+ linear.c : Multiple Devices driver for Linux
+ Copyright (C) 1994-96 Marc ZYNGIER
+ <zyngier@ufr-info-p7.ibp.fr> or
+ <maz@gloups.fdn.fr>
+
+ Linear mode management functions.
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2, or (at your option)
+ any later version.
+
+ You should have received a copy of the GNU General Public License
+ (for example /usr/src/linux/COPYING); if not, write to the Free
+ Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+*/
+
+#include <linux/module.h>
+
+#include <linux/raid/md.h>
+#include <linux/slab.h>
+#include <linux/raid/linear.h>
+
+#define MAJOR_NR MD_MAJOR
+#define MD_DRIVER
+#define MD_PERSONALITY
+
+/*
+ * find which device holds a particular offset
+ */
+static inline dev_info_t *which_dev(mddev_t *mddev, sector_t sector)
+{
+ dev_info_t *hash;
+ linear_conf_t *conf = mddev_to_conf(mddev);
+ sector_t block = sector >> 1;
+
+ /*
+ * sector_div(a,b) returns the remainer and sets a to a/b
+ */
+ (void)sector_div(block, conf->smallest->size);
+ hash = conf->hash_table[block];
+
+ while ((sector>>1) >= (hash->size + hash->offset))
+ hash++;
+ return hash;
+}
+
+/**
+ * linear_mergeable_bvec -- tell bio layer if a two requests can be merged
+ * @q: request queue
+ * @bio: the buffer head that's been built up so far
+ * @biovec: the request that could be merged to it.
+ *
+ * Return amount of bytes we can take at this offset
+ */
+static int linear_mergeable_bvec(request_queue_t *q, struct bio *bio, struct bio_vec *biovec)
+{
+ mddev_t *mddev = q->queuedata;
+ dev_info_t *dev0;
+ unsigned long maxsectors, bio_sectors = bio->bi_size >> 9;
+ sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev);
+
+ dev0 = which_dev(mddev, sector);
+ maxsectors = (dev0->size << 1) - (sector - (dev0->offset<<1));
+
+ if (maxsectors < bio_sectors)
+ maxsectors = 0;
+ else
+ maxsectors -= bio_sectors;
+
+ if (maxsectors <= (PAGE_SIZE >> 9 ) && bio_sectors == 0)
+ return biovec->bv_len;
+ /* The bytes available at this offset could be really big,
+ * so we cap at 2^31 to avoid overflow */
+ if (maxsectors > (1 << (31-9)))
+ return 1<<31;
+ return maxsectors << 9;
+}
+
+static void linear_unplug(request_queue_t *q)
+{
+ mddev_t *mddev = q->queuedata;
+ linear_conf_t *conf = mddev_to_conf(mddev);
+ int i;
+
+ for (i=0; i < mddev->raid_disks; i++) {
+ request_queue_t *r_queue = bdev_get_queue(conf->disks[i].rdev->bdev);
+ if (r_queue->unplug_fn)
+ r_queue->unplug_fn(r_queue);
+ }
+}
+
+static int linear_issue_flush(request_queue_t *q, struct gendisk *disk,
+ sector_t *error_sector)
+{
+ mddev_t *mddev = q->queuedata;
+ linear_conf_t *conf = mddev_to_conf(mddev);
+ int i, ret = 0;
+
+ for (i=0; i < mddev->raid_disks && ret == 0; i++) {
+ struct block_device *bdev = conf->disks[i].rdev->bdev;
+ request_queue_t *r_queue = bdev_get_queue(bdev);
+
+ if (!r_queue->issue_flush_fn)
+ ret = -EOPNOTSUPP;
+ else
+ ret = r_queue->issue_flush_fn(r_queue, bdev->bd_disk, error_sector);
+ }
+ return ret;
+}
+
+static int linear_run (mddev_t *mddev)
+{
+ linear_conf_t *conf;
+ dev_info_t **table;
+ mdk_rdev_t *rdev;
+ int i, nb_zone, cnt;
+ sector_t start;
+ sector_t curr_offset;
+ struct list_head *tmp;
+
+ conf = kmalloc (sizeof (*conf) + mddev->raid_disks*sizeof(dev_info_t),
+ GFP_KERNEL);
+ if (!conf)
+ goto out;
+ memset(conf, 0, sizeof(*conf) + mddev->raid_disks*sizeof(dev_info_t));
+ mddev->private = conf;
+
+ /*
+ * Find the smallest device.
+ */
+
+ conf->smallest = NULL;
+ cnt = 0;
+ mddev->array_size = 0;
+
+ ITERATE_RDEV(mddev,rdev,tmp) {
+ int j = rdev->raid_disk;
+ dev_info_t *disk = conf->disks + j;
+
+ if (j < 0 || j > mddev->raid_disks || disk->rdev) {
+ printk("linear: disk numbering problem. Aborting!\n");
+ goto out;
+ }
+
+ disk->rdev = rdev;
+
+ blk_queue_stack_limits(mddev->queue,
+ rdev->bdev->bd_disk->queue);
+ /* as we don't honour merge_bvec_fn, we must never risk
+ * violating it, so limit ->max_sector to one PAGE, as
+ * a one page request is never in violation.
+ */
+ if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
+ mddev->queue->max_sectors > (PAGE_SIZE>>9))
+ blk_queue_max_sectors(mddev->queue, PAGE_SIZE>>9);
+
+ disk->size = rdev->size;
+ mddev->array_size += rdev->size;
+
+ if (!conf->smallest || (disk->size < conf->smallest->size))
+ conf->smallest = disk;
+ cnt++;
+ }
+ if (cnt != mddev->raid_disks) {
+ printk("linear: not enough drives present. Aborting!\n");
+ goto out;
+ }
+
+ /*
+ * This code was restructured to work around a gcc-2.95.3 internal
+ * compiler error. Alter it with care.
+ */
+ {
+ sector_t sz;
+ unsigned round;
+ unsigned long base;
+
+ sz = mddev->array_size;
+ base = conf->smallest->size;
+ round = sector_div(sz, base);
+ nb_zone = conf->nr_zones = sz + (round ? 1 : 0);
+ }
+
+ conf->hash_table = kmalloc (sizeof (dev_info_t*) * nb_zone,
+ GFP_KERNEL);
+ if (!conf->hash_table)
+ goto out;
+
+ /*
+ * Here we generate the linear hash table
+ */
+ table = conf->hash_table;
+ start = 0;
+ curr_offset = 0;
+ for (i = 0; i < cnt; i++) {
+ dev_info_t *disk = conf->disks + i;
+
+ disk->offset = curr_offset;
+ curr_offset += disk->size;
+
+ /* 'curr_offset' is the end of this disk
+ * 'start' is the start of table
+ */
+ while (start < curr_offset) {
+ *table++ = disk;
+ start += conf->smallest->size;
+ }
+ }
+ if (table-conf->hash_table != nb_zone)
+ BUG();
+
+ blk_queue_merge_bvec(mddev->queue, linear_mergeable_bvec);
+ mddev->queue->unplug_fn = linear_unplug;
+ mddev->queue->issue_flush_fn = linear_issue_flush;
+ return 0;
+
+out:
+ if (conf)
+ kfree(conf);
+ return 1;
+}
+
+static int linear_stop (mddev_t *mddev)
+{
+ linear_conf_t *conf = mddev_to_conf(mddev);
+
+ blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
+ kfree(conf->hash_table);
+ kfree(conf);
+
+ return 0;
+}
+
+static int linear_make_request (request_queue_t *q, struct bio *bio)
+{
+ mddev_t *mddev = q->queuedata;
+ dev_info_t *tmp_dev;
+ sector_t block;
+
+ if (bio_data_dir(bio)==WRITE) {
+ disk_stat_inc(mddev->gendisk, writes);
+ disk_stat_add(mddev->gendisk, write_sectors, bio_sectors(bio));
+ } else {
+ disk_stat_inc(mddev->gendisk, reads);
+ disk_stat_add(mddev->gendisk, read_sectors, bio_sectors(bio));
+ }
+
+ tmp_dev = which_dev(mddev, bio->bi_sector);
+ block = bio->bi_sector >> 1;
+
+ if (unlikely(block >= (tmp_dev->size + tmp_dev->offset)
+ || block < tmp_dev->offset)) {
+ char b[BDEVNAME_SIZE];
+
+ printk("linear_make_request: Block %llu out of bounds on "
+ "dev %s size %llu offset %llu\n",
+ (unsigned long long)block,
+ bdevname(tmp_dev->rdev->bdev, b),
+ (unsigned long long)tmp_dev->size,
+ (unsigned long long)tmp_dev->offset);
+ bio_io_error(bio, bio->bi_size);
+ return 0;
+ }
+ if (unlikely(bio->bi_sector + (bio->bi_size >> 9) >
+ (tmp_dev->offset + tmp_dev->size)<<1)) {
+ /* This bio crosses a device boundary, so we have to
+ * split it.
+ */
+ struct bio_pair *bp;
+ bp = bio_split(bio, bio_split_pool,
+ (bio->bi_sector + (bio->bi_size >> 9) -
+ (tmp_dev->offset + tmp_dev->size))<<1);
+ if (linear_make_request(q, &bp->bio1))
+ generic_make_request(&bp->bio1);
+ if (linear_make_request(q, &bp->bio2))
+ generic_make_request(&bp->bio2);
+ bio_pair_release(bp);
+ return 0;
+ }
+
+ bio->bi_bdev = tmp_dev->rdev->bdev;
+ bio->bi_sector = bio->bi_sector - (tmp_dev->offset << 1) + tmp_dev->rdev->data_offset;
+
+ return 1;
+}
+
+static void linear_status (struct seq_file *seq, mddev_t *mddev)
+{
+
+#undef MD_DEBUG
+#ifdef MD_DEBUG
+ int j;
+ linear_conf_t *conf = mddev_to_conf(mddev);
+ sector_t s = 0;
+
+ seq_printf(seq, " ");
+ for (j = 0; j < conf->nr_zones; j++)
+ {
+ char b[BDEVNAME_SIZE];
+ s += conf->smallest_size;
+ seq_printf(seq, "[%s",
+ bdevname(conf->hash_table[j][0].rdev->bdev,b));
+
+ while (s > conf->hash_table[j][0].offset +
+ conf->hash_table[j][0].size)
+ seq_printf(seq, "/%s] ",
+ bdevname(conf->hash_table[j][1].rdev->bdev,b));
+ else
+ seq_printf(seq, "] ");
+ }
+ seq_printf(seq, "\n");
+#endif
+ seq_printf(seq, " %dk rounding", mddev->chunk_size/1024);
+}
+
+
+static mdk_personality_t linear_personality=
+{
+ .name = "linear",
+ .owner = THIS_MODULE,
+ .make_request = linear_make_request,
+ .run = linear_run,
+ .stop = linear_stop,
+ .status = linear_status,
+};
+
+static int __init linear_init (void)
+{
+ return register_md_personality (LINEAR, &linear_personality);
+}
+
+static void linear_exit (void)
+{
+ unregister_md_personality (LINEAR);
+}
+
+
+module_init(linear_init);
+module_exit(linear_exit);
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("md-personality-1"); /* LINEAR */
diff --git a/drivers/md/md.c b/drivers/md/md.c
new file mode 100644
index 000000000000..04562add1920
--- /dev/null
+++ b/drivers/md/md.c
@@ -0,0 +1,3766 @@
+/*
+ md.c : Multiple Devices driver for Linux
+ Copyright (C) 1998, 1999, 2000 Ingo Molnar
+
+ completely rewritten, based on the MD driver code from Marc Zyngier
+
+ Changes:
+
+ - RAID-1/RAID-5 extensions by Miguel de Icaza, Gadi Oxman, Ingo Molnar
+ - RAID-6 extensions by H. Peter Anvin <hpa@zytor.com>
+ - boot support for linear and striped mode by Harald Hoyer <HarryH@Royal.Net>
+ - kerneld support by Boris Tobotras <boris@xtalk.msk.su>
+ - kmod support by: Cyrus Durgin
+ - RAID0 bugfixes: Mark Anthony Lisher <markal@iname.com>
+ - Devfs support by Richard Gooch <rgooch@atnf.csiro.au>
+
+ - lots of fixes and improvements to the RAID1/RAID5 and generic
+ RAID code (such as request based resynchronization):
+
+ Neil Brown <neilb@cse.unsw.edu.au>.
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2, or (at your option)
+ any later version.
+
+ You should have received a copy of the GNU General Public License
+ (for example /usr/src/linux/COPYING); if not, write to the Free
+ Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+*/
+
+#include <linux/module.h>
+#include <linux/config.h>
+#include <linux/linkage.h>
+#include <linux/raid/md.h>
+#include <linux/sysctl.h>
+#include <linux/devfs_fs_kernel.h>
+#include <linux/buffer_head.h> /* for invalidate_bdev */
+#include <linux/suspend.h>
+
+#include <linux/init.h>
+
+#ifdef CONFIG_KMOD
+#include <linux/kmod.h>
+#endif
+
+#include <asm/unaligned.h>
+
+#define MAJOR_NR MD_MAJOR
+#define MD_DRIVER
+
+/* 63 partitions with the alternate major number (mdp) */
+#define MdpMinorShift 6
+
+#define DEBUG 0
+#define dprintk(x...) ((void)(DEBUG && printk(x)))
+
+
+#ifndef MODULE
+static void autostart_arrays (int part);
+#endif
+
+static mdk_personality_t *pers[MAX_PERSONALITY];
+static DEFINE_SPINLOCK(pers_lock);
+
+/*
+ * Current RAID-1,4,5 parallel reconstruction 'guaranteed speed limit'
+ * is 1000 KB/sec, so the extra system load does not show up that much.
+ * Increase it if you want to have more _guaranteed_ speed. Note that
+ * the RAID driver will use the maximum available bandwith if the IO
+ * subsystem is idle. There is also an 'absolute maximum' reconstruction
+ * speed limit - in case reconstruction slows down your system despite
+ * idle IO detection.
+ *
+ * you can change it via /proc/sys/dev/raid/speed_limit_min and _max.
+ */
+
+static int sysctl_speed_limit_min = 1000;
+static int sysctl_speed_limit_max = 200000;
+
+static struct ctl_table_header *raid_table_header;
+
+static ctl_table raid_table[] = {
+ {
+ .ctl_name = DEV_RAID_SPEED_LIMIT_MIN,
+ .procname = "speed_limit_min",
+ .data = &sysctl_speed_limit_min,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = &proc_dointvec,
+ },
+ {
+ .ctl_name = DEV_RAID_SPEED_LIMIT_MAX,
+ .procname = "speed_limit_max",
+ .data = &sysctl_speed_limit_max,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = &proc_dointvec,
+ },
+ { .ctl_name = 0 }
+};
+
+static ctl_table raid_dir_table[] = {
+ {
+ .ctl_name = DEV_RAID,
+ .procname = "raid",
+ .maxlen = 0,
+ .mode = 0555,
+ .child = raid_table,
+ },
+ { .ctl_name = 0 }
+};
+
+static ctl_table raid_root_table[] = {
+ {
+ .ctl_name = CTL_DEV,
+ .procname = "dev",
+ .maxlen = 0,
+ .mode = 0555,
+ .child = raid_dir_table,
+ },
+ { .ctl_name = 0 }
+};
+
+static struct block_device_operations md_fops;
+
+/*
+ * Enables to iterate over all existing md arrays
+ * all_mddevs_lock protects this list.
+ */
+static LIST_HEAD(all_mddevs);
+static DEFINE_SPINLOCK(all_mddevs_lock);
+
+
+/*
+ * iterates through all used mddevs in the system.
+ * We take care to grab the all_mddevs_lock whenever navigating
+ * the list, and to always hold a refcount when unlocked.
+ * Any code which breaks out of this loop while own
+ * a reference to the current mddev and must mddev_put it.
+ */
+#define ITERATE_MDDEV(mddev,tmp) \
+ \
+ for (({ spin_lock(&all_mddevs_lock); \
+ tmp = all_mddevs.next; \
+ mddev = NULL;}); \
+ ({ if (tmp != &all_mddevs) \
+ mddev_get(list_entry(tmp, mddev_t, all_mddevs));\
+ spin_unlock(&all_mddevs_lock); \
+ if (mddev) mddev_put(mddev); \
+ mddev = list_entry(tmp, mddev_t, all_mddevs); \
+ tmp != &all_mddevs;}); \
+ ({ spin_lock(&all_mddevs_lock); \
+ tmp = tmp->next;}) \
+ )
+
+
+static int md_fail_request (request_queue_t *q, struct bio *bio)
+{
+ bio_io_error(bio, bio->bi_size);
+ return 0;
+}
+
+static inline mddev_t *mddev_get(mddev_t *mddev)
+{
+ atomic_inc(&mddev->active);
+ return mddev;
+}
+
+static void mddev_put(mddev_t *mddev)
+{
+ if (!atomic_dec_and_lock(&mddev->active, &all_mddevs_lock))
+ return;
+ if (!mddev->raid_disks && list_empty(&mddev->disks)) {
+ list_del(&mddev->all_mddevs);
+ blk_put_queue(mddev->queue);
+ kfree(mddev);
+ }
+ spin_unlock(&all_mddevs_lock);
+}
+
+static mddev_t * mddev_find(dev_t unit)
+{
+ mddev_t *mddev, *new = NULL;
+
+ retry:
+ spin_lock(&all_mddevs_lock);
+ list_for_each_entry(mddev, &all_mddevs, all_mddevs)
+ if (mddev->unit == unit) {
+ mddev_get(mddev);
+ spin_unlock(&all_mddevs_lock);
+ if (new)
+ kfree(new);
+ return mddev;
+ }
+
+ if (new) {
+ list_add(&new->all_mddevs, &all_mddevs);
+ spin_unlock(&all_mddevs_lock);
+ return new;
+ }
+ spin_unlock(&all_mddevs_lock);
+
+ new = (mddev_t *) kmalloc(sizeof(*new), GFP_KERNEL);
+ if (!new)
+ return NULL;
+
+ memset(new, 0, sizeof(*new));
+
+ new->unit = unit;
+ if (MAJOR(unit) == MD_MAJOR)
+ new->md_minor = MINOR(unit);
+ else
+ new->md_minor = MINOR(unit) >> MdpMinorShift;
+
+ init_MUTEX(&new->reconfig_sem);
+ INIT_LIST_HEAD(&new->disks);
+ INIT_LIST_HEAD(&new->all_mddevs);
+ init_timer(&new->safemode_timer);
+ atomic_set(&new->active, 1);
+
+ new->queue = blk_alloc_queue(GFP_KERNEL);
+ if (!new->queue) {
+ kfree(new);
+ return NULL;
+ }
+
+ blk_queue_make_request(new->queue, md_fail_request);
+
+ goto retry;
+}
+
+static inline int mddev_lock(mddev_t * mddev)
+{
+ return down_interruptible(&mddev->reconfig_sem);
+}
+
+static inline void mddev_lock_uninterruptible(mddev_t * mddev)
+{
+ down(&mddev->reconfig_sem);
+}
+
+static inline int mddev_trylock(mddev_t * mddev)
+{
+ return down_trylock(&mddev->reconfig_sem);
+}
+
+static inline void mddev_unlock(mddev_t * mddev)
+{
+ up(&mddev->reconfig_sem);
+
+ if (mddev->thread)
+ md_wakeup_thread(mddev->thread);
+}
+
+mdk_rdev_t * find_rdev_nr(mddev_t *mddev, int nr)
+{
+ mdk_rdev_t * rdev;
+ struct list_head *tmp;
+
+ ITERATE_RDEV(mddev,rdev,tmp) {
+ if (rdev->desc_nr == nr)
+ return rdev;
+ }
+ return NULL;
+}
+
+static mdk_rdev_t * find_rdev(mddev_t * mddev, dev_t dev)
+{
+ struct list_head *tmp;
+ mdk_rdev_t *rdev;
+
+ ITERATE_RDEV(mddev,rdev,tmp) {
+ if (rdev->bdev->bd_dev == dev)
+ return rdev;
+ }
+ return NULL;
+}
+
+inline static sector_t calc_dev_sboffset(struct block_device *bdev)
+{
+ sector_t size = bdev->bd_inode->i_size >> BLOCK_SIZE_BITS;
+ return MD_NEW_SIZE_BLOCKS(size);
+}
+
+static sector_t calc_dev_size(mdk_rdev_t *rdev, unsigned chunk_size)
+{
+ sector_t size;
+
+ size = rdev->sb_offset;
+
+ if (chunk_size)
+ size &= ~((sector_t)chunk_size/1024 - 1);
+ return size;
+}
+
+static int alloc_disk_sb(mdk_rdev_t * rdev)
+{
+ if (rdev->sb_page)
+ MD_BUG();
+
+ rdev->sb_page = alloc_page(GFP_KERNEL);
+ if (!rdev->sb_page) {
+ printk(KERN_ALERT "md: out of memory.\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static void free_disk_sb(mdk_rdev_t * rdev)
+{
+ if (rdev->sb_page) {
+ page_cache_release(rdev->sb_page);
+ rdev->sb_loaded = 0;
+ rdev->sb_page = NULL;
+ rdev->sb_offset = 0;
+ rdev->size = 0;
+ }
+}
+
+
+static int bi_complete(struct bio *bio, unsigned int bytes_done, int error)
+{
+ if (bio->bi_size)
+ return 1;
+
+ complete((struct completion*)bio->bi_private);
+ return 0;
+}
+
+static int sync_page_io(struct block_device *bdev, sector_t sector, int size,
+ struct page *page, int rw)
+{
+ struct bio *bio = bio_alloc(GFP_KERNEL, 1);
+ struct completion event;
+ int ret;
+
+ rw |= (1 << BIO_RW_SYNC);
+
+ bio->bi_bdev = bdev;
+ bio->bi_sector = sector;
+ bio_add_page(bio, page, size, 0);
+ init_completion(&event);
+ bio->bi_private = &event;
+ bio->bi_end_io = bi_complete;
+ submit_bio(rw, bio);
+ wait_for_completion(&event);
+
+ ret = test_bit(BIO_UPTODATE, &bio->bi_flags);
+ bio_put(bio);
+ return ret;
+}
+
+static int read_disk_sb(mdk_rdev_t * rdev)
+{
+ char b[BDEVNAME_SIZE];
+ if (!rdev->sb_page) {
+ MD_BUG();
+ return -EINVAL;
+ }
+ if (rdev->sb_loaded)
+ return 0;
+
+
+ if (!sync_page_io(rdev->bdev, rdev->sb_offset<<1, MD_SB_BYTES, rdev->sb_page, READ))
+ goto fail;
+ rdev->sb_loaded = 1;
+ return 0;
+
+fail:
+ printk(KERN_WARNING "md: disabled device %s, could not read superblock.\n",
+ bdevname(rdev->bdev,b));
+ return -EINVAL;
+}
+
+static int uuid_equal(mdp_super_t *sb1, mdp_super_t *sb2)
+{
+ if ( (sb1->set_uuid0 == sb2->set_uuid0) &&
+ (sb1->set_uuid1 == sb2->set_uuid1) &&
+ (sb1->set_uuid2 == sb2->set_uuid2) &&
+ (sb1->set_uuid3 == sb2->set_uuid3))
+
+ return 1;
+
+ return 0;
+}
+
+
+static int sb_equal(mdp_super_t *sb1, mdp_super_t *sb2)
+{
+ int ret;
+ mdp_super_t *tmp1, *tmp2;
+
+ tmp1 = kmalloc(sizeof(*tmp1),GFP_KERNEL);
+ tmp2 = kmalloc(sizeof(*tmp2),GFP_KERNEL);
+
+ if (!tmp1 || !tmp2) {
+ ret = 0;
+ printk(KERN_INFO "md.c: sb1 is not equal to sb2!\n");
+ goto abort;
+ }
+
+ *tmp1 = *sb1;
+ *tmp2 = *sb2;
+
+ /*
+ * nr_disks is not constant
+ */
+ tmp1->nr_disks = 0;
+ tmp2->nr_disks = 0;
+
+ if (memcmp(tmp1, tmp2, MD_SB_GENERIC_CONSTANT_WORDS * 4))
+ ret = 0;
+ else
+ ret = 1;
+
+abort:
+ if (tmp1)
+ kfree(tmp1);
+ if (tmp2)
+ kfree(tmp2);
+
+ return ret;
+}
+
+static unsigned int calc_sb_csum(mdp_super_t * sb)
+{
+ unsigned int disk_csum, csum;
+
+ disk_csum = sb->sb_csum;
+ sb->sb_csum = 0;
+ csum = csum_partial((void *)sb, MD_SB_BYTES, 0);
+ sb->sb_csum = disk_csum;
+ return csum;
+}
+
+
+/*
+ * Handle superblock details.
+ * We want to be able to handle multiple superblock formats
+ * so we have a common interface to them all, and an array of
+ * different handlers.
+ * We rely on user-space to write the initial superblock, and support
+ * reading and updating of superblocks.
+ * Interface methods are:
+ * int load_super(mdk_rdev_t *dev, mdk_rdev_t *refdev, int minor_version)
+ * loads and validates a superblock on dev.
+ * if refdev != NULL, compare superblocks on both devices
+ * Return:
+ * 0 - dev has a superblock that is compatible with refdev
+ * 1 - dev has a superblock that is compatible and newer than refdev
+ * so dev should be used as the refdev in future
+ * -EINVAL superblock incompatible or invalid
+ * -othererror e.g. -EIO
+ *
+ * int validate_super(mddev_t *mddev, mdk_rdev_t *dev)
+ * Verify that dev is acceptable into mddev.
+ * The first time, mddev->raid_disks will be 0, and data from
+ * dev should be merged in. Subsequent calls check that dev
+ * is new enough. Return 0 or -EINVAL
+ *
+ * void sync_super(mddev_t *mddev, mdk_rdev_t *dev)
+ * Update the superblock for rdev with data in mddev
+ * This does not write to disc.
+ *
+ */
+
+struct super_type {
+ char *name;
+ struct module *owner;
+ int (*load_super)(mdk_rdev_t *rdev, mdk_rdev_t *refdev, int minor_version);
+ int (*validate_super)(mddev_t *mddev, mdk_rdev_t *rdev);
+ void (*sync_super)(mddev_t *mddev, mdk_rdev_t *rdev);
+};
+
+/*
+ * load_super for 0.90.0
+ */
+static int super_90_load(mdk_rdev_t *rdev, mdk_rdev_t *refdev, int minor_version)
+{
+ char b[BDEVNAME_SIZE], b2[BDEVNAME_SIZE];
+ mdp_super_t *sb;
+ int ret;
+ sector_t sb_offset;
+
+ /*
+ * Calculate the position of the superblock,
+ * it's at the end of the disk.
+ *
+ * It also happens to be a multiple of 4Kb.
+ */
+ sb_offset = calc_dev_sboffset(rdev->bdev);
+ rdev->sb_offset = sb_offset;
+
+ ret = read_disk_sb(rdev);
+ if (ret) return ret;
+
+ ret = -EINVAL;
+
+ bdevname(rdev->bdev, b);
+ sb = (mdp_super_t*)page_address(rdev->sb_page);
+
+ if (sb->md_magic != MD_SB_MAGIC) {
+ printk(KERN_ERR "md: invalid raid superblock magic on %s\n",
+ b);
+ goto abort;
+ }
+
+ if (sb->major_version != 0 ||
+ sb->minor_version != 90) {
+ printk(KERN_WARNING "Bad version number %d.%d on %s\n",
+ sb->major_version, sb->minor_version,
+ b);
+ goto abort;
+ }
+
+ if (sb->raid_disks <= 0)
+ goto abort;
+
+ if (csum_fold(calc_sb_csum(sb)) != csum_fold(sb->sb_csum)) {
+ printk(KERN_WARNING "md: invalid superblock checksum on %s\n",
+ b);
+ goto abort;
+ }
+
+ rdev->preferred_minor = sb->md_minor;
+ rdev->data_offset = 0;
+
+ if (sb->level == LEVEL_MULTIPATH)
+ rdev->desc_nr = -1;
+ else
+ rdev->desc_nr = sb->this_disk.number;
+
+ if (refdev == 0)
+ ret = 1;
+ else {
+ __u64 ev1, ev2;
+ mdp_super_t *refsb = (mdp_super_t*)page_address(refdev->sb_page);
+ if (!uuid_equal(refsb, sb)) {
+ printk(KERN_WARNING "md: %s has different UUID to %s\n",
+ b, bdevname(refdev->bdev,b2));
+ goto abort;
+ }
+ if (!sb_equal(refsb, sb)) {
+ printk(KERN_WARNING "md: %s has same UUID"
+ " but different superblock to %s\n",
+ b, bdevname(refdev->bdev, b2));
+ goto abort;
+ }
+ ev1 = md_event(sb);
+ ev2 = md_event(refsb);
+ if (ev1 > ev2)
+ ret = 1;
+ else
+ ret = 0;
+ }
+ rdev->size = calc_dev_size(rdev, sb->chunk_size);
+
+ abort:
+ return ret;
+}
+
+/*
+ * validate_super for 0.90.0
+ */
+static int super_90_validate(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+ mdp_disk_t *desc;
+ mdp_super_t *sb = (mdp_super_t *)page_address(rdev->sb_page);
+
+ if (mddev->raid_disks == 0) {
+ mddev->major_version = 0;
+ mddev->minor_version = sb->minor_version;
+ mddev->patch_version = sb->patch_version;
+ mddev->persistent = ! sb->not_persistent;
+ mddev->chunk_size = sb->chunk_size;
+ mddev->ctime = sb->ctime;
+ mddev->utime = sb->utime;
+ mddev->level = sb->level;
+ mddev->layout = sb->layout;
+ mddev->raid_disks = sb->raid_disks;
+ mddev->size = sb->size;
+ mddev->events = md_event(sb);
+
+ if (sb->state & (1<<MD_SB_CLEAN))
+ mddev->recovery_cp = MaxSector;
+ else {
+ if (sb->events_hi == sb->cp_events_hi &&
+ sb->events_lo == sb->cp_events_lo) {
+ mddev->recovery_cp = sb->recovery_cp;
+ } else
+ mddev->recovery_cp = 0;
+ }
+
+ memcpy(mddev->uuid+0, &sb->set_uuid0, 4);
+ memcpy(mddev->uuid+4, &sb->set_uuid1, 4);
+ memcpy(mddev->uuid+8, &sb->set_uuid2, 4);
+ memcpy(mddev->uuid+12,&sb->set_uuid3, 4);
+
+ mddev->max_disks = MD_SB_DISKS;
+ } else {
+ __u64 ev1;
+ ev1 = md_event(sb);
+ ++ev1;
+ if (ev1 < mddev->events)
+ return -EINVAL;
+ }
+ if (mddev->level != LEVEL_MULTIPATH) {
+ rdev->raid_disk = -1;
+ rdev->in_sync = rdev->faulty = 0;
+ desc = sb->disks + rdev->desc_nr;
+
+ if (desc->state & (1<<MD_DISK_FAULTY))
+ rdev->faulty = 1;
+ else if (desc->state & (1<<MD_DISK_SYNC) &&
+ desc->raid_disk < mddev->raid_disks) {
+ rdev->in_sync = 1;
+ rdev->raid_disk = desc->raid_disk;
+ }
+ }
+ return 0;
+}
+
+/*
+ * sync_super for 0.90.0
+ */
+static void super_90_sync(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+ mdp_super_t *sb;
+ struct list_head *tmp;
+ mdk_rdev_t *rdev2;
+ int next_spare = mddev->raid_disks;
+
+ /* make rdev->sb match mddev data..
+ *
+ * 1/ zero out disks
+ * 2/ Add info for each disk, keeping track of highest desc_nr (next_spare);
+ * 3/ any empty disks < next_spare become removed
+ *
+ * disks[0] gets initialised to REMOVED because
+ * we cannot be sure from other fields if it has
+ * been initialised or not.
+ */
+ int i;
+ int active=0, working=0,failed=0,spare=0,nr_disks=0;
+
+ sb = (mdp_super_t*)page_address(rdev->sb_page);
+
+ memset(sb, 0, sizeof(*sb));
+
+ sb->md_magic = MD_SB_MAGIC;
+ sb->major_version = mddev->major_version;
+ sb->minor_version = mddev->minor_version;
+ sb->patch_version = mddev->patch_version;
+ sb->gvalid_words = 0; /* ignored */
+ memcpy(&sb->set_uuid0, mddev->uuid+0, 4);
+ memcpy(&sb->set_uuid1, mddev->uuid+4, 4);
+ memcpy(&sb->set_uuid2, mddev->uuid+8, 4);
+ memcpy(&sb->set_uuid3, mddev->uuid+12,4);
+
+ sb->ctime = mddev->ctime;
+ sb->level = mddev->level;
+ sb->size = mddev->size;
+ sb->raid_disks = mddev->raid_disks;
+ sb->md_minor = mddev->md_minor;
+ sb->not_persistent = !mddev->persistent;
+ sb->utime = mddev->utime;
+ sb->state = 0;
+ sb->events_hi = (mddev->events>>32);
+ sb->events_lo = (u32)mddev->events;
+
+ if (mddev->in_sync)
+ {
+ sb->recovery_cp = mddev->recovery_cp;
+ sb->cp_events_hi = (mddev->events>>32);
+ sb->cp_events_lo = (u32)mddev->events;
+ if (mddev->recovery_cp == MaxSector)
+ sb->state = (1<< MD_SB_CLEAN);
+ } else
+ sb->recovery_cp = 0;
+
+ sb->layout = mddev->layout;
+ sb->chunk_size = mddev->chunk_size;
+
+ sb->disks[0].state = (1<<MD_DISK_REMOVED);
+ ITERATE_RDEV(mddev,rdev2,tmp) {
+ mdp_disk_t *d;
+ if (rdev2->raid_disk >= 0 && rdev2->in_sync && !rdev2->faulty)
+ rdev2->desc_nr = rdev2->raid_disk;
+ else
+ rdev2->desc_nr = next_spare++;
+ d = &sb->disks[rdev2->desc_nr];
+ nr_disks++;
+ d->number = rdev2->desc_nr;
+ d->major = MAJOR(rdev2->bdev->bd_dev);
+ d->minor = MINOR(rdev2->bdev->bd_dev);
+ if (rdev2->raid_disk >= 0 && rdev->in_sync && !rdev2->faulty)
+ d->raid_disk = rdev2->raid_disk;
+ else
+ d->raid_disk = rdev2->desc_nr; /* compatibility */
+ if (rdev2->faulty) {
+ d->state = (1<<MD_DISK_FAULTY);
+ failed++;
+ } else if (rdev2->in_sync) {
+ d->state = (1<<MD_DISK_ACTIVE);
+ d->state |= (1<<MD_DISK_SYNC);
+ active++;
+ working++;
+ } else {
+ d->state = 0;
+ spare++;
+ working++;
+ }
+ }
+
+ /* now set the "removed" and "faulty" bits on any missing devices */
+ for (i=0 ; i < mddev->raid_disks ; i++) {
+ mdp_disk_t *d = &sb->disks[i];
+ if (d->state == 0 && d->number == 0) {
+ d->number = i;
+ d->raid_disk = i;
+ d->state = (1<<MD_DISK_REMOVED);
+ d->state |= (1<<MD_DISK_FAULTY);
+ failed++;
+ }
+ }
+ sb->nr_disks = nr_disks;
+ sb->active_disks = active;
+ sb->working_disks = working;
+ sb->failed_disks = failed;
+ sb->spare_disks = spare;
+
+ sb->this_disk = sb->disks[rdev->desc_nr];
+ sb->sb_csum = calc_sb_csum(sb);
+}
+
+/*
+ * version 1 superblock
+ */
+
+static unsigned int calc_sb_1_csum(struct mdp_superblock_1 * sb)
+{
+ unsigned int disk_csum, csum;
+ unsigned long long newcsum;
+ int size = 256 + le32_to_cpu(sb->max_dev)*2;
+ unsigned int *isuper = (unsigned int*)sb;
+ int i;
+
+ disk_csum = sb->sb_csum;
+ sb->sb_csum = 0;
+ newcsum = 0;
+ for (i=0; size>=4; size -= 4 )
+ newcsum += le32_to_cpu(*isuper++);
+
+ if (size == 2)
+ newcsum += le16_to_cpu(*(unsigned short*) isuper);
+
+ csum = (newcsum & 0xffffffff) + (newcsum >> 32);
+ sb->sb_csum = disk_csum;
+ return cpu_to_le32(csum);
+}
+
+static int super_1_load(mdk_rdev_t *rdev, mdk_rdev_t *refdev, int minor_version)
+{
+ struct mdp_superblock_1 *sb;
+ int ret;
+ sector_t sb_offset;
+ char b[BDEVNAME_SIZE], b2[BDEVNAME_SIZE];
+
+ /*
+ * Calculate the position of the superblock.
+ * It is always aligned to a 4K boundary and
+ * depeding on minor_version, it can be:
+ * 0: At least 8K, but less than 12K, from end of device
+ * 1: At start of device
+ * 2: 4K from start of device.
+ */
+ switch(minor_version) {
+ case 0:
+ sb_offset = rdev->bdev->bd_inode->i_size >> 9;
+ sb_offset -= 8*2;
+ sb_offset &= ~(4*2-1);
+ /* convert from sectors to K */
+ sb_offset /= 2;
+ break;
+ case 1:
+ sb_offset = 0;
+ break;
+ case 2:
+ sb_offset = 4;
+ break;
+ default:
+ return -EINVAL;
+ }
+ rdev->sb_offset = sb_offset;
+
+ ret = read_disk_sb(rdev);
+ if (ret) return ret;
+
+
+ sb = (struct mdp_superblock_1*)page_address(rdev->sb_page);
+
+ if (sb->magic != cpu_to_le32(MD_SB_MAGIC) ||
+ sb->major_version != cpu_to_le32(1) ||
+ le32_to_cpu(sb->max_dev) > (4096-256)/2 ||
+ le64_to_cpu(sb->super_offset) != (rdev->sb_offset<<1) ||
+ sb->feature_map != 0)
+ return -EINVAL;
+
+ if (calc_sb_1_csum(sb) != sb->sb_csum) {
+ printk("md: invalid superblock checksum on %s\n",
+ bdevname(rdev->bdev,b));
+ return -EINVAL;
+ }
+ if (le64_to_cpu(sb->data_size) < 10) {
+ printk("md: data_size too small on %s\n",
+ bdevname(rdev->bdev,b));
+ return -EINVAL;
+ }
+ rdev->preferred_minor = 0xffff;
+ rdev->data_offset = le64_to_cpu(sb->data_offset);
+
+ if (refdev == 0)
+ return 1;
+ else {
+ __u64 ev1, ev2;
+ struct mdp_superblock_1 *refsb =
+ (struct mdp_superblock_1*)page_address(refdev->sb_page);
+
+ if (memcmp(sb->set_uuid, refsb->set_uuid, 16) != 0 ||
+ sb->level != refsb->level ||
+ sb->layout != refsb->layout ||
+ sb->chunksize != refsb->chunksize) {
+ printk(KERN_WARNING "md: %s has strangely different"
+ " superblock to %s\n",
+ bdevname(rdev->bdev,b),
+ bdevname(refdev->bdev,b2));
+ return -EINVAL;
+ }
+ ev1 = le64_to_cpu(sb->events);
+ ev2 = le64_to_cpu(refsb->events);
+
+ if (ev1 > ev2)
+ return 1;
+ }
+ if (minor_version)
+ rdev->size = ((rdev->bdev->bd_inode->i_size>>9) - le64_to_cpu(sb->data_offset)) / 2;
+ else
+ rdev->size = rdev->sb_offset;
+ if (rdev->size < le64_to_cpu(sb->data_size)/2)
+ return -EINVAL;
+ rdev->size = le64_to_cpu(sb->data_size)/2;
+ if (le32_to_cpu(sb->chunksize))
+ rdev->size &= ~((sector_t)le32_to_cpu(sb->chunksize)/2 - 1);
+ return 0;
+}
+
+static int super_1_validate(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+ struct mdp_superblock_1 *sb = (struct mdp_superblock_1*)page_address(rdev->sb_page);
+
+ if (mddev->raid_disks == 0) {
+ mddev->major_version = 1;
+ mddev->patch_version = 0;
+ mddev->persistent = 1;
+ mddev->chunk_size = le32_to_cpu(sb->chunksize) << 9;
+ mddev->ctime = le64_to_cpu(sb->ctime) & ((1ULL << 32)-1);
+ mddev->utime = le64_to_cpu(sb->utime) & ((1ULL << 32)-1);
+ mddev->level = le32_to_cpu(sb->level);
+ mddev->layout = le32_to_cpu(sb->layout);
+ mddev->raid_disks = le32_to_cpu(sb->raid_disks);
+ mddev->size = le64_to_cpu(sb->size)/2;
+ mddev->events = le64_to_cpu(sb->events);
+
+ mddev->recovery_cp = le64_to_cpu(sb->resync_offset);
+ memcpy(mddev->uuid, sb->set_uuid, 16);
+
+ mddev->max_disks = (4096-256)/2;
+ } else {
+ __u64 ev1;
+ ev1 = le64_to_cpu(sb->events);
+ ++ev1;
+ if (ev1 < mddev->events)
+ return -EINVAL;
+ }
+
+ if (mddev->level != LEVEL_MULTIPATH) {
+ int role;
+ rdev->desc_nr = le32_to_cpu(sb->dev_number);
+ role = le16_to_cpu(sb->dev_roles[rdev->desc_nr]);
+ switch(role) {
+ case 0xffff: /* spare */
+ rdev->in_sync = 0;
+ rdev->faulty = 0;
+ rdev->raid_disk = -1;
+ break;
+ case 0xfffe: /* faulty */
+ rdev->in_sync = 0;
+ rdev->faulty = 1;
+ rdev->raid_disk = -1;
+ break;
+ default:
+ rdev->in_sync = 1;
+ rdev->faulty = 0;
+ rdev->raid_disk = role;
+ break;
+ }
+ }
+ return 0;
+}
+
+static void super_1_sync(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+ struct mdp_superblock_1 *sb;
+ struct list_head *tmp;
+ mdk_rdev_t *rdev2;
+ int max_dev, i;
+ /* make rdev->sb match mddev and rdev data. */
+
+ sb = (struct mdp_superblock_1*)page_address(rdev->sb_page);
+
+ sb->feature_map = 0;
+ sb->pad0 = 0;
+ memset(sb->pad1, 0, sizeof(sb->pad1));
+ memset(sb->pad2, 0, sizeof(sb->pad2));
+ memset(sb->pad3, 0, sizeof(sb->pad3));
+
+ sb->utime = cpu_to_le64((__u64)mddev->utime);
+ sb->events = cpu_to_le64(mddev->events);
+ if (mddev->in_sync)
+ sb->resync_offset = cpu_to_le64(mddev->recovery_cp);
+ else
+ sb->resync_offset = cpu_to_le64(0);
+
+ max_dev = 0;
+ ITERATE_RDEV(mddev,rdev2,tmp)
+ if (rdev2->desc_nr+1 > max_dev)
+ max_dev = rdev2->desc_nr+1;
+
+ sb->max_dev = cpu_to_le32(max_dev);
+ for (i=0; i<max_dev;i++)
+ sb->dev_roles[i] = cpu_to_le16(0xfffe);
+
+ ITERATE_RDEV(mddev,rdev2,tmp) {
+ i = rdev2->desc_nr;
+ if (rdev2->faulty)
+ sb->dev_roles[i] = cpu_to_le16(0xfffe);
+ else if (rdev2->in_sync)
+ sb->dev_roles[i] = cpu_to_le16(rdev2->raid_disk);
+ else
+ sb->dev_roles[i] = cpu_to_le16(0xffff);
+ }
+
+ sb->recovery_offset = cpu_to_le64(0); /* not supported yet */
+ sb->sb_csum = calc_sb_1_csum(sb);
+}
+
+
+struct super_type super_types[] = {
+ [0] = {
+ .name = "0.90.0",
+ .owner = THIS_MODULE,
+ .load_super = super_90_load,
+ .validate_super = super_90_validate,
+ .sync_super = super_90_sync,
+ },
+ [1] = {
+ .name = "md-1",
+ .owner = THIS_MODULE,
+ .load_super = super_1_load,
+ .validate_super = super_1_validate,
+ .sync_super = super_1_sync,
+ },
+};
+
+static mdk_rdev_t * match_dev_unit(mddev_t *mddev, mdk_rdev_t *dev)
+{
+ struct list_head *tmp;
+ mdk_rdev_t *rdev;
+
+ ITERATE_RDEV(mddev,rdev,tmp)
+ if (rdev->bdev->bd_contains == dev->bdev->bd_contains)
+ return rdev;
+
+ return NULL;
+}
+
+static int match_mddev_units(mddev_t *mddev1, mddev_t *mddev2)
+{
+ struct list_head *tmp;
+ mdk_rdev_t *rdev;
+
+ ITERATE_RDEV(mddev1,rdev,tmp)
+ if (match_dev_unit(mddev2, rdev))
+ return 1;
+
+ return 0;
+}
+
+static LIST_HEAD(pending_raid_disks);
+
+static int bind_rdev_to_array(mdk_rdev_t * rdev, mddev_t * mddev)
+{
+ mdk_rdev_t *same_pdev;
+ char b[BDEVNAME_SIZE], b2[BDEVNAME_SIZE];
+
+ if (rdev->mddev) {
+ MD_BUG();
+ return -EINVAL;
+ }
+ same_pdev = match_dev_unit(mddev, rdev);
+ if (same_pdev)
+ printk(KERN_WARNING
+ "%s: WARNING: %s appears to be on the same physical"
+ " disk as %s. True\n protection against single-disk"
+ " failure might be compromised.\n",
+ mdname(mddev), bdevname(rdev->bdev,b),
+ bdevname(same_pdev->bdev,b2));
+
+ /* Verify rdev->desc_nr is unique.
+ * If it is -1, assign a free number, else
+ * check number is not in use
+ */
+ if (rdev->desc_nr < 0) {
+ int choice = 0;
+ if (mddev->pers) choice = mddev->raid_disks;
+ while (find_rdev_nr(mddev, choice))
+ choice++;
+ rdev->desc_nr = choice;
+ } else {
+ if (find_rdev_nr(mddev, rdev->desc_nr))
+ return -EBUSY;
+ }
+
+ list_add(&rdev->same_set, &mddev->disks);
+ rdev->mddev = mddev;
+ printk(KERN_INFO "md: bind<%s>\n", bdevname(rdev->bdev,b));
+ return 0;
+}
+
+static void unbind_rdev_from_array(mdk_rdev_t * rdev)
+{
+ char b[BDEVNAME_SIZE];
+ if (!rdev->mddev) {
+ MD_BUG();
+ return;
+ }
+ list_del_init(&rdev->same_set);
+ printk(KERN_INFO "md: unbind<%s>\n", bdevname(rdev->bdev,b));
+ rdev->mddev = NULL;
+}
+
+/*
+ * prevent the device from being mounted, repartitioned or
+ * otherwise reused by a RAID array (or any other kernel
+ * subsystem), by bd_claiming the device.
+ */
+static int lock_rdev(mdk_rdev_t *rdev, dev_t dev)
+{
+ int err = 0;
+ struct block_device *bdev;
+ char b[BDEVNAME_SIZE];
+
+ bdev = open_by_devnum(dev, FMODE_READ|FMODE_WRITE);
+ if (IS_ERR(bdev)) {
+ printk(KERN_ERR "md: could not open %s.\n",
+ __bdevname(dev, b));
+ return PTR_ERR(bdev);
+ }
+ err = bd_claim(bdev, rdev);
+ if (err) {
+ printk(KERN_ERR "md: could not bd_claim %s.\n",
+ bdevname(bdev, b));
+ blkdev_put(bdev);
+ return err;
+ }
+ rdev->bdev = bdev;
+ return err;
+}
+
+static void unlock_rdev(mdk_rdev_t *rdev)
+{
+ struct block_device *bdev = rdev->bdev;
+ rdev->bdev = NULL;
+ if (!bdev)
+ MD_BUG();
+ bd_release(bdev);
+ blkdev_put(bdev);
+}
+
+void md_autodetect_dev(dev_t dev);
+
+static void export_rdev(mdk_rdev_t * rdev)
+{
+ char b[BDEVNAME_SIZE];
+ printk(KERN_INFO "md: export_rdev(%s)\n",
+ bdevname(rdev->bdev,b));
+ if (rdev->mddev)
+ MD_BUG();
+ free_disk_sb(rdev);
+ list_del_init(&rdev->same_set);
+#ifndef MODULE
+ md_autodetect_dev(rdev->bdev->bd_dev);
+#endif
+ unlock_rdev(rdev);
+ kfree(rdev);
+}
+
+static void kick_rdev_from_array(mdk_rdev_t * rdev)
+{
+ unbind_rdev_from_array(rdev);
+ export_rdev(rdev);
+}
+
+static void export_array(mddev_t *mddev)
+{
+ struct list_head *tmp;
+ mdk_rdev_t *rdev;
+
+ ITERATE_RDEV(mddev,rdev,tmp) {
+ if (!rdev->mddev) {
+ MD_BUG();
+ continue;
+ }
+ kick_rdev_from_array(rdev);
+ }
+ if (!list_empty(&mddev->disks))
+ MD_BUG();
+ mddev->raid_disks = 0;
+ mddev->major_version = 0;
+}
+
+static void print_desc(mdp_disk_t *desc)
+{
+ printk(" DISK<N:%d,(%d,%d),R:%d,S:%d>\n", desc->number,
+ desc->major,desc->minor,desc->raid_disk,desc->state);
+}
+
+static void print_sb(mdp_super_t *sb)
+{
+ int i;
+
+ printk(KERN_INFO
+ "md: SB: (V:%d.%d.%d) ID:<%08x.%08x.%08x.%08x> CT:%08x\n",
+ sb->major_version, sb->minor_version, sb->patch_version,
+ sb->set_uuid0, sb->set_uuid1, sb->set_uuid2, sb->set_uuid3,
+ sb->ctime);
+ printk(KERN_INFO "md: L%d S%08d ND:%d RD:%d md%d LO:%d CS:%d\n",
+ sb->level, sb->size, sb->nr_disks, sb->raid_disks,
+ sb->md_minor, sb->layout, sb->chunk_size);
+ printk(KERN_INFO "md: UT:%08x ST:%d AD:%d WD:%d"
+ " FD:%d SD:%d CSUM:%08x E:%08lx\n",
+ sb->utime, sb->state, sb->active_disks, sb->working_disks,
+ sb->failed_disks, sb->spare_disks,
+ sb->sb_csum, (unsigned long)sb->events_lo);
+
+ printk(KERN_INFO);
+ for (i = 0; i < MD_SB_DISKS; i++) {
+ mdp_disk_t *desc;
+
+ desc = sb->disks + i;
+ if (desc->number || desc->major || desc->minor ||
+ desc->raid_disk || (desc->state && (desc->state != 4))) {
+ printk(" D %2d: ", i);
+ print_desc(desc);
+ }
+ }
+ printk(KERN_INFO "md: THIS: ");
+ print_desc(&sb->this_disk);
+
+}
+
+static void print_rdev(mdk_rdev_t *rdev)
+{
+ char b[BDEVNAME_SIZE];
+ printk(KERN_INFO "md: rdev %s, SZ:%08llu F:%d S:%d DN:%u\n",
+ bdevname(rdev->bdev,b), (unsigned long long)rdev->size,
+ rdev->faulty, rdev->in_sync, rdev->desc_nr);
+ if (rdev->sb_loaded) {
+ printk(KERN_INFO "md: rdev superblock:\n");
+ print_sb((mdp_super_t*)page_address(rdev->sb_page));
+ } else
+ printk(KERN_INFO "md: no rdev superblock!\n");
+}
+
+void md_print_devices(void)
+{
+ struct list_head *tmp, *tmp2;
+ mdk_rdev_t *rdev;
+ mddev_t *mddev;
+ char b[BDEVNAME_SIZE];
+
+ printk("\n");
+ printk("md: **********************************\n");
+ printk("md: * <COMPLETE RAID STATE PRINTOUT> *\n");
+ printk("md: **********************************\n");
+ ITERATE_MDDEV(mddev,tmp) {
+ printk("%s: ", mdname(mddev));
+
+ ITERATE_RDEV(mddev,rdev,tmp2)
+ printk("<%s>", bdevname(rdev->bdev,b));
+ printk("\n");
+
+ ITERATE_RDEV(mddev,rdev,tmp2)
+ print_rdev(rdev);
+ }
+ printk("md: **********************************\n");
+ printk("\n");
+}
+
+
+static int write_disk_sb(mdk_rdev_t * rdev)
+{
+ char b[BDEVNAME_SIZE];
+ if (!rdev->sb_loaded) {
+ MD_BUG();
+ return 1;
+ }
+ if (rdev->faulty) {
+ MD_BUG();
+ return 1;
+ }
+
+ dprintk(KERN_INFO "(write) %s's sb offset: %llu\n",
+ bdevname(rdev->bdev,b),
+ (unsigned long long)rdev->sb_offset);
+
+ if (sync_page_io(rdev->bdev, rdev->sb_offset<<1, MD_SB_BYTES, rdev->sb_page, WRITE))
+ return 0;
+
+ printk("md: write_disk_sb failed for device %s\n",
+ bdevname(rdev->bdev,b));
+ return 1;
+}
+
+static void sync_sbs(mddev_t * mddev)
+{
+ mdk_rdev_t *rdev;
+ struct list_head *tmp;
+
+ ITERATE_RDEV(mddev,rdev,tmp) {
+ super_types[mddev->major_version].
+ sync_super(mddev, rdev);
+ rdev->sb_loaded = 1;
+ }
+}
+
+static void md_update_sb(mddev_t * mddev)
+{
+ int err, count = 100;
+ struct list_head *tmp;
+ mdk_rdev_t *rdev;
+
+ mddev->sb_dirty = 0;
+repeat:
+ mddev->utime = get_seconds();
+ mddev->events ++;
+
+ if (!mddev->events) {
+ /*
+ * oops, this 64-bit counter should never wrap.
+ * Either we are in around ~1 trillion A.C., assuming
+ * 1 reboot per second, or we have a bug:
+ */
+ MD_BUG();
+ mddev->events --;
+ }
+ sync_sbs(mddev);
+
+ /*
+ * do not write anything to disk if using
+ * nonpersistent superblocks
+ */
+ if (!mddev->persistent)
+ return;
+
+ dprintk(KERN_INFO
+ "md: updating %s RAID superblock on device (in sync %d)\n",
+ mdname(mddev),mddev->in_sync);
+
+ err = 0;
+ ITERATE_RDEV(mddev,rdev,tmp) {
+ char b[BDEVNAME_SIZE];
+ dprintk(KERN_INFO "md: ");
+ if (rdev->faulty)
+ dprintk("(skipping faulty ");
+
+ dprintk("%s ", bdevname(rdev->bdev,b));
+ if (!rdev->faulty) {
+ err += write_disk_sb(rdev);
+ } else
+ dprintk(")\n");
+ if (!err && mddev->level == LEVEL_MULTIPATH)
+ /* only need to write one superblock... */
+ break;
+ }
+ if (err) {
+ if (--count) {
+ printk(KERN_ERR "md: errors occurred during superblock"
+ " update, repeating\n");
+ goto repeat;
+ }
+ printk(KERN_ERR \
+ "md: excessive errors occurred during superblock update, exiting\n");
+ }
+}
+
+/*
+ * Import a device. If 'super_format' >= 0, then sanity check the superblock
+ *
+ * mark the device faulty if:
+ *
+ * - the device is nonexistent (zero size)
+ * - the device has no valid superblock
+ *
+ * a faulty rdev _never_ has rdev->sb set.
+ */
+static mdk_rdev_t *md_import_device(dev_t newdev, int super_format, int super_minor)
+{
+ char b[BDEVNAME_SIZE];
+ int err;
+ mdk_rdev_t *rdev;
+ sector_t size;
+
+ rdev = (mdk_rdev_t *) kmalloc(sizeof(*rdev), GFP_KERNEL);
+ if (!rdev) {
+ printk(KERN_ERR "md: could not alloc mem for new device!\n");
+ return ERR_PTR(-ENOMEM);
+ }
+ memset(rdev, 0, sizeof(*rdev));
+
+ if ((err = alloc_disk_sb(rdev)))
+ goto abort_free;
+
+ err = lock_rdev(rdev, newdev);
+ if (err)
+ goto abort_free;
+
+ rdev->desc_nr = -1;
+ rdev->faulty = 0;
+ rdev->in_sync = 0;
+ rdev->data_offset = 0;
+ atomic_set(&rdev->nr_pending, 0);
+
+ size = rdev->bdev->bd_inode->i_size >> BLOCK_SIZE_BITS;
+ if (!size) {
+ printk(KERN_WARNING
+ "md: %s has zero or unknown size, marking faulty!\n",
+ bdevname(rdev->bdev,b));
+ err = -EINVAL;
+ goto abort_free;
+ }
+
+ if (super_format >= 0) {
+ err = super_types[super_format].
+ load_super(rdev, NULL, super_minor);
+ if (err == -EINVAL) {
+ printk(KERN_WARNING
+ "md: %s has invalid sb, not importing!\n",
+ bdevname(rdev->bdev,b));
+ goto abort_free;
+ }
+ if (err < 0) {
+ printk(KERN_WARNING
+ "md: could not read %s's sb, not importing!\n",
+ bdevname(rdev->bdev,b));
+ goto abort_free;
+ }
+ }
+ INIT_LIST_HEAD(&rdev->same_set);
+
+ return rdev;
+
+abort_free:
+ if (rdev->sb_page) {
+ if (rdev->bdev)
+ unlock_rdev(rdev);
+ free_disk_sb(rdev);
+ }
+ kfree(rdev);
+ return ERR_PTR(err);
+}
+
+/*
+ * Check a full RAID array for plausibility
+ */
+
+
+static int analyze_sbs(mddev_t * mddev)
+{
+ int i;
+ struct list_head *tmp;
+ mdk_rdev_t *rdev, *freshest;
+ char b[BDEVNAME_SIZE];
+
+ freshest = NULL;
+ ITERATE_RDEV(mddev,rdev,tmp)
+ switch (super_types[mddev->major_version].
+ load_super(rdev, freshest, mddev->minor_version)) {
+ case 1:
+ freshest = rdev;
+ break;
+ case 0:
+ break;
+ default:
+ printk( KERN_ERR \
+ "md: fatal superblock inconsistency in %s"
+ " -- removing from array\n",
+ bdevname(rdev->bdev,b));
+ kick_rdev_from_array(rdev);
+ }
+
+
+ super_types[mddev->major_version].
+ validate_super(mddev, freshest);
+
+ i = 0;
+ ITERATE_RDEV(mddev,rdev,tmp) {
+ if (rdev != freshest)
+ if (super_types[mddev->major_version].
+ validate_super(mddev, rdev)) {
+ printk(KERN_WARNING "md: kicking non-fresh %s"
+ " from array!\n",
+ bdevname(rdev->bdev,b));
+ kick_rdev_from_array(rdev);
+ continue;
+ }
+ if (mddev->level == LEVEL_MULTIPATH) {
+ rdev->desc_nr = i++;
+ rdev->raid_disk = rdev->desc_nr;
+ rdev->in_sync = 1;
+ }
+ }
+
+
+
+ if (mddev->recovery_cp != MaxSector &&
+ mddev->level >= 1)
+ printk(KERN_ERR "md: %s: raid array is not clean"
+ " -- starting background reconstruction\n",
+ mdname(mddev));
+
+ return 0;
+}
+
+int mdp_major = 0;
+
+static struct kobject *md_probe(dev_t dev, int *part, void *data)
+{
+ static DECLARE_MUTEX(disks_sem);
+ mddev_t *mddev = mddev_find(dev);
+ struct gendisk *disk;
+ int partitioned = (MAJOR(dev) != MD_MAJOR);
+ int shift = partitioned ? MdpMinorShift : 0;
+ int unit = MINOR(dev) >> shift;
+
+ if (!mddev)
+ return NULL;
+
+ down(&disks_sem);
+ if (mddev->gendisk) {
+ up(&disks_sem);
+ mddev_put(mddev);
+ return NULL;
+ }
+ disk = alloc_disk(1 << shift);
+ if (!disk) {
+ up(&disks_sem);
+ mddev_put(mddev);
+ return NULL;
+ }
+ disk->major = MAJOR(dev);
+ disk->first_minor = unit << shift;
+ if (partitioned) {
+ sprintf(disk->disk_name, "md_d%d", unit);
+ sprintf(disk->devfs_name, "md/d%d", unit);
+ } else {
+ sprintf(disk->disk_name, "md%d", unit);
+ sprintf(disk->devfs_name, "md/%d", unit);
+ }
+ disk->fops = &md_fops;
+ disk->private_data = mddev;
+ disk->queue = mddev->queue;
+ add_disk(disk);
+ mddev->gendisk = disk;
+ up(&disks_sem);
+ return NULL;
+}
+
+void md_wakeup_thread(mdk_thread_t *thread);
+
+static void md_safemode_timeout(unsigned long data)
+{
+ mddev_t *mddev = (mddev_t *) data;
+
+ mddev->safemode = 1;
+ md_wakeup_thread(mddev->thread);
+}
+
+
+static int do_md_run(mddev_t * mddev)
+{
+ int pnum, err;
+ int chunk_size;
+ struct list_head *tmp;
+ mdk_rdev_t *rdev;
+ struct gendisk *disk;
+ char b[BDEVNAME_SIZE];
+
+ if (list_empty(&mddev->disks)) {
+ MD_BUG();
+ return -EINVAL;
+ }
+
+ if (mddev->pers)
+ return -EBUSY;
+
+ /*
+ * Analyze all RAID superblock(s)
+ */
+ if (!mddev->raid_disks && analyze_sbs(mddev)) {
+ MD_BUG();
+ return -EINVAL;
+ }
+
+ chunk_size = mddev->chunk_size;
+ pnum = level_to_pers(mddev->level);
+
+ if ((pnum != MULTIPATH) && (pnum != RAID1)) {
+ if (!chunk_size) {
+ /*
+ * 'default chunksize' in the old md code used to
+ * be PAGE_SIZE, baaad.
+ * we abort here to be on the safe side. We don't
+ * want to continue the bad practice.
+ */
+ printk(KERN_ERR
+ "no chunksize specified, see 'man raidtab'\n");
+ return -EINVAL;
+ }
+ if (chunk_size > MAX_CHUNK_SIZE) {
+ printk(KERN_ERR "too big chunk_size: %d > %d\n",
+ chunk_size, MAX_CHUNK_SIZE);
+ return -EINVAL;
+ }
+ /*
+ * chunk-size has to be a power of 2 and multiples of PAGE_SIZE
+ */
+ if ( (1 << ffz(~chunk_size)) != chunk_size) {
+ MD_BUG();
+ return -EINVAL;
+ }
+ if (chunk_size < PAGE_SIZE) {
+ printk(KERN_ERR "too small chunk_size: %d < %ld\n",
+ chunk_size, PAGE_SIZE);
+ return -EINVAL;
+ }
+
+ /* devices must have minimum size of one chunk */
+ ITERATE_RDEV(mddev,rdev,tmp) {
+ if (rdev->faulty)
+ continue;
+ if (rdev->size < chunk_size / 1024) {
+ printk(KERN_WARNING
+ "md: Dev %s smaller than chunk_size:"
+ " %lluk < %dk\n",
+ bdevname(rdev->bdev,b),
+ (unsigned long long)rdev->size,
+ chunk_size / 1024);
+ return -EINVAL;
+ }
+ }
+ }
+
+ if (pnum >= MAX_PERSONALITY) {
+ MD_BUG();
+ return -EINVAL;
+ }
+
+#ifdef CONFIG_KMOD
+ if (!pers[pnum])
+ {
+ request_module("md-personality-%d", pnum);
+ }
+#endif
+
+ /*
+ * Drop all container device buffers, from now on
+ * the only valid external interface is through the md
+ * device.
+ * Also find largest hardsector size
+ */
+ ITERATE_RDEV(mddev,rdev,tmp) {
+ if (rdev->faulty)
+ continue;
+ sync_blockdev(rdev->bdev);
+ invalidate_bdev(rdev->bdev, 0);
+ }
+
+ md_probe(mddev->unit, NULL, NULL);
+ disk = mddev->gendisk;
+ if (!disk)
+ return -ENOMEM;
+
+ spin_lock(&pers_lock);
+ if (!pers[pnum] || !try_module_get(pers[pnum]->owner)) {
+ spin_unlock(&pers_lock);
+ printk(KERN_WARNING "md: personality %d is not loaded!\n",
+ pnum);
+ return -EINVAL;
+ }
+
+ mddev->pers = pers[pnum];
+ spin_unlock(&pers_lock);
+
+ mddev->resync_max_sectors = mddev->size << 1; /* may be over-ridden by personality */
+
+ err = mddev->pers->run(mddev);
+ if (err) {
+ printk(KERN_ERR "md: pers->run() failed ...\n");
+ module_put(mddev->pers->owner);
+ mddev->pers = NULL;
+ return -EINVAL;
+ }
+ atomic_set(&mddev->writes_pending,0);
+ mddev->safemode = 0;
+ mddev->safemode_timer.function = md_safemode_timeout;
+ mddev->safemode_timer.data = (unsigned long) mddev;
+ mddev->safemode_delay = (20 * HZ)/1000 +1; /* 20 msec delay */
+ mddev->in_sync = 1;
+
+ set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+
+ if (mddev->sb_dirty)
+ md_update_sb(mddev);
+
+ set_capacity(disk, mddev->array_size<<1);
+
+ /* If we call blk_queue_make_request here, it will
+ * re-initialise max_sectors etc which may have been
+ * refined inside -> run. So just set the bits we need to set.
+ * Most initialisation happended when we called
+ * blk_queue_make_request(..., md_fail_request)
+ * earlier.
+ */
+ mddev->queue->queuedata = mddev;
+ mddev->queue->make_request_fn = mddev->pers->make_request;
+
+ mddev->changed = 1;
+ return 0;
+}
+
+static int restart_array(mddev_t *mddev)
+{
+ struct gendisk *disk = mddev->gendisk;
+ int err;
+
+ /*
+ * Complain if it has no devices
+ */
+ err = -ENXIO;
+ if (list_empty(&mddev->disks))
+ goto out;
+
+ if (mddev->pers) {
+ err = -EBUSY;
+ if (!mddev->ro)
+ goto out;
+
+ mddev->safemode = 0;
+ mddev->ro = 0;
+ set_disk_ro(disk, 0);
+
+ printk(KERN_INFO "md: %s switched to read-write mode.\n",
+ mdname(mddev));
+ /*
+ * Kick recovery or resync if necessary
+ */
+ set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+ md_wakeup_thread(mddev->thread);
+ err = 0;
+ } else {
+ printk(KERN_ERR "md: %s has no personality assigned.\n",
+ mdname(mddev));
+ err = -EINVAL;
+ }
+
+out:
+ return err;
+}
+
+static int do_md_stop(mddev_t * mddev, int ro)
+{
+ int err = 0;
+ struct gendisk *disk = mddev->gendisk;
+
+ if (mddev->pers) {
+ if (atomic_read(&mddev->active)>2) {
+ printk("md: %s still in use.\n",mdname(mddev));
+ return -EBUSY;
+ }
+
+ if (mddev->sync_thread) {
+ set_bit(MD_RECOVERY_INTR, &mddev->recovery);
+ md_unregister_thread(mddev->sync_thread);
+ mddev->sync_thread = NULL;
+ }
+
+ del_timer_sync(&mddev->safemode_timer);
+
+ invalidate_partition(disk, 0);
+
+ if (ro) {
+ err = -ENXIO;
+ if (mddev->ro)
+ goto out;
+ mddev->ro = 1;
+ } else {
+ if (mddev->ro)
+ set_disk_ro(disk, 0);
+ blk_queue_make_request(mddev->queue, md_fail_request);
+ mddev->pers->stop(mddev);
+ module_put(mddev->pers->owner);
+ mddev->pers = NULL;
+ if (mddev->ro)
+ mddev->ro = 0;
+ }
+ if (!mddev->in_sync) {
+ /* mark array as shutdown cleanly */
+ mddev->in_sync = 1;
+ md_update_sb(mddev);
+ }
+ if (ro)
+ set_disk_ro(disk, 1);
+ }
+ /*
+ * Free resources if final stop
+ */
+ if (!ro) {
+ struct gendisk *disk;
+ printk(KERN_INFO "md: %s stopped.\n", mdname(mddev));
+
+ export_array(mddev);
+
+ mddev->array_size = 0;
+ disk = mddev->gendisk;
+ if (disk)
+ set_capacity(disk, 0);
+ mddev->changed = 1;
+ } else
+ printk(KERN_INFO "md: %s switched to read-only mode.\n",
+ mdname(mddev));
+ err = 0;
+out:
+ return err;
+}
+
+static void autorun_array(mddev_t *mddev)
+{
+ mdk_rdev_t *rdev;
+ struct list_head *tmp;
+ int err;
+
+ if (list_empty(&mddev->disks)) {
+ MD_BUG();
+ return;
+ }
+
+ printk(KERN_INFO "md: running: ");
+
+ ITERATE_RDEV(mddev,rdev,tmp) {
+ char b[BDEVNAME_SIZE];
+ printk("<%s>", bdevname(rdev->bdev,b));
+ }
+ printk("\n");
+
+ err = do_md_run (mddev);
+ if (err) {
+ printk(KERN_WARNING "md: do_md_run() returned %d\n", err);
+ do_md_stop (mddev, 0);
+ }
+}
+
+/*
+ * lets try to run arrays based on all disks that have arrived
+ * until now. (those are in pending_raid_disks)
+ *
+ * the method: pick the first pending disk, collect all disks with
+ * the same UUID, remove all from the pending list and put them into
+ * the 'same_array' list. Then order this list based on superblock
+ * update time (freshest comes first), kick out 'old' disks and
+ * compare superblocks. If everything's fine then run it.
+ *
+ * If "unit" is allocated, then bump its reference count
+ */
+static void autorun_devices(int part)
+{
+ struct list_head candidates;
+ struct list_head *tmp;
+ mdk_rdev_t *rdev0, *rdev;
+ mddev_t *mddev;
+ char b[BDEVNAME_SIZE];
+
+ printk(KERN_INFO "md: autorun ...\n");
+ while (!list_empty(&pending_raid_disks)) {
+ dev_t dev;
+ rdev0 = list_entry(pending_raid_disks.next,
+ mdk_rdev_t, same_set);
+
+ printk(KERN_INFO "md: considering %s ...\n",
+ bdevname(rdev0->bdev,b));
+ INIT_LIST_HEAD(&candidates);
+ ITERATE_RDEV_PENDING(rdev,tmp)
+ if (super_90_load(rdev, rdev0, 0) >= 0) {
+ printk(KERN_INFO "md: adding %s ...\n",
+ bdevname(rdev->bdev,b));
+ list_move(&rdev->same_set, &candidates);
+ }
+ /*
+ * now we have a set of devices, with all of them having
+ * mostly sane superblocks. It's time to allocate the
+ * mddev.
+ */
+ if (rdev0->preferred_minor < 0 || rdev0->preferred_minor >= MAX_MD_DEVS) {
+ printk(KERN_INFO "md: unit number in %s is bad: %d\n",
+ bdevname(rdev0->bdev, b), rdev0->preferred_minor);
+ break;
+ }
+ if (part)
+ dev = MKDEV(mdp_major,
+ rdev0->preferred_minor << MdpMinorShift);
+ else
+ dev = MKDEV(MD_MAJOR, rdev0->preferred_minor);
+
+ md_probe(dev, NULL, NULL);
+ mddev = mddev_find(dev);
+ if (!mddev) {
+ printk(KERN_ERR
+ "md: cannot allocate memory for md drive.\n");
+ break;
+ }
+ if (mddev_lock(mddev))
+ printk(KERN_WARNING "md: %s locked, cannot run\n",
+ mdname(mddev));
+ else if (mddev->raid_disks || mddev->major_version
+ || !list_empty(&mddev->disks)) {
+ printk(KERN_WARNING
+ "md: %s already running, cannot run %s\n",
+ mdname(mddev), bdevname(rdev0->bdev,b));
+ mddev_unlock(mddev);
+ } else {
+ printk(KERN_INFO "md: created %s\n", mdname(mddev));
+ ITERATE_RDEV_GENERIC(candidates,rdev,tmp) {
+ list_del_init(&rdev->same_set);
+ if (bind_rdev_to_array(rdev, mddev))
+ export_rdev(rdev);
+ }
+ autorun_array(mddev);
+ mddev_unlock(mddev);
+ }
+ /* on success, candidates will be empty, on error
+ * it won't...
+ */
+ ITERATE_RDEV_GENERIC(candidates,rdev,tmp)
+ export_rdev(rdev);
+ mddev_put(mddev);
+ }
+ printk(KERN_INFO "md: ... autorun DONE.\n");
+}
+
+/*
+ * import RAID devices based on one partition
+ * if possible, the array gets run as well.
+ */
+
+static int autostart_array(dev_t startdev)
+{
+ char b[BDEVNAME_SIZE];
+ int err = -EINVAL, i;
+ mdp_super_t *sb = NULL;
+ mdk_rdev_t *start_rdev = NULL, *rdev;
+
+ start_rdev = md_import_device(startdev, 0, 0);
+ if (IS_ERR(start_rdev))
+ return err;
+
+
+ /* NOTE: this can only work for 0.90.0 superblocks */
+ sb = (mdp_super_t*)page_address(start_rdev->sb_page);
+ if (sb->major_version != 0 ||
+ sb->minor_version != 90 ) {
+ printk(KERN_WARNING "md: can only autostart 0.90.0 arrays\n");
+ export_rdev(start_rdev);
+ return err;
+ }
+
+ if (start_rdev->faulty) {
+ printk(KERN_WARNING
+ "md: can not autostart based on faulty %s!\n",
+ bdevname(start_rdev->bdev,b));
+ export_rdev(start_rdev);
+ return err;
+ }
+ list_add(&start_rdev->same_set, &pending_raid_disks);
+
+ for (i = 0; i < MD_SB_DISKS; i++) {
+ mdp_disk_t *desc = sb->disks + i;
+ dev_t dev = MKDEV(desc->major, desc->minor);
+
+ if (!dev)
+ continue;
+ if (dev == startdev)
+ continue;
+ if (MAJOR(dev) != desc->major || MINOR(dev) != desc->minor)
+ continue;
+ rdev = md_import_device(dev, 0, 0);
+ if (IS_ERR(rdev))
+ continue;
+
+ list_add(&rdev->same_set, &pending_raid_disks);
+ }
+
+ /*
+ * possibly return codes
+ */
+ autorun_devices(0);
+ return 0;
+
+}
+
+
+static int get_version(void __user * arg)
+{
+ mdu_version_t ver;
+
+ ver.major = MD_MAJOR_VERSION;
+ ver.minor = MD_MINOR_VERSION;
+ ver.patchlevel = MD_PATCHLEVEL_VERSION;
+
+ if (copy_to_user(arg, &ver, sizeof(ver)))
+ return -EFAULT;
+
+ return 0;
+}
+
+static int get_array_info(mddev_t * mddev, void __user * arg)
+{
+ mdu_array_info_t info;
+ int nr,working,active,failed,spare;
+ mdk_rdev_t *rdev;
+ struct list_head *tmp;
+
+ nr=working=active=failed=spare=0;
+ ITERATE_RDEV(mddev,rdev,tmp) {
+ nr++;
+ if (rdev->faulty)
+ failed++;
+ else {
+ working++;
+ if (rdev->in_sync)
+ active++;
+ else
+ spare++;
+ }
+ }
+
+ info.major_version = mddev->major_version;
+ info.minor_version = mddev->minor_version;
+ info.patch_version = MD_PATCHLEVEL_VERSION;
+ info.ctime = mddev->ctime;
+ info.level = mddev->level;
+ info.size = mddev->size;
+ info.nr_disks = nr;
+ info.raid_disks = mddev->raid_disks;
+ info.md_minor = mddev->md_minor;
+ info.not_persistent= !mddev->persistent;
+
+ info.utime = mddev->utime;
+ info.state = 0;
+ if (mddev->in_sync)
+ info.state = (1<<MD_SB_CLEAN);
+ info.active_disks = active;
+ info.working_disks = working;
+ info.failed_disks = failed;
+ info.spare_disks = spare;
+
+ info.layout = mddev->layout;
+ info.chunk_size = mddev->chunk_size;
+
+ if (copy_to_user(arg, &info, sizeof(info)))
+ return -EFAULT;
+
+ return 0;
+}
+
+static int get_disk_info(mddev_t * mddev, void __user * arg)
+{
+ mdu_disk_info_t info;
+ unsigned int nr;
+ mdk_rdev_t *rdev;
+
+ if (copy_from_user(&info, arg, sizeof(info)))
+ return -EFAULT;
+
+ nr = info.number;
+
+ rdev = find_rdev_nr(mddev, nr);
+ if (rdev) {
+ info.major = MAJOR(rdev->bdev->bd_dev);
+ info.minor = MINOR(rdev->bdev->bd_dev);
+ info.raid_disk = rdev->raid_disk;
+ info.state = 0;
+ if (rdev->faulty)
+ info.state |= (1<<MD_DISK_FAULTY);
+ else if (rdev->in_sync) {
+ info.state |= (1<<MD_DISK_ACTIVE);
+ info.state |= (1<<MD_DISK_SYNC);
+ }
+ } else {
+ info.major = info.minor = 0;
+ info.raid_disk = -1;
+ info.state = (1<<MD_DISK_REMOVED);
+ }
+
+ if (copy_to_user(arg, &info, sizeof(info)))
+ return -EFAULT;
+
+ return 0;
+}
+
+static int add_new_disk(mddev_t * mddev, mdu_disk_info_t *info)
+{
+ char b[BDEVNAME_SIZE], b2[BDEVNAME_SIZE];
+ mdk_rdev_t *rdev;
+ dev_t dev = MKDEV(info->major,info->minor);
+
+ if (info->major != MAJOR(dev) || info->minor != MINOR(dev))
+ return -EOVERFLOW;
+
+ if (!mddev->raid_disks) {
+ int err;
+ /* expecting a device which has a superblock */
+ rdev = md_import_device(dev, mddev->major_version, mddev->minor_version);
+ if (IS_ERR(rdev)) {
+ printk(KERN_WARNING
+ "md: md_import_device returned %ld\n",
+ PTR_ERR(rdev));
+ return PTR_ERR(rdev);
+ }
+ if (!list_empty(&mddev->disks)) {
+ mdk_rdev_t *rdev0 = list_entry(mddev->disks.next,
+ mdk_rdev_t, same_set);
+ int err = super_types[mddev->major_version]
+ .load_super(rdev, rdev0, mddev->minor_version);
+ if (err < 0) {
+ printk(KERN_WARNING
+ "md: %s has different UUID to %s\n",
+ bdevname(rdev->bdev,b),
+ bdevname(rdev0->bdev,b2));
+ export_rdev(rdev);
+ return -EINVAL;
+ }
+ }
+ err = bind_rdev_to_array(rdev, mddev);
+ if (err)
+ export_rdev(rdev);
+ return err;
+ }
+
+ /*
+ * add_new_disk can be used once the array is assembled
+ * to add "hot spares". They must already have a superblock
+ * written
+ */
+ if (mddev->pers) {
+ int err;
+ if (!mddev->pers->hot_add_disk) {
+ printk(KERN_WARNING
+ "%s: personality does not support diskops!\n",
+ mdname(mddev));
+ return -EINVAL;
+ }
+ rdev = md_import_device(dev, mddev->major_version,
+ mddev->minor_version);
+ if (IS_ERR(rdev)) {
+ printk(KERN_WARNING
+ "md: md_import_device returned %ld\n",
+ PTR_ERR(rdev));
+ return PTR_ERR(rdev);
+ }
+ rdev->in_sync = 0; /* just to be sure */
+ rdev->raid_disk = -1;
+ err = bind_rdev_to_array(rdev, mddev);
+ if (err)
+ export_rdev(rdev);
+ if (mddev->thread)
+ md_wakeup_thread(mddev->thread);
+ return err;
+ }
+
+ /* otherwise, add_new_disk is only allowed
+ * for major_version==0 superblocks
+ */
+ if (mddev->major_version != 0) {
+ printk(KERN_WARNING "%s: ADD_NEW_DISK not supported\n",
+ mdname(mddev));
+ return -EINVAL;
+ }
+
+ if (!(info->state & (1<<MD_DISK_FAULTY))) {
+ int err;
+ rdev = md_import_device (dev, -1, 0);
+ if (IS_ERR(rdev)) {
+ printk(KERN_WARNING
+ "md: error, md_import_device() returned %ld\n",
+ PTR_ERR(rdev));
+ return PTR_ERR(rdev);
+ }
+ rdev->desc_nr = info->number;
+ if (info->raid_disk < mddev->raid_disks)
+ rdev->raid_disk = info->raid_disk;
+ else
+ rdev->raid_disk = -1;
+
+ rdev->faulty = 0;
+ if (rdev->raid_disk < mddev->raid_disks)
+ rdev->in_sync = (info->state & (1<<MD_DISK_SYNC));
+ else
+ rdev->in_sync = 0;
+
+ err = bind_rdev_to_array(rdev, mddev);
+ if (err) {
+ export_rdev(rdev);
+ return err;
+ }
+
+ if (!mddev->persistent) {
+ printk(KERN_INFO "md: nonpersistent superblock ...\n");
+ rdev->sb_offset = rdev->bdev->bd_inode->i_size >> BLOCK_SIZE_BITS;
+ } else
+ rdev->sb_offset = calc_dev_sboffset(rdev->bdev);
+ rdev->size = calc_dev_size(rdev, mddev->chunk_size);
+
+ if (!mddev->size || (mddev->size > rdev->size))
+ mddev->size = rdev->size;
+ }
+
+ return 0;
+}
+
+static int hot_remove_disk(mddev_t * mddev, dev_t dev)
+{
+ char b[BDEVNAME_SIZE];
+ mdk_rdev_t *rdev;
+
+ if (!mddev->pers)
+ return -ENODEV;
+
+ rdev = find_rdev(mddev, dev);
+ if (!rdev)
+ return -ENXIO;
+
+ if (rdev->raid_disk >= 0)
+ goto busy;
+
+ kick_rdev_from_array(rdev);
+ md_update_sb(mddev);
+
+ return 0;
+busy:
+ printk(KERN_WARNING "md: cannot remove active disk %s from %s ... \n",
+ bdevname(rdev->bdev,b), mdname(mddev));
+ return -EBUSY;
+}
+
+static int hot_add_disk(mddev_t * mddev, dev_t dev)
+{
+ char b[BDEVNAME_SIZE];
+ int err;
+ unsigned int size;
+ mdk_rdev_t *rdev;
+
+ if (!mddev->pers)
+ return -ENODEV;
+
+ if (mddev->major_version != 0) {
+ printk(KERN_WARNING "%s: HOT_ADD may only be used with"
+ " version-0 superblocks.\n",
+ mdname(mddev));
+ return -EINVAL;
+ }
+ if (!mddev->pers->hot_add_disk) {
+ printk(KERN_WARNING
+ "%s: personality does not support diskops!\n",
+ mdname(mddev));
+ return -EINVAL;
+ }
+
+ rdev = md_import_device (dev, -1, 0);
+ if (IS_ERR(rdev)) {
+ printk(KERN_WARNING
+ "md: error, md_import_device() returned %ld\n",
+ PTR_ERR(rdev));
+ return -EINVAL;
+ }
+
+ if (mddev->persistent)
+ rdev->sb_offset = calc_dev_sboffset(rdev->bdev);
+ else
+ rdev->sb_offset =
+ rdev->bdev->bd_inode->i_size >> BLOCK_SIZE_BITS;
+
+ size = calc_dev_size(rdev, mddev->chunk_size);
+ rdev->size = size;
+
+ if (size < mddev->size) {
+ printk(KERN_WARNING
+ "%s: disk size %llu blocks < array size %llu\n",
+ mdname(mddev), (unsigned long long)size,
+ (unsigned long long)mddev->size);
+ err = -ENOSPC;
+ goto abort_export;
+ }
+
+ if (rdev->faulty) {
+ printk(KERN_WARNING
+ "md: can not hot-add faulty %s disk to %s!\n",
+ bdevname(rdev->bdev,b), mdname(mddev));
+ err = -EINVAL;
+ goto abort_export;
+ }
+ rdev->in_sync = 0;
+ rdev->desc_nr = -1;
+ bind_rdev_to_array(rdev, mddev);
+
+ /*
+ * The rest should better be atomic, we can have disk failures
+ * noticed in interrupt contexts ...
+ */
+
+ if (rdev->desc_nr == mddev->max_disks) {
+ printk(KERN_WARNING "%s: can not hot-add to full array!\n",
+ mdname(mddev));
+ err = -EBUSY;
+ goto abort_unbind_export;
+ }
+
+ rdev->raid_disk = -1;
+
+ md_update_sb(mddev);
+
+ /*
+ * Kick recovery, maybe this spare has to be added to the
+ * array immediately.
+ */
+ set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+ md_wakeup_thread(mddev->thread);
+
+ return 0;
+
+abort_unbind_export:
+ unbind_rdev_from_array(rdev);
+
+abort_export:
+ export_rdev(rdev);
+ return err;
+}
+
+/*
+ * set_array_info is used two different ways
+ * The original usage is when creating a new array.
+ * In this usage, raid_disks is > 0 and it together with
+ * level, size, not_persistent,layout,chunksize determine the
+ * shape of the array.
+ * This will always create an array with a type-0.90.0 superblock.
+ * The newer usage is when assembling an array.
+ * In this case raid_disks will be 0, and the major_version field is
+ * use to determine which style super-blocks are to be found on the devices.
+ * The minor and patch _version numbers are also kept incase the
+ * super_block handler wishes to interpret them.
+ */
+static int set_array_info(mddev_t * mddev, mdu_array_info_t *info)
+{
+
+ if (info->raid_disks == 0) {
+ /* just setting version number for superblock loading */
+ if (info->major_version < 0 ||
+ info->major_version >= sizeof(super_types)/sizeof(super_types[0]) ||
+ super_types[info->major_version].name == NULL) {
+ /* maybe try to auto-load a module? */
+ printk(KERN_INFO
+ "md: superblock version %d not known\n",
+ info->major_version);
+ return -EINVAL;
+ }
+ mddev->major_version = info->major_version;
+ mddev->minor_version = info->minor_version;
+ mddev->patch_version = info->patch_version;
+ return 0;
+ }
+ mddev->major_version = MD_MAJOR_VERSION;
+ mddev->minor_version = MD_MINOR_VERSION;
+ mddev->patch_version = MD_PATCHLEVEL_VERSION;
+ mddev->ctime = get_seconds();
+
+ mddev->level = info->level;
+ mddev->size = info->size;
+ mddev->raid_disks = info->raid_disks;
+ /* don't set md_minor, it is determined by which /dev/md* was
+ * openned
+ */
+ if (info->state & (1<<MD_SB_CLEAN))
+ mddev->recovery_cp = MaxSector;
+ else
+ mddev->recovery_cp = 0;
+ mddev->persistent = ! info->not_persistent;
+
+ mddev->layout = info->layout;
+ mddev->chunk_size = info->chunk_size;
+
+ mddev->max_disks = MD_SB_DISKS;
+
+ mddev->sb_dirty = 1;
+
+ /*
+ * Generate a 128 bit UUID
+ */
+ get_random_bytes(mddev->uuid, 16);
+
+ return 0;
+}
+
+/*
+ * update_array_info is used to change the configuration of an
+ * on-line array.
+ * The version, ctime,level,size,raid_disks,not_persistent, layout,chunk_size
+ * fields in the info are checked against the array.
+ * Any differences that cannot be handled will cause an error.
+ * Normally, only one change can be managed at a time.
+ */
+static int update_array_info(mddev_t *mddev, mdu_array_info_t *info)
+{
+ int rv = 0;
+ int cnt = 0;
+
+ if (mddev->major_version != info->major_version ||
+ mddev->minor_version != info->minor_version ||
+/* mddev->patch_version != info->patch_version || */
+ mddev->ctime != info->ctime ||
+ mddev->level != info->level ||
+/* mddev->layout != info->layout || */
+ !mddev->persistent != info->not_persistent||
+ mddev->chunk_size != info->chunk_size )
+ return -EINVAL;
+ /* Check there is only one change */
+ if (mddev->size != info->size) cnt++;
+ if (mddev->raid_disks != info->raid_disks) cnt++;
+ if (mddev->layout != info->layout) cnt++;
+ if (cnt == 0) return 0;
+ if (cnt > 1) return -EINVAL;
+
+ if (mddev->layout != info->layout) {
+ /* Change layout
+ * we don't need to do anything at the md level, the
+ * personality will take care of it all.
+ */
+ if (mddev->pers->reconfig == NULL)
+ return -EINVAL;
+ else
+ return mddev->pers->reconfig(mddev, info->layout, -1);
+ }
+ if (mddev->size != info->size) {
+ mdk_rdev_t * rdev;
+ struct list_head *tmp;
+ if (mddev->pers->resize == NULL)
+ return -EINVAL;
+ /* The "size" is the amount of each device that is used.
+ * This can only make sense for arrays with redundancy.
+ * linear and raid0 always use whatever space is available
+ * We can only consider changing the size if no resync
+ * or reconstruction is happening, and if the new size
+ * is acceptable. It must fit before the sb_offset or,
+ * if that is <data_offset, it must fit before the
+ * size of each device.
+ * If size is zero, we find the largest size that fits.
+ */
+ if (mddev->sync_thread)
+ return -EBUSY;
+ ITERATE_RDEV(mddev,rdev,tmp) {
+ sector_t avail;
+ int fit = (info->size == 0);
+ if (rdev->sb_offset > rdev->data_offset)
+ avail = (rdev->sb_offset*2) - rdev->data_offset;
+ else
+ avail = get_capacity(rdev->bdev->bd_disk)
+ - rdev->data_offset;
+ if (fit && (info->size == 0 || info->size > avail/2))
+ info->size = avail/2;
+ if (avail < ((sector_t)info->size << 1))
+ return -ENOSPC;
+ }
+ rv = mddev->pers->resize(mddev, (sector_t)info->size *2);
+ if (!rv) {
+ struct block_device *bdev;
+
+ bdev = bdget_disk(mddev->gendisk, 0);
+ if (bdev) {
+ down(&bdev->bd_inode->i_sem);
+ i_size_write(bdev->bd_inode, mddev->array_size << 10);
+ up(&bdev->bd_inode->i_sem);
+ bdput(bdev);
+ }
+ }
+ }
+ if (mddev->raid_disks != info->raid_disks) {
+ /* change the number of raid disks */
+ if (mddev->pers->reshape == NULL)
+ return -EINVAL;
+ if (info->raid_disks <= 0 ||
+ info->raid_disks >= mddev->max_disks)
+ return -EINVAL;
+ if (mddev->sync_thread)
+ return -EBUSY;
+ rv = mddev->pers->reshape(mddev, info->raid_disks);
+ if (!rv) {
+ struct block_device *bdev;
+
+ bdev = bdget_disk(mddev->gendisk, 0);
+ if (bdev) {
+ down(&bdev->bd_inode->i_sem);
+ i_size_write(bdev->bd_inode, mddev->array_size << 10);
+ up(&bdev->bd_inode->i_sem);
+ bdput(bdev);
+ }
+ }
+ }
+ md_update_sb(mddev);
+ return rv;
+}
+
+static int set_disk_faulty(mddev_t *mddev, dev_t dev)
+{
+ mdk_rdev_t *rdev;
+
+ if (mddev->pers == NULL)
+ return -ENODEV;
+
+ rdev = find_rdev(mddev, dev);
+ if (!rdev)
+ return -ENODEV;
+
+ md_error(mddev, rdev);
+ return 0;
+}
+
+static int md_ioctl(struct inode *inode, struct file *file,
+ unsigned int cmd, unsigned long arg)
+{
+ int err = 0;
+ void __user *argp = (void __user *)arg;
+ struct hd_geometry __user *loc = argp;
+ mddev_t *mddev = NULL;
+
+ if (!capable(CAP_SYS_ADMIN))
+ return -EACCES;
+
+ /*
+ * Commands dealing with the RAID driver but not any
+ * particular array:
+ */
+ switch (cmd)
+ {
+ case RAID_VERSION:
+ err = get_version(argp);
+ goto done;
+
+ case PRINT_RAID_DEBUG:
+ err = 0;
+ md_print_devices();
+ goto done;
+
+#ifndef MODULE
+ case RAID_AUTORUN:
+ err = 0;
+ autostart_arrays(arg);
+ goto done;
+#endif
+ default:;
+ }
+
+ /*
+ * Commands creating/starting a new array:
+ */
+
+ mddev = inode->i_bdev->bd_disk->private_data;
+
+ if (!mddev) {
+ BUG();
+ goto abort;
+ }
+
+
+ if (cmd == START_ARRAY) {
+ /* START_ARRAY doesn't need to lock the array as autostart_array
+ * does the locking, and it could even be a different array
+ */
+ static int cnt = 3;
+ if (cnt > 0 ) {
+ printk(KERN_WARNING
+ "md: %s(pid %d) used deprecated START_ARRAY ioctl. "
+ "This will not be supported beyond 2.6\n",
+ current->comm, current->pid);
+ cnt--;
+ }
+ err = autostart_array(new_decode_dev(arg));
+ if (err) {
+ printk(KERN_WARNING "md: autostart failed!\n");
+ goto abort;
+ }
+ goto done;
+ }
+
+ err = mddev_lock(mddev);
+ if (err) {
+ printk(KERN_INFO
+ "md: ioctl lock interrupted, reason %d, cmd %d\n",
+ err, cmd);
+ goto abort;
+ }
+
+ switch (cmd)
+ {
+ case SET_ARRAY_INFO:
+ {
+ mdu_array_info_t info;
+ if (!arg)
+ memset(&info, 0, sizeof(info));
+ else if (copy_from_user(&info, argp, sizeof(info))) {
+ err = -EFAULT;
+ goto abort_unlock;
+ }
+ if (mddev->pers) {
+ err = update_array_info(mddev, &info);
+ if (err) {
+ printk(KERN_WARNING "md: couldn't update"
+ " array info. %d\n", err);
+ goto abort_unlock;
+ }
+ goto done_unlock;
+ }
+ if (!list_empty(&mddev->disks)) {
+ printk(KERN_WARNING
+ "md: array %s already has disks!\n",
+ mdname(mddev));
+ err = -EBUSY;
+ goto abort_unlock;
+ }
+ if (mddev->raid_disks) {
+ printk(KERN_WARNING
+ "md: array %s already initialised!\n",
+ mdname(mddev));
+ err = -EBUSY;
+ goto abort_unlock;
+ }
+ err = set_array_info(mddev, &info);
+ if (err) {
+ printk(KERN_WARNING "md: couldn't set"
+ " array info. %d\n", err);
+ goto abort_unlock;
+ }
+ }
+ goto done_unlock;
+
+ default:;
+ }
+
+ /*
+ * Commands querying/configuring an existing array:
+ */
+ /* if we are initialised yet, only ADD_NEW_DISK or STOP_ARRAY is allowed */
+ if (!mddev->raid_disks && cmd != ADD_NEW_DISK && cmd != STOP_ARRAY && cmd != RUN_ARRAY) {
+ err = -ENODEV;
+ goto abort_unlock;
+ }
+
+ /*
+ * Commands even a read-only array can execute:
+ */
+ switch (cmd)
+ {
+ case GET_ARRAY_INFO:
+ err = get_array_info(mddev, argp);
+ goto done_unlock;
+
+ case GET_DISK_INFO:
+ err = get_disk_info(mddev, argp);
+ goto done_unlock;
+
+ case RESTART_ARRAY_RW:
+ err = restart_array(mddev);
+ goto done_unlock;
+
+ case STOP_ARRAY:
+ err = do_md_stop (mddev, 0);
+ goto done_unlock;
+
+ case STOP_ARRAY_RO:
+ err = do_md_stop (mddev, 1);
+ goto done_unlock;
+
+ /*
+ * We have a problem here : there is no easy way to give a CHS
+ * virtual geometry. We currently pretend that we have a 2 heads
+ * 4 sectors (with a BIG number of cylinders...). This drives
+ * dosfs just mad... ;-)
+ */
+ case HDIO_GETGEO:
+ if (!loc) {
+ err = -EINVAL;
+ goto abort_unlock;
+ }
+ err = put_user (2, (char __user *) &loc->heads);
+ if (err)
+ goto abort_unlock;
+ err = put_user (4, (char __user *) &loc->sectors);
+ if (err)
+ goto abort_unlock;
+ err = put_user(get_capacity(mddev->gendisk)/8,
+ (short __user *) &loc->cylinders);
+ if (err)
+ goto abort_unlock;
+ err = put_user (get_start_sect(inode->i_bdev),
+ (long __user *) &loc->start);
+ goto done_unlock;
+ }
+
+ /*
+ * The remaining ioctls are changing the state of the
+ * superblock, so we do not allow read-only arrays
+ * here:
+ */
+ if (mddev->ro) {
+ err = -EROFS;
+ goto abort_unlock;
+ }
+
+ switch (cmd)
+ {
+ case ADD_NEW_DISK:
+ {
+ mdu_disk_info_t info;
+ if (copy_from_user(&info, argp, sizeof(info)))
+ err = -EFAULT;
+ else
+ err = add_new_disk(mddev, &info);
+ goto done_unlock;
+ }
+
+ case HOT_REMOVE_DISK:
+ err = hot_remove_disk(mddev, new_decode_dev(arg));
+ goto done_unlock;
+
+ case HOT_ADD_DISK:
+ err = hot_add_disk(mddev, new_decode_dev(arg));
+ goto done_unlock;
+
+ case SET_DISK_FAULTY:
+ err = set_disk_faulty(mddev, new_decode_dev(arg));
+ goto done_unlock;
+
+ case RUN_ARRAY:
+ err = do_md_run (mddev);
+ goto done_unlock;
+
+ default:
+ if (_IOC_TYPE(cmd) == MD_MAJOR)
+ printk(KERN_WARNING "md: %s(pid %d) used"
+ " obsolete MD ioctl, upgrade your"
+ " software to use new ictls.\n",
+ current->comm, current->pid);
+ err = -EINVAL;
+ goto abort_unlock;
+ }
+
+done_unlock:
+abort_unlock:
+ mddev_unlock(mddev);
+
+ return err;
+done:
+ if (err)
+ MD_BUG();
+abort:
+ return err;
+}
+
+static int md_open(struct inode *inode, struct file *file)
+{
+ /*
+ * Succeed if we can lock the mddev, which confirms that
+ * it isn't being stopped right now.
+ */
+ mddev_t *mddev = inode->i_bdev->bd_disk->private_data;
+ int err;
+
+ if ((err = mddev_lock(mddev)))
+ goto out;
+
+ err = 0;
+ mddev_get(mddev);
+ mddev_unlock(mddev);
+
+ check_disk_change(inode->i_bdev);
+ out:
+ return err;
+}
+
+static int md_release(struct inode *inode, struct file * file)
+{
+ mddev_t *mddev = inode->i_bdev->bd_disk->private_data;
+
+ if (!mddev)
+ BUG();
+ mddev_put(mddev);
+
+ return 0;
+}
+
+static int md_media_changed(struct gendisk *disk)
+{
+ mddev_t *mddev = disk->private_data;
+
+ return mddev->changed;
+}
+
+static int md_revalidate(struct gendisk *disk)
+{
+ mddev_t *mddev = disk->private_data;
+
+ mddev->changed = 0;
+ return 0;
+}
+static struct block_device_operations md_fops =
+{
+ .owner = THIS_MODULE,
+ .open = md_open,
+ .release = md_release,
+ .ioctl = md_ioctl,
+ .media_changed = md_media_changed,
+ .revalidate_disk= md_revalidate,
+};
+
+int md_thread(void * arg)
+{
+ mdk_thread_t *thread = arg;
+
+ lock_kernel();
+
+ /*
+ * Detach thread
+ */
+
+ daemonize(thread->name, mdname(thread->mddev));
+
+ current->exit_signal = SIGCHLD;
+ allow_signal(SIGKILL);
+ thread->tsk = current;
+
+ /*
+ * md_thread is a 'system-thread', it's priority should be very
+ * high. We avoid resource deadlocks individually in each
+ * raid personality. (RAID5 does preallocation) We also use RR and
+ * the very same RT priority as kswapd, thus we will never get
+ * into a priority inversion deadlock.
+ *
+ * we definitely have to have equal or higher priority than
+ * bdflush, otherwise bdflush will deadlock if there are too
+ * many dirty RAID5 blocks.
+ */
+ unlock_kernel();
+
+ complete(thread->event);
+ while (thread->run) {
+ void (*run)(mddev_t *);
+
+ wait_event_interruptible(thread->wqueue,
+ test_bit(THREAD_WAKEUP, &thread->flags));
+ if (current->flags & PF_FREEZE)
+ refrigerator(PF_FREEZE);
+
+ clear_bit(THREAD_WAKEUP, &thread->flags);
+
+ run = thread->run;
+ if (run)
+ run(thread->mddev);
+
+ if (signal_pending(current))
+ flush_signals(current);
+ }
+ complete(thread->event);
+ return 0;
+}
+
+void md_wakeup_thread(mdk_thread_t *thread)
+{
+ if (thread) {
+ dprintk("md: waking up MD thread %s.\n", thread->tsk->comm);
+ set_bit(THREAD_WAKEUP, &thread->flags);
+ wake_up(&thread->wqueue);
+ }
+}
+
+mdk_thread_t *md_register_thread(void (*run) (mddev_t *), mddev_t *mddev,
+ const char *name)
+{
+ mdk_thread_t *thread;
+ int ret;
+ struct completion event;
+
+ thread = (mdk_thread_t *) kmalloc
+ (sizeof(mdk_thread_t), GFP_KERNEL);
+ if (!thread)
+ return NULL;
+
+ memset(thread, 0, sizeof(mdk_thread_t));
+ init_waitqueue_head(&thread->wqueue);
+
+ init_completion(&event);
+ thread->event = &event;
+ thread->run = run;
+ thread->mddev = mddev;
+ thread->name = name;
+ ret = kernel_thread(md_thread, thread, 0);
+ if (ret < 0) {
+ kfree(thread);
+ return NULL;
+ }
+ wait_for_completion(&event);
+ return thread;
+}
+
+static void md_interrupt_thread(mdk_thread_t *thread)
+{
+ if (!thread->tsk) {
+ MD_BUG();
+ return;
+ }
+ dprintk("interrupting MD-thread pid %d\n", thread->tsk->pid);
+ send_sig(SIGKILL, thread->tsk, 1);
+}
+
+void md_unregister_thread(mdk_thread_t *thread)
+{
+ struct completion event;
+
+ init_completion(&event);
+
+ thread->event = &event;
+ thread->run = NULL;
+ thread->name = NULL;
+ md_interrupt_thread(thread);
+ wait_for_completion(&event);
+ kfree(thread);
+}
+
+void md_error(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+ if (!mddev) {
+ MD_BUG();
+ return;
+ }
+
+ if (!rdev || rdev->faulty)
+ return;
+
+ dprintk("md_error dev:%s, rdev:(%d:%d), (caller: %p,%p,%p,%p).\n",
+ mdname(mddev),
+ MAJOR(rdev->bdev->bd_dev), MINOR(rdev->bdev->bd_dev),
+ __builtin_return_address(0),__builtin_return_address(1),
+ __builtin_return_address(2),__builtin_return_address(3));
+
+ if (!mddev->pers->error_handler)
+ return;
+ mddev->pers->error_handler(mddev,rdev);
+ set_bit(MD_RECOVERY_INTR, &mddev->recovery);
+ set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+ md_wakeup_thread(mddev->thread);
+}
+
+/* seq_file implementation /proc/mdstat */
+
+static void status_unused(struct seq_file *seq)
+{
+ int i = 0;
+ mdk_rdev_t *rdev;
+ struct list_head *tmp;
+
+ seq_printf(seq, "unused devices: ");
+
+ ITERATE_RDEV_PENDING(rdev,tmp) {
+ char b[BDEVNAME_SIZE];
+ i++;
+ seq_printf(seq, "%s ",
+ bdevname(rdev->bdev,b));
+ }
+ if (!i)
+ seq_printf(seq, "<none>");
+
+ seq_printf(seq, "\n");
+}
+
+
+static void status_resync(struct seq_file *seq, mddev_t * mddev)
+{
+ unsigned long max_blocks, resync, res, dt, db, rt;
+
+ resync = (mddev->curr_resync - atomic_read(&mddev->recovery_active))/2;
+
+ if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
+ max_blocks = mddev->resync_max_sectors >> 1;
+ else
+ max_blocks = mddev->size;
+
+ /*
+ * Should not happen.
+ */
+ if (!max_blocks) {
+ MD_BUG();
+ return;
+ }
+ res = (resync/1024)*1000/(max_blocks/1024 + 1);
+ {
+ int i, x = res/50, y = 20-x;
+ seq_printf(seq, "[");
+ for (i = 0; i < x; i++)
+ seq_printf(seq, "=");
+ seq_printf(seq, ">");
+ for (i = 0; i < y; i++)
+ seq_printf(seq, ".");
+ seq_printf(seq, "] ");
+ }
+ seq_printf(seq, " %s =%3lu.%lu%% (%lu/%lu)",
+ (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ?
+ "resync" : "recovery"),
+ res/10, res % 10, resync, max_blocks);
+
+ /*
+ * We do not want to overflow, so the order of operands and
+ * the * 100 / 100 trick are important. We do a +1 to be
+ * safe against division by zero. We only estimate anyway.
+ *
+ * dt: time from mark until now
+ * db: blocks written from mark until now
+ * rt: remaining time
+ */
+ dt = ((jiffies - mddev->resync_mark) / HZ);
+ if (!dt) dt++;
+ db = resync - (mddev->resync_mark_cnt/2);
+ rt = (dt * ((max_blocks-resync) / (db/100+1)))/100;
+
+ seq_printf(seq, " finish=%lu.%lumin", rt / 60, (rt % 60)/6);
+
+ seq_printf(seq, " speed=%ldK/sec", db/dt);
+}
+
+static void *md_seq_start(struct seq_file *seq, loff_t *pos)
+{
+ struct list_head *tmp;
+ loff_t l = *pos;
+ mddev_t *mddev;
+
+ if (l >= 0x10000)
+ return NULL;
+ if (!l--)
+ /* header */
+ return (void*)1;
+
+ spin_lock(&all_mddevs_lock);
+ list_for_each(tmp,&all_mddevs)
+ if (!l--) {
+ mddev = list_entry(tmp, mddev_t, all_mddevs);
+ mddev_get(mddev);
+ spin_unlock(&all_mddevs_lock);
+ return mddev;
+ }
+ spin_unlock(&all_mddevs_lock);
+ if (!l--)
+ return (void*)2;/* tail */
+ return NULL;
+}
+
+static void *md_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+ struct list_head *tmp;
+ mddev_t *next_mddev, *mddev = v;
+
+ ++*pos;
+ if (v == (void*)2)
+ return NULL;
+
+ spin_lock(&all_mddevs_lock);
+ if (v == (void*)1)
+ tmp = all_mddevs.next;
+ else
+ tmp = mddev->all_mddevs.next;
+ if (tmp != &all_mddevs)
+ next_mddev = mddev_get(list_entry(tmp,mddev_t,all_mddevs));
+ else {
+ next_mddev = (void*)2;
+ *pos = 0x10000;
+ }
+ spin_unlock(&all_mddevs_lock);
+
+ if (v != (void*)1)
+ mddev_put(mddev);
+ return next_mddev;
+
+}
+
+static void md_seq_stop(struct seq_file *seq, void *v)
+{
+ mddev_t *mddev = v;
+
+ if (mddev && v != (void*)1 && v != (void*)2)
+ mddev_put(mddev);
+}
+
+static int md_seq_show(struct seq_file *seq, void *v)
+{
+ mddev_t *mddev = v;
+ sector_t size;
+ struct list_head *tmp2;
+ mdk_rdev_t *rdev;
+ int i;
+
+ if (v == (void*)1) {
+ seq_printf(seq, "Personalities : ");
+ spin_lock(&pers_lock);
+ for (i = 0; i < MAX_PERSONALITY; i++)
+ if (pers[i])
+ seq_printf(seq, "[%s] ", pers[i]->name);
+
+ spin_unlock(&pers_lock);
+ seq_printf(seq, "\n");
+ return 0;
+ }
+ if (v == (void*)2) {
+ status_unused(seq);
+ return 0;
+ }
+
+ if (mddev_lock(mddev)!=0)
+ return -EINTR;
+ if (mddev->pers || mddev->raid_disks || !list_empty(&mddev->disks)) {
+ seq_printf(seq, "%s : %sactive", mdname(mddev),
+ mddev->pers ? "" : "in");
+ if (mddev->pers) {
+ if (mddev->ro)
+ seq_printf(seq, " (read-only)");
+ seq_printf(seq, " %s", mddev->pers->name);
+ }
+
+ size = 0;
+ ITERATE_RDEV(mddev,rdev,tmp2) {
+ char b[BDEVNAME_SIZE];
+ seq_printf(seq, " %s[%d]",
+ bdevname(rdev->bdev,b), rdev->desc_nr);
+ if (rdev->faulty) {
+ seq_printf(seq, "(F)");
+ continue;
+ }
+ size += rdev->size;
+ }
+
+ if (!list_empty(&mddev->disks)) {
+ if (mddev->pers)
+ seq_printf(seq, "\n %llu blocks",
+ (unsigned long long)mddev->array_size);
+ else
+ seq_printf(seq, "\n %llu blocks",
+ (unsigned long long)size);
+ }
+
+ if (mddev->pers) {
+ mddev->pers->status (seq, mddev);
+ seq_printf(seq, "\n ");
+ if (mddev->curr_resync > 2)
+ status_resync (seq, mddev);
+ else if (mddev->curr_resync == 1 || mddev->curr_resync == 2)
+ seq_printf(seq, " resync=DELAYED");
+ }
+
+ seq_printf(seq, "\n");
+ }
+ mddev_unlock(mddev);
+
+ return 0;
+}
+
+static struct seq_operations md_seq_ops = {
+ .start = md_seq_start,
+ .next = md_seq_next,
+ .stop = md_seq_stop,
+ .show = md_seq_show,
+};
+
+static int md_seq_open(struct inode *inode, struct file *file)
+{
+ int error;
+
+ error = seq_open(file, &md_seq_ops);
+ return error;
+}
+
+static struct file_operations md_seq_fops = {
+ .open = md_seq_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = seq_release,
+};
+
+int register_md_personality(int pnum, mdk_personality_t *p)
+{
+ if (pnum >= MAX_PERSONALITY) {
+ printk(KERN_ERR
+ "md: tried to install personality %s as nr %d, but max is %lu\n",
+ p->name, pnum, MAX_PERSONALITY-1);
+ return -EINVAL;
+ }
+
+ spin_lock(&pers_lock);
+ if (pers[pnum]) {
+ spin_unlock(&pers_lock);
+ MD_BUG();
+ return -EBUSY;
+ }
+
+ pers[pnum] = p;
+ printk(KERN_INFO "md: %s personality registered as nr %d\n", p->name, pnum);
+ spin_unlock(&pers_lock);
+ return 0;
+}
+
+int unregister_md_personality(int pnum)
+{
+ if (pnum >= MAX_PERSONALITY) {
+ MD_BUG();
+ return -EINVAL;
+ }
+
+ printk(KERN_INFO "md: %s personality unregistered\n", pers[pnum]->name);
+ spin_lock(&pers_lock);
+ pers[pnum] = NULL;
+ spin_unlock(&pers_lock);
+ return 0;
+}
+
+static int is_mddev_idle(mddev_t *mddev)
+{
+ mdk_rdev_t * rdev;
+ struct list_head *tmp;
+ int idle;
+ unsigned long curr_events;
+
+ idle = 1;
+ ITERATE_RDEV(mddev,rdev,tmp) {
+ struct gendisk *disk = rdev->bdev->bd_contains->bd_disk;
+ curr_events = disk_stat_read(disk, read_sectors) +
+ disk_stat_read(disk, write_sectors) -
+ atomic_read(&disk->sync_io);
+ /* Allow some slack between valud of curr_events and last_events,
+ * as there are some uninteresting races.
+ * Note: the following is an unsigned comparison.
+ */
+ if ((curr_events - rdev->last_events + 32) > 64) {
+ rdev->last_events = curr_events;
+ idle = 0;
+ }
+ }
+ return idle;
+}
+
+void md_done_sync(mddev_t *mddev, int blocks, int ok)
+{
+ /* another "blocks" (512byte) blocks have been synced */
+ atomic_sub(blocks, &mddev->recovery_active);
+ wake_up(&mddev->recovery_wait);
+ if (!ok) {
+ set_bit(MD_RECOVERY_ERR, &mddev->recovery);
+ md_wakeup_thread(mddev->thread);
+ // stop recovery, signal do_sync ....
+ }
+}
+
+
+void md_write_start(mddev_t *mddev)
+{
+ if (!atomic_read(&mddev->writes_pending)) {
+ mddev_lock_uninterruptible(mddev);
+ if (mddev->in_sync) {
+ mddev->in_sync = 0;
+ del_timer(&mddev->safemode_timer);
+ md_update_sb(mddev);
+ }
+ atomic_inc(&mddev->writes_pending);
+ mddev_unlock(mddev);
+ } else
+ atomic_inc(&mddev->writes_pending);
+}
+
+void md_write_end(mddev_t *mddev)
+{
+ if (atomic_dec_and_test(&mddev->writes_pending)) {
+ if (mddev->safemode == 2)
+ md_wakeup_thread(mddev->thread);
+ else
+ mod_timer(&mddev->safemode_timer, jiffies + mddev->safemode_delay);
+ }
+}
+
+static inline void md_enter_safemode(mddev_t *mddev)
+{
+ if (!mddev->safemode) return;
+ if (mddev->safemode == 2 &&
+ (atomic_read(&mddev->writes_pending) || mddev->in_sync ||
+ mddev->recovery_cp != MaxSector))
+ return; /* avoid the lock */
+ mddev_lock_uninterruptible(mddev);
+ if (mddev->safemode && !atomic_read(&mddev->writes_pending) &&
+ !mddev->in_sync && mddev->recovery_cp == MaxSector) {
+ mddev->in_sync = 1;
+ md_update_sb(mddev);
+ }
+ mddev_unlock(mddev);
+
+ if (mddev->safemode == 1)
+ mddev->safemode = 0;
+}
+
+void md_handle_safemode(mddev_t *mddev)
+{
+ if (signal_pending(current)) {
+ printk(KERN_INFO "md: %s in immediate safe mode\n",
+ mdname(mddev));
+ mddev->safemode = 2;
+ flush_signals(current);
+ }
+ md_enter_safemode(mddev);
+}
+
+
+DECLARE_WAIT_QUEUE_HEAD(resync_wait);
+
+#define SYNC_MARKS 10
+#define SYNC_MARK_STEP (3*HZ)
+static void md_do_sync(mddev_t *mddev)
+{
+ mddev_t *mddev2;
+ unsigned int currspeed = 0,
+ window;
+ sector_t max_sectors,j;
+ unsigned long mark[SYNC_MARKS];
+ sector_t mark_cnt[SYNC_MARKS];
+ int last_mark,m;
+ struct list_head *tmp;
+ sector_t last_check;
+
+ /* just incase thread restarts... */
+ if (test_bit(MD_RECOVERY_DONE, &mddev->recovery))
+ return;
+
+ /* we overload curr_resync somewhat here.
+ * 0 == not engaged in resync at all
+ * 2 == checking that there is no conflict with another sync
+ * 1 == like 2, but have yielded to allow conflicting resync to
+ * commense
+ * other == active in resync - this many blocks
+ *
+ * Before starting a resync we must have set curr_resync to
+ * 2, and then checked that every "conflicting" array has curr_resync
+ * less than ours. When we find one that is the same or higher
+ * we wait on resync_wait. To avoid deadlock, we reduce curr_resync
+ * to 1 if we choose to yield (based arbitrarily on address of mddev structure).
+ * This will mean we have to start checking from the beginning again.
+ *
+ */
+
+ do {
+ mddev->curr_resync = 2;
+
+ try_again:
+ if (signal_pending(current)) {
+ flush_signals(current);
+ goto skip;
+ }
+ ITERATE_MDDEV(mddev2,tmp) {
+ printk(".");
+ if (mddev2 == mddev)
+ continue;
+ if (mddev2->curr_resync &&
+ match_mddev_units(mddev,mddev2)) {
+ DEFINE_WAIT(wq);
+ if (mddev < mddev2 && mddev->curr_resync == 2) {
+ /* arbitrarily yield */
+ mddev->curr_resync = 1;
+ wake_up(&resync_wait);
+ }
+ if (mddev > mddev2 && mddev->curr_resync == 1)
+ /* no need to wait here, we can wait the next
+ * time 'round when curr_resync == 2
+ */
+ continue;
+ prepare_to_wait(&resync_wait, &wq, TASK_INTERRUPTIBLE);
+ if (!signal_pending(current)
+ && mddev2->curr_resync >= mddev->curr_resync) {
+ printk(KERN_INFO "md: delaying resync of %s"
+ " until %s has finished resync (they"
+ " share one or more physical units)\n",
+ mdname(mddev), mdname(mddev2));
+ mddev_put(mddev2);
+ schedule();
+ finish_wait(&resync_wait, &wq);
+ goto try_again;
+ }
+ finish_wait(&resync_wait, &wq);
+ }
+ }
+ } while (mddev->curr_resync < 2);
+
+ if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
+ /* resync follows the size requested by the personality,
+ * which default to physical size, but can be virtual size
+ */
+ max_sectors = mddev->resync_max_sectors;
+ else
+ /* recovery follows the physical size of devices */
+ max_sectors = mddev->size << 1;
+
+ printk(KERN_INFO "md: syncing RAID array %s\n", mdname(mddev));
+ printk(KERN_INFO "md: minimum _guaranteed_ reconstruction speed:"
+ " %d KB/sec/disc.\n", sysctl_speed_limit_min);
+ printk(KERN_INFO "md: using maximum available idle IO bandwith "
+ "(but not more than %d KB/sec) for reconstruction.\n",
+ sysctl_speed_limit_max);
+
+ is_mddev_idle(mddev); /* this also initializes IO event counters */
+ if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
+ j = mddev->recovery_cp;
+ else
+ j = 0;
+ for (m = 0; m < SYNC_MARKS; m++) {
+ mark[m] = jiffies;
+ mark_cnt[m] = j;
+ }
+ last_mark = 0;
+ mddev->resync_mark = mark[last_mark];
+ mddev->resync_mark_cnt = mark_cnt[last_mark];
+
+ /*
+ * Tune reconstruction:
+ */
+ window = 32*(PAGE_SIZE/512);
+ printk(KERN_INFO "md: using %dk window, over a total of %llu blocks.\n",
+ window/2,(unsigned long long) max_sectors/2);
+
+ atomic_set(&mddev->recovery_active, 0);
+ init_waitqueue_head(&mddev->recovery_wait);
+ last_check = 0;
+
+ if (j>2) {
+ printk(KERN_INFO
+ "md: resuming recovery of %s from checkpoint.\n",
+ mdname(mddev));
+ mddev->curr_resync = j;
+ }
+
+ while (j < max_sectors) {
+ int sectors;
+
+ sectors = mddev->pers->sync_request(mddev, j, currspeed < sysctl_speed_limit_min);
+ if (sectors < 0) {
+ set_bit(MD_RECOVERY_ERR, &mddev->recovery);
+ goto out;
+ }
+ atomic_add(sectors, &mddev->recovery_active);
+ j += sectors;
+ if (j>1) mddev->curr_resync = j;
+
+ if (last_check + window > j || j == max_sectors)
+ continue;
+
+ last_check = j;
+
+ if (test_bit(MD_RECOVERY_INTR, &mddev->recovery) ||
+ test_bit(MD_RECOVERY_ERR, &mddev->recovery))
+ break;
+
+ repeat:
+ if (time_after_eq(jiffies, mark[last_mark] + SYNC_MARK_STEP )) {
+ /* step marks */
+ int next = (last_mark+1) % SYNC_MARKS;
+
+ mddev->resync_mark = mark[next];
+ mddev->resync_mark_cnt = mark_cnt[next];
+ mark[next] = jiffies;
+ mark_cnt[next] = j - atomic_read(&mddev->recovery_active);
+ last_mark = next;
+ }
+
+
+ if (signal_pending(current)) {
+ /*
+ * got a signal, exit.
+ */
+ printk(KERN_INFO
+ "md: md_do_sync() got signal ... exiting\n");
+ flush_signals(current);
+ set_bit(MD_RECOVERY_INTR, &mddev->recovery);
+ goto out;
+ }
+
+ /*
+ * this loop exits only if either when we are slower than
+ * the 'hard' speed limit, or the system was IO-idle for
+ * a jiffy.
+ * the system might be non-idle CPU-wise, but we only care
+ * about not overloading the IO subsystem. (things like an
+ * e2fsck being done on the RAID array should execute fast)
+ */
+ mddev->queue->unplug_fn(mddev->queue);
+ cond_resched();
+
+ currspeed = ((unsigned long)(j-mddev->resync_mark_cnt))/2/((jiffies-mddev->resync_mark)/HZ +1) +1;
+
+ if (currspeed > sysctl_speed_limit_min) {
+ if ((currspeed > sysctl_speed_limit_max) ||
+ !is_mddev_idle(mddev)) {
+ msleep_interruptible(250);
+ goto repeat;
+ }
+ }
+ }
+ printk(KERN_INFO "md: %s: sync done.\n",mdname(mddev));
+ /*
+ * this also signals 'finished resyncing' to md_stop
+ */
+ out:
+ mddev->queue->unplug_fn(mddev->queue);
+
+ wait_event(mddev->recovery_wait, !atomic_read(&mddev->recovery_active));
+
+ /* tell personality that we are finished */
+ mddev->pers->sync_request(mddev, max_sectors, 1);
+
+ if (!test_bit(MD_RECOVERY_ERR, &mddev->recovery) &&
+ mddev->curr_resync > 2 &&
+ mddev->curr_resync >= mddev->recovery_cp) {
+ if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
+ printk(KERN_INFO
+ "md: checkpointing recovery of %s.\n",
+ mdname(mddev));
+ mddev->recovery_cp = mddev->curr_resync;
+ } else
+ mddev->recovery_cp = MaxSector;
+ }
+
+ md_enter_safemode(mddev);
+ skip:
+ mddev->curr_resync = 0;
+ wake_up(&resync_wait);
+ set_bit(MD_RECOVERY_DONE, &mddev->recovery);
+ md_wakeup_thread(mddev->thread);
+}
+
+
+/*
+ * This routine is regularly called by all per-raid-array threads to
+ * deal with generic issues like resync and super-block update.
+ * Raid personalities that don't have a thread (linear/raid0) do not
+ * need this as they never do any recovery or update the superblock.
+ *
+ * It does not do any resync itself, but rather "forks" off other threads
+ * to do that as needed.
+ * When it is determined that resync is needed, we set MD_RECOVERY_RUNNING in
+ * "->recovery" and create a thread at ->sync_thread.
+ * When the thread finishes it sets MD_RECOVERY_DONE (and might set MD_RECOVERY_ERR)
+ * and wakeups up this thread which will reap the thread and finish up.
+ * This thread also removes any faulty devices (with nr_pending == 0).
+ *
+ * The overall approach is:
+ * 1/ if the superblock needs updating, update it.
+ * 2/ If a recovery thread is running, don't do anything else.
+ * 3/ If recovery has finished, clean up, possibly marking spares active.
+ * 4/ If there are any faulty devices, remove them.
+ * 5/ If array is degraded, try to add spares devices
+ * 6/ If array has spares or is not in-sync, start a resync thread.
+ */
+void md_check_recovery(mddev_t *mddev)
+{
+ mdk_rdev_t *rdev;
+ struct list_head *rtmp;
+
+
+ dprintk(KERN_INFO "md: recovery thread got woken up ...\n");
+
+ if (mddev->ro)
+ return;
+ if ( ! (
+ mddev->sb_dirty ||
+ test_bit(MD_RECOVERY_NEEDED, &mddev->recovery) ||
+ test_bit(MD_RECOVERY_DONE, &mddev->recovery)
+ ))
+ return;
+ if (mddev_trylock(mddev)==0) {
+ int spares =0;
+ if (mddev->sb_dirty)
+ md_update_sb(mddev);
+ if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) &&
+ !test_bit(MD_RECOVERY_DONE, &mddev->recovery)) {
+ /* resync/recovery still happening */
+ clear_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+ goto unlock;
+ }
+ if (mddev->sync_thread) {
+ /* resync has finished, collect result */
+ md_unregister_thread(mddev->sync_thread);
+ mddev->sync_thread = NULL;
+ if (!test_bit(MD_RECOVERY_ERR, &mddev->recovery) &&
+ !test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
+ /* success...*/
+ /* activate any spares */
+ mddev->pers->spare_active(mddev);
+ }
+ md_update_sb(mddev);
+ mddev->recovery = 0;
+ /* flag recovery needed just to double check */
+ set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+ goto unlock;
+ }
+ if (mddev->recovery)
+ /* probably just the RECOVERY_NEEDED flag */
+ mddev->recovery = 0;
+
+ /* no recovery is running.
+ * remove any failed drives, then
+ * add spares if possible.
+ * Spare are also removed and re-added, to allow
+ * the personality to fail the re-add.
+ */
+ ITERATE_RDEV(mddev,rdev,rtmp)
+ if (rdev->raid_disk >= 0 &&
+ (rdev->faulty || ! rdev->in_sync) &&
+ atomic_read(&rdev->nr_pending)==0) {
+ if (mddev->pers->hot_remove_disk(mddev, rdev->raid_disk)==0)
+ rdev->raid_disk = -1;
+ }
+
+ if (mddev->degraded) {
+ ITERATE_RDEV(mddev,rdev,rtmp)
+ if (rdev->raid_disk < 0
+ && !rdev->faulty) {
+ if (mddev->pers->hot_add_disk(mddev,rdev))
+ spares++;
+ else
+ break;
+ }
+ }
+
+ if (!spares && (mddev->recovery_cp == MaxSector )) {
+ /* nothing we can do ... */
+ goto unlock;
+ }
+ if (mddev->pers->sync_request) {
+ set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
+ if (!spares)
+ set_bit(MD_RECOVERY_SYNC, &mddev->recovery);
+ mddev->sync_thread = md_register_thread(md_do_sync,
+ mddev,
+ "%s_resync");
+ if (!mddev->sync_thread) {
+ printk(KERN_ERR "%s: could not start resync"
+ " thread...\n",
+ mdname(mddev));
+ /* leave the spares where they are, it shouldn't hurt */
+ mddev->recovery = 0;
+ } else {
+ md_wakeup_thread(mddev->sync_thread);
+ }
+ }
+ unlock:
+ mddev_unlock(mddev);
+ }
+}
+
+int md_notify_reboot(struct notifier_block *this,
+ unsigned long code, void *x)
+{
+ struct list_head *tmp;
+ mddev_t *mddev;
+
+ if ((code == SYS_DOWN) || (code == SYS_HALT) || (code == SYS_POWER_OFF)) {
+
+ printk(KERN_INFO "md: stopping all md devices.\n");
+
+ ITERATE_MDDEV(mddev,tmp)
+ if (mddev_trylock(mddev)==0)
+ do_md_stop (mddev, 1);
+ /*
+ * certain more exotic SCSI devices are known to be
+ * volatile wrt too early system reboots. While the
+ * right place to handle this issue is the given
+ * driver, we do want to have a safe RAID driver ...
+ */
+ mdelay(1000*1);
+ }
+ return NOTIFY_DONE;
+}
+
+struct notifier_block md_notifier = {
+ .notifier_call = md_notify_reboot,
+ .next = NULL,
+ .priority = INT_MAX, /* before any real devices */
+};
+
+static void md_geninit(void)
+{
+ struct proc_dir_entry *p;
+
+ dprintk("md: sizeof(mdp_super_t) = %d\n", (int)sizeof(mdp_super_t));
+
+ p = create_proc_entry("mdstat", S_IRUGO, NULL);
+ if (p)
+ p->proc_fops = &md_seq_fops;
+}
+
+int __init md_init(void)
+{
+ int minor;
+
+ printk(KERN_INFO "md: md driver %d.%d.%d MAX_MD_DEVS=%d,"
+ " MD_SB_DISKS=%d\n",
+ MD_MAJOR_VERSION, MD_MINOR_VERSION,
+ MD_PATCHLEVEL_VERSION, MAX_MD_DEVS, MD_SB_DISKS);
+
+ if (register_blkdev(MAJOR_NR, "md"))
+ return -1;
+ if ((mdp_major=register_blkdev(0, "mdp"))<=0) {
+ unregister_blkdev(MAJOR_NR, "md");
+ return -1;
+ }
+ devfs_mk_dir("md");
+ blk_register_region(MKDEV(MAJOR_NR, 0), MAX_MD_DEVS, THIS_MODULE,
+ md_probe, NULL, NULL);
+ blk_register_region(MKDEV(mdp_major, 0), MAX_MD_DEVS<<MdpMinorShift, THIS_MODULE,
+ md_probe, NULL, NULL);
+
+ for (minor=0; minor < MAX_MD_DEVS; ++minor)
+ devfs_mk_bdev(MKDEV(MAJOR_NR, minor),
+ S_IFBLK|S_IRUSR|S_IWUSR,
+ "md/%d", minor);
+
+ for (minor=0; minor < MAX_MD_DEVS; ++minor)
+ devfs_mk_bdev(MKDEV(mdp_major, minor<<MdpMinorShift),
+ S_IFBLK|S_IRUSR|S_IWUSR,
+ "md/mdp%d", minor);
+
+
+ register_reboot_notifier(&md_notifier);
+ raid_table_header = register_sysctl_table(raid_root_table, 1);
+
+ md_geninit();
+ return (0);
+}
+
+
+#ifndef MODULE
+
+/*
+ * Searches all registered partitions for autorun RAID arrays
+ * at boot time.
+ */
+static dev_t detected_devices[128];
+static int dev_cnt;
+
+void md_autodetect_dev(dev_t dev)
+{
+ if (dev_cnt >= 0 && dev_cnt < 127)
+ detected_devices[dev_cnt++] = dev;
+}
+
+
+static void autostart_arrays(int part)
+{
+ mdk_rdev_t *rdev;
+ int i;
+
+ printk(KERN_INFO "md: Autodetecting RAID arrays.\n");
+
+ for (i = 0; i < dev_cnt; i++) {
+ dev_t dev = detected_devices[i];
+
+ rdev = md_import_device(dev,0, 0);
+ if (IS_ERR(rdev))
+ continue;
+
+ if (rdev->faulty) {
+ MD_BUG();
+ continue;
+ }
+ list_add(&rdev->same_set, &pending_raid_disks);
+ }
+ dev_cnt = 0;
+
+ autorun_devices(part);
+}
+
+#endif
+
+static __exit void md_exit(void)
+{
+ mddev_t *mddev;
+ struct list_head *tmp;
+ int i;
+ blk_unregister_region(MKDEV(MAJOR_NR,0), MAX_MD_DEVS);
+ blk_unregister_region(MKDEV(mdp_major,0), MAX_MD_DEVS << MdpMinorShift);
+ for (i=0; i < MAX_MD_DEVS; i++)
+ devfs_remove("md/%d", i);
+ for (i=0; i < MAX_MD_DEVS; i++)
+ devfs_remove("md/d%d", i);
+
+ devfs_remove("md");
+
+ unregister_blkdev(MAJOR_NR,"md");
+ unregister_blkdev(mdp_major, "mdp");
+ unregister_reboot_notifier(&md_notifier);
+ unregister_sysctl_table(raid_table_header);
+ remove_proc_entry("mdstat", NULL);
+ ITERATE_MDDEV(mddev,tmp) {
+ struct gendisk *disk = mddev->gendisk;
+ if (!disk)
+ continue;
+ export_array(mddev);
+ del_gendisk(disk);
+ put_disk(disk);
+ mddev->gendisk = NULL;
+ mddev_put(mddev);
+ }
+}
+
+module_init(md_init)
+module_exit(md_exit)
+
+EXPORT_SYMBOL(register_md_personality);
+EXPORT_SYMBOL(unregister_md_personality);
+EXPORT_SYMBOL(md_error);
+EXPORT_SYMBOL(md_done_sync);
+EXPORT_SYMBOL(md_write_start);
+EXPORT_SYMBOL(md_write_end);
+EXPORT_SYMBOL(md_handle_safemode);
+EXPORT_SYMBOL(md_register_thread);
+EXPORT_SYMBOL(md_unregister_thread);
+EXPORT_SYMBOL(md_wakeup_thread);
+EXPORT_SYMBOL(md_print_devices);
+EXPORT_SYMBOL(md_check_recovery);
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/mktables.c b/drivers/md/mktables.c
new file mode 100644
index 000000000000..adef299908cf
--- /dev/null
+++ b/drivers/md/mktables.c
@@ -0,0 +1,125 @@
+#ident "$Id: mktables.c,v 1.2 2002/12/12 22:41:27 hpa Exp $"
+/* ----------------------------------------------------------------------- *
+ *
+ * Copyright 2002 H. Peter Anvin - All Rights Reserved
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+ * Bostom MA 02111-1307, USA; either version 2 of the License, or
+ * (at your option) any later version; incorporated herein by reference.
+ *
+ * ----------------------------------------------------------------------- */
+
+/*
+ * mktables.c
+ *
+ * Make RAID-6 tables. This is a host user space program to be run at
+ * compile time.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <inttypes.h>
+#include <stdlib.h>
+#include <time.h>
+
+static uint8_t gfmul(uint8_t a, uint8_t b)
+{
+ uint8_t v = 0;
+
+ while ( b ) {
+ if ( b & 1 ) v ^= a;
+ a = (a << 1) ^ (a & 0x80 ? 0x1d : 0);
+ b >>= 1;
+ }
+ return v;
+}
+
+static uint8_t gfpow(uint8_t a, int b)
+{
+ uint8_t v = 1;
+
+ b %= 255;
+ if ( b < 0 )
+ b += 255;
+
+ while ( b ) {
+ if ( b & 1 ) v = gfmul(v,a);
+ a = gfmul(a,a);
+ b >>= 1;
+ }
+ return v;
+}
+
+int main(int argc, char *argv[])
+{
+ int i, j, k;
+ uint8_t v;
+ uint8_t exptbl[256], invtbl[256];
+
+ printf("#include \"raid6.h\"\n");
+
+ /* Compute multiplication table */
+ printf("\nconst u8 __attribute__((aligned(256)))\n"
+ "raid6_gfmul[256][256] =\n"
+ "{\n");
+ for ( i = 0 ; i < 256 ; i++ ) {
+ printf("\t{\n");
+ for ( j = 0 ; j < 256 ; j += 8 ) {
+ printf("\t\t");
+ for ( k = 0 ; k < 8 ; k++ ) {
+ printf("0x%02x, ", gfmul(i,j+k));
+ }
+ printf("\n");
+ }
+ printf("\t},\n");
+ }
+ printf("};\n");
+
+ /* Compute power-of-2 table (exponent) */
+ v = 1;
+ printf("\nconst u8 __attribute__((aligned(256)))\n"
+ "raid6_gfexp[256] =\n"
+ "{\n");
+ for ( i = 0 ; i < 256 ; i += 8 ) {
+ printf("\t");
+ for ( j = 0 ; j < 8 ; j++ ) {
+ exptbl[i+j] = v;
+ printf("0x%02x, ", v);
+ v = gfmul(v,2);
+ if ( v == 1 ) v = 0; /* For entry 255, not a real entry */
+ }
+ printf("\n");
+ }
+ printf("};\n");
+
+ /* Compute inverse table x^-1 == x^254 */
+ printf("\nconst u8 __attribute__((aligned(256)))\n"
+ "raid6_gfinv[256] =\n"
+ "{\n");
+ for ( i = 0 ; i < 256 ; i += 8 ) {
+ printf("\t");
+ for ( j = 0 ; j < 8 ; j++ ) {
+ invtbl[i+j] = v = gfpow(i+j,254);
+ printf("0x%02x, ", v);
+ }
+ printf("\n");
+ }
+ printf("};\n");
+
+ /* Compute inv(2^x + 1) (exponent-xor-inverse) table */
+ printf("\nconst u8 __attribute__((aligned(256)))\n"
+ "raid6_gfexi[256] =\n"
+ "{\n");
+ for ( i = 0 ; i < 256 ; i += 8 ) {
+ printf("\t");
+ for ( j = 0 ; j < 8 ; j++ ) {
+ printf("0x%02x, ", invtbl[exptbl[i+j]^1]);
+ }
+ printf("\n");
+ }
+ printf("};\n\n");
+
+ return 0;
+}
diff --git a/drivers/md/multipath.c b/drivers/md/multipath.c
new file mode 100644
index 000000000000..c9b134cd1532
--- /dev/null
+++ b/drivers/md/multipath.c
@@ -0,0 +1,584 @@
+/*
+ * multipath.c : Multiple Devices driver for Linux
+ *
+ * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat
+ *
+ * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman
+ *
+ * MULTIPATH management functions.
+ *
+ * derived from raid1.c.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * (for example /usr/src/linux/COPYING); if not, write to the Free
+ * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/raid/multipath.h>
+#include <linux/buffer_head.h>
+#include <asm/atomic.h>
+
+#define MAJOR_NR MD_MAJOR
+#define MD_DRIVER
+#define MD_PERSONALITY
+
+#define MAX_WORK_PER_DISK 128
+
+#define NR_RESERVED_BUFS 32
+
+
+static mdk_personality_t multipath_personality;
+
+
+static void *mp_pool_alloc(unsigned int __nocast gfp_flags, void *data)
+{
+ struct multipath_bh *mpb;
+ mpb = kmalloc(sizeof(*mpb), gfp_flags);
+ if (mpb)
+ memset(mpb, 0, sizeof(*mpb));
+ return mpb;
+}
+
+static void mp_pool_free(void *mpb, void *data)
+{
+ kfree(mpb);
+}
+
+static int multipath_map (multipath_conf_t *conf)
+{
+ int i, disks = conf->raid_disks;
+
+ /*
+ * Later we do read balancing on the read side
+ * now we use the first available disk.
+ */
+
+ rcu_read_lock();
+ for (i = 0; i < disks; i++) {
+ mdk_rdev_t *rdev = conf->multipaths[i].rdev;
+ if (rdev && rdev->in_sync) {
+ atomic_inc(&rdev->nr_pending);
+ rcu_read_unlock();
+ return i;
+ }
+ }
+ rcu_read_unlock();
+
+ printk(KERN_ERR "multipath_map(): no more operational IO paths?\n");
+ return (-1);
+}
+
+static void multipath_reschedule_retry (struct multipath_bh *mp_bh)
+{
+ unsigned long flags;
+ mddev_t *mddev = mp_bh->mddev;
+ multipath_conf_t *conf = mddev_to_conf(mddev);
+
+ spin_lock_irqsave(&conf->device_lock, flags);
+ list_add(&mp_bh->retry_list, &conf->retry_list);
+ spin_unlock_irqrestore(&conf->device_lock, flags);
+ md_wakeup_thread(mddev->thread);
+}
+
+
+/*
+ * multipath_end_bh_io() is called when we have finished servicing a multipathed
+ * operation and are ready to return a success/failure code to the buffer
+ * cache layer.
+ */
+static void multipath_end_bh_io (struct multipath_bh *mp_bh, int err)
+{
+ struct bio *bio = mp_bh->master_bio;
+ multipath_conf_t *conf = mddev_to_conf(mp_bh->mddev);
+
+ bio_endio(bio, bio->bi_size, err);
+ mempool_free(mp_bh, conf->pool);
+}
+
+int multipath_end_request(struct bio *bio, unsigned int bytes_done, int error)
+{
+ int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
+ struct multipath_bh * mp_bh = (struct multipath_bh *)(bio->bi_private);
+ multipath_conf_t *conf = mddev_to_conf(mp_bh->mddev);
+ mdk_rdev_t *rdev = conf->multipaths[mp_bh->path].rdev;
+
+ if (bio->bi_size)
+ return 1;
+
+ if (uptodate)
+ multipath_end_bh_io(mp_bh, 0);
+ else if (!bio_rw_ahead(bio)) {
+ /*
+ * oops, IO error:
+ */
+ char b[BDEVNAME_SIZE];
+ md_error (mp_bh->mddev, rdev);
+ printk(KERN_ERR "multipath: %s: rescheduling sector %llu\n",
+ bdevname(rdev->bdev,b),
+ (unsigned long long)bio->bi_sector);
+ multipath_reschedule_retry(mp_bh);
+ } else
+ multipath_end_bh_io(mp_bh, error);
+ rdev_dec_pending(rdev, conf->mddev);
+ return 0;
+}
+
+static void unplug_slaves(mddev_t *mddev)
+{
+ multipath_conf_t *conf = mddev_to_conf(mddev);
+ int i;
+
+ rcu_read_lock();
+ for (i=0; i<mddev->raid_disks; i++) {
+ mdk_rdev_t *rdev = conf->multipaths[i].rdev;
+ if (rdev && !rdev->faulty && atomic_read(&rdev->nr_pending)) {
+ request_queue_t *r_queue = bdev_get_queue(rdev->bdev);
+
+ atomic_inc(&rdev->nr_pending);
+ rcu_read_unlock();
+
+ if (r_queue->unplug_fn)
+ r_queue->unplug_fn(r_queue);
+
+ rdev_dec_pending(rdev, mddev);
+ rcu_read_lock();
+ }
+ }
+ rcu_read_unlock();
+}
+
+static void multipath_unplug(request_queue_t *q)
+{
+ unplug_slaves(q->queuedata);
+}
+
+
+static int multipath_make_request (request_queue_t *q, struct bio * bio)
+{
+ mddev_t *mddev = q->queuedata;
+ multipath_conf_t *conf = mddev_to_conf(mddev);
+ struct multipath_bh * mp_bh;
+ struct multipath_info *multipath;
+
+ mp_bh = mempool_alloc(conf->pool, GFP_NOIO);
+
+ mp_bh->master_bio = bio;
+ mp_bh->mddev = mddev;
+
+ if (bio_data_dir(bio)==WRITE) {
+ disk_stat_inc(mddev->gendisk, writes);
+ disk_stat_add(mddev->gendisk, write_sectors, bio_sectors(bio));
+ } else {
+ disk_stat_inc(mddev->gendisk, reads);
+ disk_stat_add(mddev->gendisk, read_sectors, bio_sectors(bio));
+ }
+
+ mp_bh->path = multipath_map(conf);
+ if (mp_bh->path < 0) {
+ bio_endio(bio, bio->bi_size, -EIO);
+ mempool_free(mp_bh, conf->pool);
+ return 0;
+ }
+ multipath = conf->multipaths + mp_bh->path;
+
+ mp_bh->bio = *bio;
+ mp_bh->bio.bi_sector += multipath->rdev->data_offset;
+ mp_bh->bio.bi_bdev = multipath->rdev->bdev;
+ mp_bh->bio.bi_rw |= (1 << BIO_RW_FAILFAST);
+ mp_bh->bio.bi_end_io = multipath_end_request;
+ mp_bh->bio.bi_private = mp_bh;
+ generic_make_request(&mp_bh->bio);
+ return 0;
+}
+
+static void multipath_status (struct seq_file *seq, mddev_t *mddev)
+{
+ multipath_conf_t *conf = mddev_to_conf(mddev);
+ int i;
+
+ seq_printf (seq, " [%d/%d] [", conf->raid_disks,
+ conf->working_disks);
+ for (i = 0; i < conf->raid_disks; i++)
+ seq_printf (seq, "%s",
+ conf->multipaths[i].rdev &&
+ conf->multipaths[i].rdev->in_sync ? "U" : "_");
+ seq_printf (seq, "]");
+}
+
+static int multipath_issue_flush(request_queue_t *q, struct gendisk *disk,
+ sector_t *error_sector)
+{
+ mddev_t *mddev = q->queuedata;
+ multipath_conf_t *conf = mddev_to_conf(mddev);
+ int i, ret = 0;
+
+ rcu_read_lock();
+ for (i=0; i<mddev->raid_disks && ret == 0; i++) {
+ mdk_rdev_t *rdev = conf->multipaths[i].rdev;
+ if (rdev && !rdev->faulty) {
+ struct block_device *bdev = rdev->bdev;
+ request_queue_t *r_queue = bdev_get_queue(bdev);
+
+ if (!r_queue->issue_flush_fn)
+ ret = -EOPNOTSUPP;
+ else {
+ atomic_inc(&rdev->nr_pending);
+ rcu_read_unlock();
+ ret = r_queue->issue_flush_fn(r_queue, bdev->bd_disk,
+ error_sector);
+ rdev_dec_pending(rdev, mddev);
+ rcu_read_lock();
+ }
+ }
+ }
+ rcu_read_unlock();
+ return ret;
+}
+
+/*
+ * Careful, this can execute in IRQ contexts as well!
+ */
+static void multipath_error (mddev_t *mddev, mdk_rdev_t *rdev)
+{
+ multipath_conf_t *conf = mddev_to_conf(mddev);
+
+ if (conf->working_disks <= 1) {
+ /*
+ * Uh oh, we can do nothing if this is our last path, but
+ * first check if this is a queued request for a device
+ * which has just failed.
+ */
+ printk(KERN_ALERT
+ "multipath: only one IO path left and IO error.\n");
+ /* leave it active... it's all we have */
+ } else {
+ /*
+ * Mark disk as unusable
+ */
+ if (!rdev->faulty) {
+ char b[BDEVNAME_SIZE];
+ rdev->in_sync = 0;
+ rdev->faulty = 1;
+ mddev->sb_dirty = 1;
+ conf->working_disks--;
+ printk(KERN_ALERT "multipath: IO failure on %s,"
+ " disabling IO path. \n Operation continuing"
+ " on %d IO paths.\n",
+ bdevname (rdev->bdev,b),
+ conf->working_disks);
+ }
+ }
+}
+
+static void print_multipath_conf (multipath_conf_t *conf)
+{
+ int i;
+ struct multipath_info *tmp;
+
+ printk("MULTIPATH conf printout:\n");
+ if (!conf) {
+ printk("(conf==NULL)\n");
+ return;
+ }
+ printk(" --- wd:%d rd:%d\n", conf->working_disks,
+ conf->raid_disks);
+
+ for (i = 0; i < conf->raid_disks; i++) {
+ char b[BDEVNAME_SIZE];
+ tmp = conf->multipaths + i;
+ if (tmp->rdev)
+ printk(" disk%d, o:%d, dev:%s\n",
+ i,!tmp->rdev->faulty,
+ bdevname(tmp->rdev->bdev,b));
+ }
+}
+
+
+static int multipath_add_disk(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+ multipath_conf_t *conf = mddev->private;
+ int found = 0;
+ int path;
+ struct multipath_info *p;
+
+ print_multipath_conf(conf);
+
+ for (path=0; path<mddev->raid_disks; path++)
+ if ((p=conf->multipaths+path)->rdev == NULL) {
+ blk_queue_stack_limits(mddev->queue,
+ rdev->bdev->bd_disk->queue);
+
+ /* as we don't honour merge_bvec_fn, we must never risk
+ * violating it, so limit ->max_sector to one PAGE, as
+ * a one page request is never in violation.
+ * (Note: it is very unlikely that a device with
+ * merge_bvec_fn will be involved in multipath.)
+ */
+ if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
+ mddev->queue->max_sectors > (PAGE_SIZE>>9))
+ blk_queue_max_sectors(mddev->queue, PAGE_SIZE>>9);
+
+ conf->working_disks++;
+ rdev->raid_disk = path;
+ rdev->in_sync = 1;
+ p->rdev = rdev;
+ found = 1;
+ }
+
+ print_multipath_conf(conf);
+ return found;
+}
+
+static int multipath_remove_disk(mddev_t *mddev, int number)
+{
+ multipath_conf_t *conf = mddev->private;
+ int err = 0;
+ mdk_rdev_t *rdev;
+ struct multipath_info *p = conf->multipaths + number;
+
+ print_multipath_conf(conf);
+
+ rdev = p->rdev;
+ if (rdev) {
+ if (rdev->in_sync ||
+ atomic_read(&rdev->nr_pending)) {
+ printk(KERN_ERR "hot-remove-disk, slot %d is identified" " but is still operational!\n", number);
+ err = -EBUSY;
+ goto abort;
+ }
+ p->rdev = NULL;
+ synchronize_kernel();
+ if (atomic_read(&rdev->nr_pending)) {
+ /* lost the race, try later */
+ err = -EBUSY;
+ p->rdev = rdev;
+ }
+ }
+abort:
+
+ print_multipath_conf(conf);
+ return err;
+}
+
+
+
+/*
+ * This is a kernel thread which:
+ *
+ * 1. Retries failed read operations on working multipaths.
+ * 2. Updates the raid superblock when problems encounter.
+ * 3. Performs writes following reads for array syncronising.
+ */
+
+static void multipathd (mddev_t *mddev)
+{
+ struct multipath_bh *mp_bh;
+ struct bio *bio;
+ unsigned long flags;
+ multipath_conf_t *conf = mddev_to_conf(mddev);
+ struct list_head *head = &conf->retry_list;
+
+ md_check_recovery(mddev);
+ for (;;) {
+ char b[BDEVNAME_SIZE];
+ spin_lock_irqsave(&conf->device_lock, flags);
+ if (list_empty(head))
+ break;
+ mp_bh = list_entry(head->prev, struct multipath_bh, retry_list);
+ list_del(head->prev);
+ spin_unlock_irqrestore(&conf->device_lock, flags);
+
+ bio = &mp_bh->bio;
+ bio->bi_sector = mp_bh->master_bio->bi_sector;
+
+ if ((mp_bh->path = multipath_map (conf))<0) {
+ printk(KERN_ALERT "multipath: %s: unrecoverable IO read"
+ " error for block %llu\n",
+ bdevname(bio->bi_bdev,b),
+ (unsigned long long)bio->bi_sector);
+ multipath_end_bh_io(mp_bh, -EIO);
+ } else {
+ printk(KERN_ERR "multipath: %s: redirecting sector %llu"
+ " to another IO path\n",
+ bdevname(bio->bi_bdev,b),
+ (unsigned long long)bio->bi_sector);
+ *bio = *(mp_bh->master_bio);
+ bio->bi_sector += conf->multipaths[mp_bh->path].rdev->data_offset;
+ bio->bi_bdev = conf->multipaths[mp_bh->path].rdev->bdev;
+ bio->bi_rw |= (1 << BIO_RW_FAILFAST);
+ bio->bi_end_io = multipath_end_request;
+ bio->bi_private = mp_bh;
+ generic_make_request(bio);
+ }
+ }
+ spin_unlock_irqrestore(&conf->device_lock, flags);
+}
+
+static int multipath_run (mddev_t *mddev)
+{
+ multipath_conf_t *conf;
+ int disk_idx;
+ struct multipath_info *disk;
+ mdk_rdev_t *rdev;
+ struct list_head *tmp;
+
+ if (mddev->level != LEVEL_MULTIPATH) {
+ printk("multipath: %s: raid level not set to multipath IO (%d)\n",
+ mdname(mddev), mddev->level);
+ goto out;
+ }
+ /*
+ * copy the already verified devices into our private MULTIPATH
+ * bookkeeping area. [whatever we allocate in multipath_run(),
+ * should be freed in multipath_stop()]
+ */
+
+ conf = kmalloc(sizeof(multipath_conf_t), GFP_KERNEL);
+ mddev->private = conf;
+ if (!conf) {
+ printk(KERN_ERR
+ "multipath: couldn't allocate memory for %s\n",
+ mdname(mddev));
+ goto out;
+ }
+ memset(conf, 0, sizeof(*conf));
+
+ conf->multipaths = kmalloc(sizeof(struct multipath_info)*mddev->raid_disks,
+ GFP_KERNEL);
+ if (!conf->multipaths) {
+ printk(KERN_ERR
+ "multipath: couldn't allocate memory for %s\n",
+ mdname(mddev));
+ goto out_free_conf;
+ }
+ memset(conf->multipaths, 0, sizeof(struct multipath_info)*mddev->raid_disks);
+
+ mddev->queue->unplug_fn = multipath_unplug;
+
+ mddev->queue->issue_flush_fn = multipath_issue_flush;
+
+ conf->working_disks = 0;
+ ITERATE_RDEV(mddev,rdev,tmp) {
+ disk_idx = rdev->raid_disk;
+ if (disk_idx < 0 ||
+ disk_idx >= mddev->raid_disks)
+ continue;
+
+ disk = conf->multipaths + disk_idx;
+ disk->rdev = rdev;
+
+ blk_queue_stack_limits(mddev->queue,
+ rdev->bdev->bd_disk->queue);
+ /* as we don't honour merge_bvec_fn, we must never risk
+ * violating it, not that we ever expect a device with
+ * a merge_bvec_fn to be involved in multipath */
+ if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
+ mddev->queue->max_sectors > (PAGE_SIZE>>9))
+ blk_queue_max_sectors(mddev->queue, PAGE_SIZE>>9);
+
+ if (!rdev->faulty)
+ conf->working_disks++;
+ }
+
+ conf->raid_disks = mddev->raid_disks;
+ mddev->sb_dirty = 1;
+ conf->mddev = mddev;
+ spin_lock_init(&conf->device_lock);
+ INIT_LIST_HEAD(&conf->retry_list);
+
+ if (!conf->working_disks) {
+ printk(KERN_ERR "multipath: no operational IO paths for %s\n",
+ mdname(mddev));
+ goto out_free_conf;
+ }
+ mddev->degraded = conf->raid_disks = conf->working_disks;
+
+ conf->pool = mempool_create(NR_RESERVED_BUFS,
+ mp_pool_alloc, mp_pool_free,
+ NULL);
+ if (conf->pool == NULL) {
+ printk(KERN_ERR
+ "multipath: couldn't allocate memory for %s\n",
+ mdname(mddev));
+ goto out_free_conf;
+ }
+
+ {
+ mddev->thread = md_register_thread(multipathd, mddev, "%s_multipath");
+ if (!mddev->thread) {
+ printk(KERN_ERR "multipath: couldn't allocate thread"
+ " for %s\n", mdname(mddev));
+ goto out_free_conf;
+ }
+ }
+
+ printk(KERN_INFO
+ "multipath: array %s active with %d out of %d IO paths\n",
+ mdname(mddev), conf->working_disks, mddev->raid_disks);
+ /*
+ * Ok, everything is just fine now
+ */
+ mddev->array_size = mddev->size;
+ return 0;
+
+out_free_conf:
+ if (conf->pool)
+ mempool_destroy(conf->pool);
+ if (conf->multipaths)
+ kfree(conf->multipaths);
+ kfree(conf);
+ mddev->private = NULL;
+out:
+ return -EIO;
+}
+
+
+static int multipath_stop (mddev_t *mddev)
+{
+ multipath_conf_t *conf = mddev_to_conf(mddev);
+
+ md_unregister_thread(mddev->thread);
+ mddev->thread = NULL;
+ blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
+ mempool_destroy(conf->pool);
+ kfree(conf->multipaths);
+ kfree(conf);
+ mddev->private = NULL;
+ return 0;
+}
+
+static mdk_personality_t multipath_personality=
+{
+ .name = "multipath",
+ .owner = THIS_MODULE,
+ .make_request = multipath_make_request,
+ .run = multipath_run,
+ .stop = multipath_stop,
+ .status = multipath_status,
+ .error_handler = multipath_error,
+ .hot_add_disk = multipath_add_disk,
+ .hot_remove_disk= multipath_remove_disk,
+};
+
+static int __init multipath_init (void)
+{
+ return register_md_personality (MULTIPATH, &multipath_personality);
+}
+
+static void __exit multipath_exit (void)
+{
+ unregister_md_personality (MULTIPATH);
+}
+
+module_init(multipath_init);
+module_exit(multipath_exit);
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("md-personality-7"); /* MULTIPATH */
diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c
new file mode 100644
index 000000000000..e7d934eca06f
--- /dev/null
+++ b/drivers/md/raid0.c
@@ -0,0 +1,539 @@
+/*
+ raid0.c : Multiple Devices driver for Linux
+ Copyright (C) 1994-96 Marc ZYNGIER
+ <zyngier@ufr-info-p7.ibp.fr> or
+ <maz@gloups.fdn.fr>
+ Copyright (C) 1999, 2000 Ingo Molnar, Red Hat
+
+
+ RAID-0 management functions.
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2, or (at your option)
+ any later version.
+
+ You should have received a copy of the GNU General Public License
+ (for example /usr/src/linux/COPYING); if not, write to the Free
+ Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+*/
+
+#include <linux/module.h>
+#include <linux/raid/raid0.h>
+
+#define MAJOR_NR MD_MAJOR
+#define MD_DRIVER
+#define MD_PERSONALITY
+
+static void raid0_unplug(request_queue_t *q)
+{
+ mddev_t *mddev = q->queuedata;
+ raid0_conf_t *conf = mddev_to_conf(mddev);
+ mdk_rdev_t **devlist = conf->strip_zone[0].dev;
+ int i;
+
+ for (i=0; i<mddev->raid_disks; i++) {
+ request_queue_t *r_queue = bdev_get_queue(devlist[i]->bdev);
+
+ if (r_queue->unplug_fn)
+ r_queue->unplug_fn(r_queue);
+ }
+}
+
+static int raid0_issue_flush(request_queue_t *q, struct gendisk *disk,
+ sector_t *error_sector)
+{
+ mddev_t *mddev = q->queuedata;
+ raid0_conf_t *conf = mddev_to_conf(mddev);
+ mdk_rdev_t **devlist = conf->strip_zone[0].dev;
+ int i, ret = 0;
+
+ for (i=0; i<mddev->raid_disks && ret == 0; i++) {
+ struct block_device *bdev = devlist[i]->bdev;
+ request_queue_t *r_queue = bdev_get_queue(bdev);
+
+ if (!r_queue->issue_flush_fn)
+ ret = -EOPNOTSUPP;
+ else
+ ret = r_queue->issue_flush_fn(r_queue, bdev->bd_disk, error_sector);
+ }
+ return ret;
+}
+
+
+static int create_strip_zones (mddev_t *mddev)
+{
+ int i, c, j;
+ sector_t current_offset, curr_zone_offset;
+ sector_t min_spacing;
+ raid0_conf_t *conf = mddev_to_conf(mddev);
+ mdk_rdev_t *smallest, *rdev1, *rdev2, *rdev;
+ struct list_head *tmp1, *tmp2;
+ struct strip_zone *zone;
+ int cnt;
+ char b[BDEVNAME_SIZE];
+
+ /*
+ * The number of 'same size groups'
+ */
+ conf->nr_strip_zones = 0;
+
+ ITERATE_RDEV(mddev,rdev1,tmp1) {
+ printk("raid0: looking at %s\n",
+ bdevname(rdev1->bdev,b));
+ c = 0;
+ ITERATE_RDEV(mddev,rdev2,tmp2) {
+ printk("raid0: comparing %s(%llu)",
+ bdevname(rdev1->bdev,b),
+ (unsigned long long)rdev1->size);
+ printk(" with %s(%llu)\n",
+ bdevname(rdev2->bdev,b),
+ (unsigned long long)rdev2->size);
+ if (rdev2 == rdev1) {
+ printk("raid0: END\n");
+ break;
+ }
+ if (rdev2->size == rdev1->size)
+ {
+ /*
+ * Not unique, don't count it as a new
+ * group
+ */
+ printk("raid0: EQUAL\n");
+ c = 1;
+ break;
+ }
+ printk("raid0: NOT EQUAL\n");
+ }
+ if (!c) {
+ printk("raid0: ==> UNIQUE\n");
+ conf->nr_strip_zones++;
+ printk("raid0: %d zones\n", conf->nr_strip_zones);
+ }
+ }
+ printk("raid0: FINAL %d zones\n", conf->nr_strip_zones);
+
+ conf->strip_zone = kmalloc(sizeof(struct strip_zone)*
+ conf->nr_strip_zones, GFP_KERNEL);
+ if (!conf->strip_zone)
+ return 1;
+ conf->devlist = kmalloc(sizeof(mdk_rdev_t*)*
+ conf->nr_strip_zones*mddev->raid_disks,
+ GFP_KERNEL);
+ if (!conf->devlist)
+ return 1;
+
+ memset(conf->strip_zone, 0,sizeof(struct strip_zone)*
+ conf->nr_strip_zones);
+ memset(conf->devlist, 0,
+ sizeof(mdk_rdev_t*) * conf->nr_strip_zones * mddev->raid_disks);
+
+ /* The first zone must contain all devices, so here we check that
+ * there is a proper alignment of slots to devices and find them all
+ */
+ zone = &conf->strip_zone[0];
+ cnt = 0;
+ smallest = NULL;
+ zone->dev = conf->devlist;
+ ITERATE_RDEV(mddev, rdev1, tmp1) {
+ int j = rdev1->raid_disk;
+
+ if (j < 0 || j >= mddev->raid_disks) {
+ printk("raid0: bad disk number %d - aborting!\n", j);
+ goto abort;
+ }
+ if (zone->dev[j]) {
+ printk("raid0: multiple devices for %d - aborting!\n",
+ j);
+ goto abort;
+ }
+ zone->dev[j] = rdev1;
+
+ blk_queue_stack_limits(mddev->queue,
+ rdev1->bdev->bd_disk->queue);
+ /* as we don't honour merge_bvec_fn, we must never risk
+ * violating it, so limit ->max_sector to one PAGE, as
+ * a one page request is never in violation.
+ */
+
+ if (rdev1->bdev->bd_disk->queue->merge_bvec_fn &&
+ mddev->queue->max_sectors > (PAGE_SIZE>>9))
+ blk_queue_max_sectors(mddev->queue, PAGE_SIZE>>9);
+
+ if (!smallest || (rdev1->size <smallest->size))
+ smallest = rdev1;
+ cnt++;
+ }
+ if (cnt != mddev->raid_disks) {
+ printk("raid0: too few disks (%d of %d) - aborting!\n",
+ cnt, mddev->raid_disks);
+ goto abort;
+ }
+ zone->nb_dev = cnt;
+ zone->size = smallest->size * cnt;
+ zone->zone_offset = 0;
+
+ current_offset = smallest->size;
+ curr_zone_offset = zone->size;
+
+ /* now do the other zones */
+ for (i = 1; i < conf->nr_strip_zones; i++)
+ {
+ zone = conf->strip_zone + i;
+ zone->dev = conf->strip_zone[i-1].dev + mddev->raid_disks;
+
+ printk("raid0: zone %d\n", i);
+ zone->dev_offset = current_offset;
+ smallest = NULL;
+ c = 0;
+
+ for (j=0; j<cnt; j++) {
+ char b[BDEVNAME_SIZE];
+ rdev = conf->strip_zone[0].dev[j];
+ printk("raid0: checking %s ...", bdevname(rdev->bdev,b));
+ if (rdev->size > current_offset)
+ {
+ printk(" contained as device %d\n", c);
+ zone->dev[c] = rdev;
+ c++;
+ if (!smallest || (rdev->size <smallest->size)) {
+ smallest = rdev;
+ printk(" (%llu) is smallest!.\n",
+ (unsigned long long)rdev->size);
+ }
+ } else
+ printk(" nope.\n");
+ }
+
+ zone->nb_dev = c;
+ zone->size = (smallest->size - current_offset) * c;
+ printk("raid0: zone->nb_dev: %d, size: %llu\n",
+ zone->nb_dev, (unsigned long long)zone->size);
+
+ zone->zone_offset = curr_zone_offset;
+ curr_zone_offset += zone->size;
+
+ current_offset = smallest->size;
+ printk("raid0: current zone offset: %llu\n",
+ (unsigned long long)current_offset);
+ }
+
+ /* Now find appropriate hash spacing.
+ * We want a number which causes most hash entries to cover
+ * at most two strips, but the hash table must be at most
+ * 1 PAGE. We choose the smallest strip, or contiguous collection
+ * of strips, that has big enough size. We never consider the last
+ * strip though as it's size has no bearing on the efficacy of the hash
+ * table.
+ */
+ conf->hash_spacing = curr_zone_offset;
+ min_spacing = curr_zone_offset;
+ sector_div(min_spacing, PAGE_SIZE/sizeof(struct strip_zone*));
+ for (i=0; i < conf->nr_strip_zones-1; i++) {
+ sector_t sz = 0;
+ for (j=i; j<conf->nr_strip_zones-1 &&
+ sz < min_spacing ; j++)
+ sz += conf->strip_zone[j].size;
+ if (sz >= min_spacing && sz < conf->hash_spacing)
+ conf->hash_spacing = sz;
+ }
+
+ mddev->queue->unplug_fn = raid0_unplug;
+
+ mddev->queue->issue_flush_fn = raid0_issue_flush;
+
+ printk("raid0: done.\n");
+ return 0;
+ abort:
+ return 1;
+}
+
+/**
+ * raid0_mergeable_bvec -- tell bio layer if a two requests can be merged
+ * @q: request queue
+ * @bio: the buffer head that's been built up so far
+ * @biovec: the request that could be merged to it.
+ *
+ * Return amount of bytes we can accept at this offset
+ */
+static int raid0_mergeable_bvec(request_queue_t *q, struct bio *bio, struct bio_vec *biovec)
+{
+ mddev_t *mddev = q->queuedata;
+ sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev);
+ int max;
+ unsigned int chunk_sectors = mddev->chunk_size >> 9;
+ unsigned int bio_sectors = bio->bi_size >> 9;
+
+ max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
+ if (max < 0) max = 0; /* bio_add cannot handle a negative return */
+ if (max <= biovec->bv_len && bio_sectors == 0)
+ return biovec->bv_len;
+ else
+ return max;
+}
+
+static int raid0_run (mddev_t *mddev)
+{
+ unsigned cur=0, i=0, nb_zone;
+ s64 size;
+ raid0_conf_t *conf;
+ mdk_rdev_t *rdev;
+ struct list_head *tmp;
+
+ printk("%s: setting max_sectors to %d, segment boundary to %d\n",
+ mdname(mddev),
+ mddev->chunk_size >> 9,
+ (mddev->chunk_size>>1)-1);
+ blk_queue_max_sectors(mddev->queue, mddev->chunk_size >> 9);
+ blk_queue_segment_boundary(mddev->queue, (mddev->chunk_size>>1) - 1);
+
+ conf = kmalloc(sizeof (raid0_conf_t), GFP_KERNEL);
+ if (!conf)
+ goto out;
+ mddev->private = (void *)conf;
+
+ conf->strip_zone = NULL;
+ conf->devlist = NULL;
+ if (create_strip_zones (mddev))
+ goto out_free_conf;
+
+ /* calculate array device size */
+ mddev->array_size = 0;
+ ITERATE_RDEV(mddev,rdev,tmp)
+ mddev->array_size += rdev->size;
+
+ printk("raid0 : md_size is %llu blocks.\n",
+ (unsigned long long)mddev->array_size);
+ printk("raid0 : conf->hash_spacing is %llu blocks.\n",
+ (unsigned long long)conf->hash_spacing);
+ {
+#if __GNUC__ < 3
+ volatile
+#endif
+ sector_t s = mddev->array_size;
+ sector_t space = conf->hash_spacing;
+ int round;
+ conf->preshift = 0;
+ if (sizeof(sector_t) > sizeof(unsigned long)) {
+ /*shift down space and s so that sector_div will work */
+ while (space > (sector_t) (~(unsigned long)0)) {
+ s >>= 1;
+ space >>= 1;
+ s += 1; /* force round-up */
+ conf->preshift++;
+ }
+ }
+ round = sector_div(s, (unsigned long)space) ? 1 : 0;
+ nb_zone = s + round;
+ }
+ printk("raid0 : nb_zone is %d.\n", nb_zone);
+
+ printk("raid0 : Allocating %Zd bytes for hash.\n",
+ nb_zone*sizeof(struct strip_zone*));
+ conf->hash_table = kmalloc (sizeof (struct strip_zone *)*nb_zone, GFP_KERNEL);
+ if (!conf->hash_table)
+ goto out_free_conf;
+ size = conf->strip_zone[cur].size;
+
+ for (i=0; i< nb_zone; i++) {
+ conf->hash_table[i] = conf->strip_zone + cur;
+ while (size <= conf->hash_spacing) {
+ cur++;
+ size += conf->strip_zone[cur].size;
+ }
+ size -= conf->hash_spacing;
+ }
+ if (conf->preshift) {
+ conf->hash_spacing >>= conf->preshift;
+ /* round hash_spacing up so when we divide by it, we
+ * err on the side of too-low, which is safest
+ */
+ conf->hash_spacing++;
+ }
+
+ /* calculate the max read-ahead size.
+ * For read-ahead of large files to be effective, we need to
+ * readahead at least twice a whole stripe. i.e. number of devices
+ * multiplied by chunk size times 2.
+ * If an individual device has an ra_pages greater than the
+ * chunk size, then we will not drive that device as hard as it
+ * wants. We consider this a configuration error: a larger
+ * chunksize should be used in that case.
+ */
+ {
+ int stripe = mddev->raid_disks * mddev->chunk_size / PAGE_CACHE_SIZE;
+ if (mddev->queue->backing_dev_info.ra_pages < 2* stripe)
+ mddev->queue->backing_dev_info.ra_pages = 2* stripe;
+ }
+
+
+ blk_queue_merge_bvec(mddev->queue, raid0_mergeable_bvec);
+ return 0;
+
+out_free_conf:
+ if (conf->strip_zone)
+ kfree(conf->strip_zone);
+ if (conf->devlist)
+ kfree (conf->devlist);
+ kfree(conf);
+ mddev->private = NULL;
+out:
+ return 1;
+}
+
+static int raid0_stop (mddev_t *mddev)
+{
+ raid0_conf_t *conf = mddev_to_conf(mddev);
+
+ blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
+ kfree (conf->hash_table);
+ conf->hash_table = NULL;
+ kfree (conf->strip_zone);
+ conf->strip_zone = NULL;
+ kfree (conf);
+ mddev->private = NULL;
+
+ return 0;
+}
+
+static int raid0_make_request (request_queue_t *q, struct bio *bio)
+{
+ mddev_t *mddev = q->queuedata;
+ unsigned int sect_in_chunk, chunksize_bits, chunk_size, chunk_sects;
+ raid0_conf_t *conf = mddev_to_conf(mddev);
+ struct strip_zone *zone;
+ mdk_rdev_t *tmp_dev;
+ unsigned long chunk;
+ sector_t block, rsect;
+
+ if (bio_data_dir(bio)==WRITE) {
+ disk_stat_inc(mddev->gendisk, writes);
+ disk_stat_add(mddev->gendisk, write_sectors, bio_sectors(bio));
+ } else {
+ disk_stat_inc(mddev->gendisk, reads);
+ disk_stat_add(mddev->gendisk, read_sectors, bio_sectors(bio));
+ }
+
+ chunk_size = mddev->chunk_size >> 10;
+ chunk_sects = mddev->chunk_size >> 9;
+ chunksize_bits = ffz(~chunk_size);
+ block = bio->bi_sector >> 1;
+
+
+ if (unlikely(chunk_sects < (bio->bi_sector & (chunk_sects - 1)) + (bio->bi_size >> 9))) {
+ struct bio_pair *bp;
+ /* Sanity check -- queue functions should prevent this happening */
+ if (bio->bi_vcnt != 1 ||
+ bio->bi_idx != 0)
+ goto bad_map;
+ /* This is a one page bio that upper layers
+ * refuse to split for us, so we need to split it.
+ */
+ bp = bio_split(bio, bio_split_pool, chunk_sects - (bio->bi_sector & (chunk_sects - 1)) );
+ if (raid0_make_request(q, &bp->bio1))
+ generic_make_request(&bp->bio1);
+ if (raid0_make_request(q, &bp->bio2))
+ generic_make_request(&bp->bio2);
+
+ bio_pair_release(bp);
+ return 0;
+ }
+
+
+ {
+#if __GNUC__ < 3
+ volatile
+#endif
+ sector_t x = block >> conf->preshift;
+ sector_div(x, (unsigned long)conf->hash_spacing);
+ zone = conf->hash_table[x];
+ }
+
+ while (block >= (zone->zone_offset + zone->size))
+ zone++;
+
+ sect_in_chunk = bio->bi_sector & ((chunk_size<<1) -1);
+
+
+ {
+ sector_t x = (block - zone->zone_offset) >> chunksize_bits;
+
+ sector_div(x, zone->nb_dev);
+ chunk = x;
+ BUG_ON(x != (sector_t)chunk);
+
+ x = block >> chunksize_bits;
+ tmp_dev = zone->dev[sector_div(x, zone->nb_dev)];
+ }
+ rsect = (((chunk << chunksize_bits) + zone->dev_offset)<<1)
+ + sect_in_chunk;
+
+ bio->bi_bdev = tmp_dev->bdev;
+ bio->bi_sector = rsect + tmp_dev->data_offset;
+
+ /*
+ * Let the main block layer submit the IO and resolve recursion:
+ */
+ return 1;
+
+bad_map:
+ printk("raid0_make_request bug: can't convert block across chunks"
+ " or bigger than %dk %llu %d\n", chunk_size,
+ (unsigned long long)bio->bi_sector, bio->bi_size >> 10);
+
+ bio_io_error(bio, bio->bi_size);
+ return 0;
+}
+
+static void raid0_status (struct seq_file *seq, mddev_t *mddev)
+{
+#undef MD_DEBUG
+#ifdef MD_DEBUG
+ int j, k, h;
+ char b[BDEVNAME_SIZE];
+ raid0_conf_t *conf = mddev_to_conf(mddev);
+
+ h = 0;
+ for (j = 0; j < conf->nr_strip_zones; j++) {
+ seq_printf(seq, " z%d", j);
+ if (conf->hash_table[h] == conf->strip_zone+j)
+ seq_printf("(h%d)", h++);
+ seq_printf(seq, "=[");
+ for (k = 0; k < conf->strip_zone[j].nb_dev; k++)
+ seq_printf (seq, "%s/", bdevname(
+ conf->strip_zone[j].dev[k]->bdev,b));
+
+ seq_printf (seq, "] zo=%d do=%d s=%d\n",
+ conf->strip_zone[j].zone_offset,
+ conf->strip_zone[j].dev_offset,
+ conf->strip_zone[j].size);
+ }
+#endif
+ seq_printf(seq, " %dk chunks", mddev->chunk_size/1024);
+ return;
+}
+
+static mdk_personality_t raid0_personality=
+{
+ .name = "raid0",
+ .owner = THIS_MODULE,
+ .make_request = raid0_make_request,
+ .run = raid0_run,
+ .stop = raid0_stop,
+ .status = raid0_status,
+};
+
+static int __init raid0_init (void)
+{
+ return register_md_personality (RAID0, &raid0_personality);
+}
+
+static void raid0_exit (void)
+{
+ unregister_md_personality (RAID0);
+}
+
+module_init(raid0_init);
+module_exit(raid0_exit);
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("md-personality-2"); /* RAID0 */
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
new file mode 100644
index 000000000000..a389394b52f6
--- /dev/null
+++ b/drivers/md/raid1.c
@@ -0,0 +1,1449 @@
+/*
+ * raid1.c : Multiple Devices driver for Linux
+ *
+ * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat
+ *
+ * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman
+ *
+ * RAID-1 management functions.
+ *
+ * Better read-balancing code written by Mika Kuoppala <miku@iki.fi>, 2000
+ *
+ * Fixes to reconstruction by Jakob Østergaard" <jakob@ostenfeld.dk>
+ * Various fixes by Neil Brown <neilb@cse.unsw.edu.au>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * (for example /usr/src/linux/COPYING); if not, write to the Free
+ * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/raid/raid1.h>
+
+/*
+ * Number of guaranteed r1bios in case of extreme VM load:
+ */
+#define NR_RAID1_BIOS 256
+
+static mdk_personality_t raid1_personality;
+
+static void unplug_slaves(mddev_t *mddev);
+
+
+static void * r1bio_pool_alloc(unsigned int __nocast gfp_flags, void *data)
+{
+ struct pool_info *pi = data;
+ r1bio_t *r1_bio;
+ int size = offsetof(r1bio_t, bios[pi->raid_disks]);
+
+ /* allocate a r1bio with room for raid_disks entries in the bios array */
+ r1_bio = kmalloc(size, gfp_flags);
+ if (r1_bio)
+ memset(r1_bio, 0, size);
+ else
+ unplug_slaves(pi->mddev);
+
+ return r1_bio;
+}
+
+static void r1bio_pool_free(void *r1_bio, void *data)
+{
+ kfree(r1_bio);
+}
+
+#define RESYNC_BLOCK_SIZE (64*1024)
+//#define RESYNC_BLOCK_SIZE PAGE_SIZE
+#define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)
+#define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
+#define RESYNC_WINDOW (2048*1024)
+
+static void * r1buf_pool_alloc(unsigned int __nocast gfp_flags, void *data)
+{
+ struct pool_info *pi = data;
+ struct page *page;
+ r1bio_t *r1_bio;
+ struct bio *bio;
+ int i, j;
+
+ r1_bio = r1bio_pool_alloc(gfp_flags, pi);
+ if (!r1_bio) {
+ unplug_slaves(pi->mddev);
+ return NULL;
+ }
+
+ /*
+ * Allocate bios : 1 for reading, n-1 for writing
+ */
+ for (j = pi->raid_disks ; j-- ; ) {
+ bio = bio_alloc(gfp_flags, RESYNC_PAGES);
+ if (!bio)
+ goto out_free_bio;
+ r1_bio->bios[j] = bio;
+ }
+ /*
+ * Allocate RESYNC_PAGES data pages and attach them to
+ * the first bio;
+ */
+ bio = r1_bio->bios[0];
+ for (i = 0; i < RESYNC_PAGES; i++) {
+ page = alloc_page(gfp_flags);
+ if (unlikely(!page))
+ goto out_free_pages;
+
+ bio->bi_io_vec[i].bv_page = page;
+ }
+
+ r1_bio->master_bio = NULL;
+
+ return r1_bio;
+
+out_free_pages:
+ for ( ; i > 0 ; i--)
+ __free_page(bio->bi_io_vec[i-1].bv_page);
+out_free_bio:
+ while ( ++j < pi->raid_disks )
+ bio_put(r1_bio->bios[j]);
+ r1bio_pool_free(r1_bio, data);
+ return NULL;
+}
+
+static void r1buf_pool_free(void *__r1_bio, void *data)
+{
+ struct pool_info *pi = data;
+ int i;
+ r1bio_t *r1bio = __r1_bio;
+ struct bio *bio = r1bio->bios[0];
+
+ for (i = 0; i < RESYNC_PAGES; i++) {
+ __free_page(bio->bi_io_vec[i].bv_page);
+ bio->bi_io_vec[i].bv_page = NULL;
+ }
+ for (i=0 ; i < pi->raid_disks; i++)
+ bio_put(r1bio->bios[i]);
+
+ r1bio_pool_free(r1bio, data);
+}
+
+static void put_all_bios(conf_t *conf, r1bio_t *r1_bio)
+{
+ int i;
+
+ for (i = 0; i < conf->raid_disks; i++) {
+ struct bio **bio = r1_bio->bios + i;
+ if (*bio)
+ bio_put(*bio);
+ *bio = NULL;
+ }
+}
+
+static inline void free_r1bio(r1bio_t *r1_bio)
+{
+ unsigned long flags;
+
+ conf_t *conf = mddev_to_conf(r1_bio->mddev);
+
+ /*
+ * Wake up any possible resync thread that waits for the device
+ * to go idle.
+ */
+ spin_lock_irqsave(&conf->resync_lock, flags);
+ if (!--conf->nr_pending) {
+ wake_up(&conf->wait_idle);
+ wake_up(&conf->wait_resume);
+ }
+ spin_unlock_irqrestore(&conf->resync_lock, flags);
+
+ put_all_bios(conf, r1_bio);
+ mempool_free(r1_bio, conf->r1bio_pool);
+}
+
+static inline void put_buf(r1bio_t *r1_bio)
+{
+ conf_t *conf = mddev_to_conf(r1_bio->mddev);
+ unsigned long flags;
+
+ mempool_free(r1_bio, conf->r1buf_pool);
+
+ spin_lock_irqsave(&conf->resync_lock, flags);
+ if (!conf->barrier)
+ BUG();
+ --conf->barrier;
+ wake_up(&conf->wait_resume);
+ wake_up(&conf->wait_idle);
+
+ if (!--conf->nr_pending) {
+ wake_up(&conf->wait_idle);
+ wake_up(&conf->wait_resume);
+ }
+ spin_unlock_irqrestore(&conf->resync_lock, flags);
+}
+
+static void reschedule_retry(r1bio_t *r1_bio)
+{
+ unsigned long flags;
+ mddev_t *mddev = r1_bio->mddev;
+ conf_t *conf = mddev_to_conf(mddev);
+
+ spin_lock_irqsave(&conf->device_lock, flags);
+ list_add(&r1_bio->retry_list, &conf->retry_list);
+ spin_unlock_irqrestore(&conf->device_lock, flags);
+
+ md_wakeup_thread(mddev->thread);
+}
+
+/*
+ * raid_end_bio_io() is called when we have finished servicing a mirrored
+ * operation and are ready to return a success/failure code to the buffer
+ * cache layer.
+ */
+static void raid_end_bio_io(r1bio_t *r1_bio)
+{
+ struct bio *bio = r1_bio->master_bio;
+
+ bio_endio(bio, bio->bi_size,
+ test_bit(R1BIO_Uptodate, &r1_bio->state) ? 0 : -EIO);
+ free_r1bio(r1_bio);
+}
+
+/*
+ * Update disk head position estimator based on IRQ completion info.
+ */
+static inline void update_head_pos(int disk, r1bio_t *r1_bio)
+{
+ conf_t *conf = mddev_to_conf(r1_bio->mddev);
+
+ conf->mirrors[disk].head_position =
+ r1_bio->sector + (r1_bio->sectors);
+}
+
+static int raid1_end_read_request(struct bio *bio, unsigned int bytes_done, int error)
+{
+ int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
+ r1bio_t * r1_bio = (r1bio_t *)(bio->bi_private);
+ int mirror;
+ conf_t *conf = mddev_to_conf(r1_bio->mddev);
+
+ if (bio->bi_size)
+ return 1;
+
+ mirror = r1_bio->read_disk;
+ /*
+ * this branch is our 'one mirror IO has finished' event handler:
+ */
+ if (!uptodate)
+ md_error(r1_bio->mddev, conf->mirrors[mirror].rdev);
+ else
+ /*
+ * Set R1BIO_Uptodate in our master bio, so that
+ * we will return a good error code for to the higher
+ * levels even if IO on some other mirrored buffer fails.
+ *
+ * The 'master' represents the composite IO operation to
+ * user-side. So if something waits for IO, then it will
+ * wait for the 'master' bio.
+ */
+ set_bit(R1BIO_Uptodate, &r1_bio->state);
+
+ update_head_pos(mirror, r1_bio);
+
+ /*
+ * we have only one bio on the read side
+ */
+ if (uptodate)
+ raid_end_bio_io(r1_bio);
+ else {
+ /*
+ * oops, read error:
+ */
+ char b[BDEVNAME_SIZE];
+ if (printk_ratelimit())
+ printk(KERN_ERR "raid1: %s: rescheduling sector %llu\n",
+ bdevname(conf->mirrors[mirror].rdev->bdev,b), (unsigned long long)r1_bio->sector);
+ reschedule_retry(r1_bio);
+ }
+
+ rdev_dec_pending(conf->mirrors[mirror].rdev, conf->mddev);
+ return 0;
+}
+
+static int raid1_end_write_request(struct bio *bio, unsigned int bytes_done, int error)
+{
+ int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
+ r1bio_t * r1_bio = (r1bio_t *)(bio->bi_private);
+ int mirror;
+ conf_t *conf = mddev_to_conf(r1_bio->mddev);
+
+ if (bio->bi_size)
+ return 1;
+
+ for (mirror = 0; mirror < conf->raid_disks; mirror++)
+ if (r1_bio->bios[mirror] == bio)
+ break;
+
+ /*
+ * this branch is our 'one mirror IO has finished' event handler:
+ */
+ if (!uptodate)
+ md_error(r1_bio->mddev, conf->mirrors[mirror].rdev);
+ else
+ /*
+ * Set R1BIO_Uptodate in our master bio, so that
+ * we will return a good error code for to the higher
+ * levels even if IO on some other mirrored buffer fails.
+ *
+ * The 'master' represents the composite IO operation to
+ * user-side. So if something waits for IO, then it will
+ * wait for the 'master' bio.
+ */
+ set_bit(R1BIO_Uptodate, &r1_bio->state);
+
+ update_head_pos(mirror, r1_bio);
+
+ /*
+ *
+ * Let's see if all mirrored write operations have finished
+ * already.
+ */
+ if (atomic_dec_and_test(&r1_bio->remaining)) {
+ md_write_end(r1_bio->mddev);
+ raid_end_bio_io(r1_bio);
+ }
+
+ rdev_dec_pending(conf->mirrors[mirror].rdev, conf->mddev);
+ return 0;
+}
+
+
+/*
+ * This routine returns the disk from which the requested read should
+ * be done. There is a per-array 'next expected sequential IO' sector
+ * number - if this matches on the next IO then we use the last disk.
+ * There is also a per-disk 'last know head position' sector that is
+ * maintained from IRQ contexts, both the normal and the resync IO
+ * completion handlers update this position correctly. If there is no
+ * perfect sequential match then we pick the disk whose head is closest.
+ *
+ * If there are 2 mirrors in the same 2 devices, performance degrades
+ * because position is mirror, not device based.
+ *
+ * The rdev for the device selected will have nr_pending incremented.
+ */
+static int read_balance(conf_t *conf, r1bio_t *r1_bio)
+{
+ const unsigned long this_sector = r1_bio->sector;
+ int new_disk = conf->last_used, disk = new_disk;
+ const int sectors = r1_bio->sectors;
+ sector_t new_distance, current_distance;
+ mdk_rdev_t *new_rdev, *rdev;
+
+ rcu_read_lock();
+ /*
+ * Check if it if we can balance. We can balance on the whole
+ * device if no resync is going on, or below the resync window.
+ * We take the first readable disk when above the resync window.
+ */
+ retry:
+ if (conf->mddev->recovery_cp < MaxSector &&
+ (this_sector + sectors >= conf->next_resync)) {
+ /* Choose the first operation device, for consistancy */
+ new_disk = 0;
+
+ while ((new_rdev=conf->mirrors[new_disk].rdev) == NULL ||
+ !new_rdev->in_sync) {
+ new_disk++;
+ if (new_disk == conf->raid_disks) {
+ new_disk = -1;
+ break;
+ }
+ }
+ goto rb_out;
+ }
+
+
+ /* make sure the disk is operational */
+ while ((new_rdev=conf->mirrors[new_disk].rdev) == NULL ||
+ !new_rdev->in_sync) {
+ if (new_disk <= 0)
+ new_disk = conf->raid_disks;
+ new_disk--;
+ if (new_disk == disk) {
+ new_disk = -1;
+ goto rb_out;
+ }
+ }
+ disk = new_disk;
+ /* now disk == new_disk == starting point for search */
+
+ /*
+ * Don't change to another disk for sequential reads:
+ */
+ if (conf->next_seq_sect == this_sector)
+ goto rb_out;
+ if (this_sector == conf->mirrors[new_disk].head_position)
+ goto rb_out;
+
+ current_distance = abs(this_sector - conf->mirrors[disk].head_position);
+
+ /* Find the disk whose head is closest */
+
+ do {
+ if (disk <= 0)
+ disk = conf->raid_disks;
+ disk--;
+
+ if ((rdev=conf->mirrors[disk].rdev) == NULL ||
+ !rdev->in_sync)
+ continue;
+
+ if (!atomic_read(&rdev->nr_pending)) {
+ new_disk = disk;
+ new_rdev = rdev;
+ break;
+ }
+ new_distance = abs(this_sector - conf->mirrors[disk].head_position);
+ if (new_distance < current_distance) {
+ current_distance = new_distance;
+ new_disk = disk;
+ new_rdev = rdev;
+ }
+ } while (disk != conf->last_used);
+
+rb_out:
+
+
+ if (new_disk >= 0) {
+ conf->next_seq_sect = this_sector + sectors;
+ conf->last_used = new_disk;
+ atomic_inc(&new_rdev->nr_pending);
+ if (!new_rdev->in_sync) {
+ /* cannot risk returning a device that failed
+ * before we inc'ed nr_pending
+ */
+ atomic_dec(&new_rdev->nr_pending);
+ goto retry;
+ }
+ }
+ rcu_read_unlock();
+
+ return new_disk;
+}
+
+static void unplug_slaves(mddev_t *mddev)
+{
+ conf_t *conf = mddev_to_conf(mddev);
+ int i;
+
+ rcu_read_lock();
+ for (i=0; i<mddev->raid_disks; i++) {
+ mdk_rdev_t *rdev = conf->mirrors[i].rdev;
+ if (rdev && !rdev->faulty && atomic_read(&rdev->nr_pending)) {
+ request_queue_t *r_queue = bdev_get_queue(rdev->bdev);
+
+ atomic_inc(&rdev->nr_pending);
+ rcu_read_unlock();
+
+ if (r_queue->unplug_fn)
+ r_queue->unplug_fn(r_queue);
+
+ rdev_dec_pending(rdev, mddev);
+ rcu_read_lock();
+ }
+ }
+ rcu_read_unlock();
+}
+
+static void raid1_unplug(request_queue_t *q)
+{
+ unplug_slaves(q->queuedata);
+}
+
+static int raid1_issue_flush(request_queue_t *q, struct gendisk *disk,
+ sector_t *error_sector)
+{
+ mddev_t *mddev = q->queuedata;
+ conf_t *conf = mddev_to_conf(mddev);
+ int i, ret = 0;
+
+ rcu_read_lock();
+ for (i=0; i<mddev->raid_disks && ret == 0; i++) {
+ mdk_rdev_t *rdev = conf->mirrors[i].rdev;
+ if (rdev && !rdev->faulty) {
+ struct block_device *bdev = rdev->bdev;
+ request_queue_t *r_queue = bdev_get_queue(bdev);
+
+ if (!r_queue->issue_flush_fn)
+ ret = -EOPNOTSUPP;
+ else {
+ atomic_inc(&rdev->nr_pending);
+ rcu_read_unlock();
+ ret = r_queue->issue_flush_fn(r_queue, bdev->bd_disk,
+ error_sector);
+ rdev_dec_pending(rdev, mddev);
+ rcu_read_lock();
+ }
+ }
+ }
+ rcu_read_unlock();
+ return ret;
+}
+
+/*
+ * Throttle resync depth, so that we can both get proper overlapping of
+ * requests, but are still able to handle normal requests quickly.
+ */
+#define RESYNC_DEPTH 32
+
+static void device_barrier(conf_t *conf, sector_t sect)
+{
+ spin_lock_irq(&conf->resync_lock);
+ wait_event_lock_irq(conf->wait_idle, !waitqueue_active(&conf->wait_resume),
+ conf->resync_lock, unplug_slaves(conf->mddev));
+
+ if (!conf->barrier++) {
+ wait_event_lock_irq(conf->wait_idle, !conf->nr_pending,
+ conf->resync_lock, unplug_slaves(conf->mddev));
+ if (conf->nr_pending)
+ BUG();
+ }
+ wait_event_lock_irq(conf->wait_resume, conf->barrier < RESYNC_DEPTH,
+ conf->resync_lock, unplug_slaves(conf->mddev));
+ conf->next_resync = sect;
+ spin_unlock_irq(&conf->resync_lock);
+}
+
+static int make_request(request_queue_t *q, struct bio * bio)
+{
+ mddev_t *mddev = q->queuedata;
+ conf_t *conf = mddev_to_conf(mddev);
+ mirror_info_t *mirror;
+ r1bio_t *r1_bio;
+ struct bio *read_bio;
+ int i, disks;
+ mdk_rdev_t *rdev;
+
+ /*
+ * Register the new request and wait if the reconstruction
+ * thread has put up a bar for new requests.
+ * Continue immediately if no resync is active currently.
+ */
+ spin_lock_irq(&conf->resync_lock);
+ wait_event_lock_irq(conf->wait_resume, !conf->barrier, conf->resync_lock, );
+ conf->nr_pending++;
+ spin_unlock_irq(&conf->resync_lock);
+
+ if (bio_data_dir(bio)==WRITE) {
+ disk_stat_inc(mddev->gendisk, writes);
+ disk_stat_add(mddev->gendisk, write_sectors, bio_sectors(bio));
+ } else {
+ disk_stat_inc(mddev->gendisk, reads);
+ disk_stat_add(mddev->gendisk, read_sectors, bio_sectors(bio));
+ }
+
+ /*
+ * make_request() can abort the operation when READA is being
+ * used and no empty request is available.
+ *
+ */
+ r1_bio = mempool_alloc(conf->r1bio_pool, GFP_NOIO);
+
+ r1_bio->master_bio = bio;
+ r1_bio->sectors = bio->bi_size >> 9;
+
+ r1_bio->mddev = mddev;
+ r1_bio->sector = bio->bi_sector;
+
+ r1_bio->state = 0;
+
+ if (bio_data_dir(bio) == READ) {
+ /*
+ * read balancing logic:
+ */
+ int rdisk = read_balance(conf, r1_bio);
+
+ if (rdisk < 0) {
+ /* couldn't find anywhere to read from */
+ raid_end_bio_io(r1_bio);
+ return 0;
+ }
+ mirror = conf->mirrors + rdisk;
+
+ r1_bio->read_disk = rdisk;
+
+ read_bio = bio_clone(bio, GFP_NOIO);
+
+ r1_bio->bios[rdisk] = read_bio;
+
+ read_bio->bi_sector = r1_bio->sector + mirror->rdev->data_offset;
+ read_bio->bi_bdev = mirror->rdev->bdev;
+ read_bio->bi_end_io = raid1_end_read_request;
+ read_bio->bi_rw = READ;
+ read_bio->bi_private = r1_bio;
+
+ generic_make_request(read_bio);
+ return 0;
+ }
+
+ /*
+ * WRITE:
+ */
+ /* first select target devices under spinlock and
+ * inc refcount on their rdev. Record them by setting
+ * bios[x] to bio
+ */
+ disks = conf->raid_disks;
+ rcu_read_lock();
+ for (i = 0; i < disks; i++) {
+ if ((rdev=conf->mirrors[i].rdev) != NULL &&
+ !rdev->faulty) {
+ atomic_inc(&rdev->nr_pending);
+ if (rdev->faulty) {
+ atomic_dec(&rdev->nr_pending);
+ r1_bio->bios[i] = NULL;
+ } else
+ r1_bio->bios[i] = bio;
+ } else
+ r1_bio->bios[i] = NULL;
+ }
+ rcu_read_unlock();
+
+ atomic_set(&r1_bio->remaining, 1);
+ md_write_start(mddev);
+ for (i = 0; i < disks; i++) {
+ struct bio *mbio;
+ if (!r1_bio->bios[i])
+ continue;
+
+ mbio = bio_clone(bio, GFP_NOIO);
+ r1_bio->bios[i] = mbio;
+
+ mbio->bi_sector = r1_bio->sector + conf->mirrors[i].rdev->data_offset;
+ mbio->bi_bdev = conf->mirrors[i].rdev->bdev;
+ mbio->bi_end_io = raid1_end_write_request;
+ mbio->bi_rw = WRITE;
+ mbio->bi_private = r1_bio;
+
+ atomic_inc(&r1_bio->remaining);
+ generic_make_request(mbio);
+ }
+
+ if (atomic_dec_and_test(&r1_bio->remaining)) {
+ md_write_end(mddev);
+ raid_end_bio_io(r1_bio);
+ }
+
+ return 0;
+}
+
+static void status(struct seq_file *seq, mddev_t *mddev)
+{
+ conf_t *conf = mddev_to_conf(mddev);
+ int i;
+
+ seq_printf(seq, " [%d/%d] [", conf->raid_disks,
+ conf->working_disks);
+ for (i = 0; i < conf->raid_disks; i++)
+ seq_printf(seq, "%s",
+ conf->mirrors[i].rdev &&
+ conf->mirrors[i].rdev->in_sync ? "U" : "_");
+ seq_printf(seq, "]");
+}
+
+
+static void error(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+ char b[BDEVNAME_SIZE];
+ conf_t *conf = mddev_to_conf(mddev);
+
+ /*
+ * If it is not operational, then we have already marked it as dead
+ * else if it is the last working disks, ignore the error, let the
+ * next level up know.
+ * else mark the drive as failed
+ */
+ if (rdev->in_sync
+ && conf->working_disks == 1)
+ /*
+ * Don't fail the drive, act as though we were just a
+ * normal single drive
+ */
+ return;
+ if (rdev->in_sync) {
+ mddev->degraded++;
+ conf->working_disks--;
+ /*
+ * if recovery is running, make sure it aborts.
+ */
+ set_bit(MD_RECOVERY_ERR, &mddev->recovery);
+ }
+ rdev->in_sync = 0;
+ rdev->faulty = 1;
+ mddev->sb_dirty = 1;
+ printk(KERN_ALERT "raid1: Disk failure on %s, disabling device. \n"
+ " Operation continuing on %d devices\n",
+ bdevname(rdev->bdev,b), conf->working_disks);
+}
+
+static void print_conf(conf_t *conf)
+{
+ int i;
+ mirror_info_t *tmp;
+
+ printk("RAID1 conf printout:\n");
+ if (!conf) {
+ printk("(!conf)\n");
+ return;
+ }
+ printk(" --- wd:%d rd:%d\n", conf->working_disks,
+ conf->raid_disks);
+
+ for (i = 0; i < conf->raid_disks; i++) {
+ char b[BDEVNAME_SIZE];
+ tmp = conf->mirrors + i;
+ if (tmp->rdev)
+ printk(" disk %d, wo:%d, o:%d, dev:%s\n",
+ i, !tmp->rdev->in_sync, !tmp->rdev->faulty,
+ bdevname(tmp->rdev->bdev,b));
+ }
+}
+
+static void close_sync(conf_t *conf)
+{
+ spin_lock_irq(&conf->resync_lock);
+ wait_event_lock_irq(conf->wait_resume, !conf->barrier,
+ conf->resync_lock, unplug_slaves(conf->mddev));
+ spin_unlock_irq(&conf->resync_lock);
+
+ if (conf->barrier) BUG();
+ if (waitqueue_active(&conf->wait_idle)) BUG();
+
+ mempool_destroy(conf->r1buf_pool);
+ conf->r1buf_pool = NULL;
+}
+
+static int raid1_spare_active(mddev_t *mddev)
+{
+ int i;
+ conf_t *conf = mddev->private;
+ mirror_info_t *tmp;
+
+ /*
+ * Find all failed disks within the RAID1 configuration
+ * and mark them readable
+ */
+ for (i = 0; i < conf->raid_disks; i++) {
+ tmp = conf->mirrors + i;
+ if (tmp->rdev
+ && !tmp->rdev->faulty
+ && !tmp->rdev->in_sync) {
+ conf->working_disks++;
+ mddev->degraded--;
+ tmp->rdev->in_sync = 1;
+ }
+ }
+
+ print_conf(conf);
+ return 0;
+}
+
+
+static int raid1_add_disk(mddev_t *mddev, mdk_rdev_t *rdev)
+{
+ conf_t *conf = mddev->private;
+ int found = 0;
+ int mirror;
+ mirror_info_t *p;
+
+ for (mirror=0; mirror < mddev->raid_disks; mirror++)
+ if ( !(p=conf->mirrors+mirror)->rdev) {
+
+ blk_queue_stack_limits(mddev->queue,
+ rdev->bdev->bd_disk->queue);
+ /* as we don't honour merge_bvec_fn, we must never risk
+ * violating it, so limit ->max_sector to one PAGE, as
+ * a one page request is never in violation.
+ */
+ if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
+ mddev->queue->max_sectors > (PAGE_SIZE>>9))
+ blk_queue_max_sectors(mddev->queue, PAGE_SIZE>>9);
+
+ p->head_position = 0;
+ rdev->raid_disk = mirror;
+ found = 1;
+ p->rdev = rdev;
+ break;
+ }
+
+ print_conf(conf);
+ return found;
+}
+
+static int raid1_remove_disk(mddev_t *mddev, int number)
+{
+ conf_t *conf = mddev->private;
+ int err = 0;
+ mdk_rdev_t *rdev;
+ mirror_info_t *p = conf->mirrors+ number;
+
+ print_conf(conf);
+ rdev = p->rdev;
+ if (rdev) {
+ if (rdev->in_sync ||
+ atomic_read(&rdev->nr_pending)) {
+ err = -EBUSY;
+ goto abort;
+ }
+ p->rdev = NULL;
+ synchronize_kernel();
+ if (atomic_read(&rdev->nr_pending)) {
+ /* lost the race, try later */
+ err = -EBUSY;
+ p->rdev = rdev;
+ }
+ }
+abort:
+
+ print_conf(conf);
+ return err;
+}
+
+
+static int end_sync_read(struct bio *bio, unsigned int bytes_done, int error)
+{
+ int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
+ r1bio_t * r1_bio = (r1bio_t *)(bio->bi_private);
+ conf_t *conf = mddev_to_conf(r1_bio->mddev);
+
+ if (bio->bi_size)
+ return 1;
+
+ if (r1_bio->bios[r1_bio->read_disk] != bio)
+ BUG();
+ update_head_pos(r1_bio->read_disk, r1_bio);
+ /*
+ * we have read a block, now it needs to be re-written,
+ * or re-read if the read failed.
+ * We don't do much here, just schedule handling by raid1d
+ */
+ if (!uptodate)
+ md_error(r1_bio->mddev,
+ conf->mirrors[r1_bio->read_disk].rdev);
+ else
+ set_bit(R1BIO_Uptodate, &r1_bio->state);
+ rdev_dec_pending(conf->mirrors[r1_bio->read_disk].rdev, conf->mddev);
+ reschedule_retry(r1_bio);
+ return 0;
+}
+
+static int end_sync_write(struct bio *bio, unsigned int bytes_done, int error)
+{
+ int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
+ r1bio_t * r1_bio = (r1bio_t *)(bio->bi_private);
+ mddev_t *mddev = r1_bio->mddev;
+ conf_t *conf = mddev_to_conf(mddev);
+ int i;
+ int mirror=0;
+
+ if (bio->bi_size)
+ return 1;
+
+ for (i = 0; i < conf->raid_disks; i++)
+ if (r1_bio->bios[i] == bio) {
+ mirror = i;
+ break;
+ }
+ if (!uptodate)
+ md_error(mddev, conf->mirrors[mirror].rdev);
+ update_head_pos(mirror, r1_bio);
+
+ if (atomic_dec_and_test(&r1_bio->remaining)) {
+ md_done_sync(mddev, r1_bio->sectors, uptodate);
+ put_buf(r1_bio);
+ }
+ rdev_dec_pending(conf->mirrors[mirror].rdev, mddev);
+ return 0;
+}
+
+static void sync_request_write(mddev_t *mddev, r1bio_t *r1_bio)
+{
+ conf_t *conf = mddev_to_conf(mddev);
+ int i;
+ int disks = conf->raid_disks;
+ struct bio *bio, *wbio;
+
+ bio = r1_bio->bios[r1_bio->read_disk];
+
+ /*
+ * schedule writes
+ */
+ if (!test_bit(R1BIO_Uptodate, &r1_bio->state)) {
+ /*
+ * There is no point trying a read-for-reconstruct as
+ * reconstruct is about to be aborted
+ */
+ char b[BDEVNAME_SIZE];
+ printk(KERN_ALERT "raid1: %s: unrecoverable I/O read error"
+ " for block %llu\n",
+ bdevname(bio->bi_bdev,b),
+ (unsigned long long)r1_bio->sector);
+ md_done_sync(mddev, r1_bio->sectors, 0);
+ put_buf(r1_bio);
+ return;
+ }
+
+ atomic_set(&r1_bio->remaining, 1);
+ for (i = 0; i < disks ; i++) {
+ wbio = r1_bio->bios[i];
+ if (wbio->bi_end_io != end_sync_write)
+ continue;
+
+ atomic_inc(&conf->mirrors[i].rdev->nr_pending);
+ atomic_inc(&r1_bio->remaining);
+ md_sync_acct(conf->mirrors[i].rdev->bdev, wbio->bi_size >> 9);
+ generic_make_request(wbio);
+ }
+
+ if (atomic_dec_and_test(&r1_bio->remaining)) {
+ md_done_sync(mddev, r1_bio->sectors, 1);
+ put_buf(r1_bio);
+ }
+}
+
+/*
+ * This is a kernel thread which:
+ *
+ * 1. Retries failed read operations on working mirrors.
+ * 2. Updates the raid superblock when problems encounter.
+ * 3. Performs writes following reads for array syncronising.
+ */
+
+static void raid1d(mddev_t *mddev)
+{
+ r1bio_t *r1_bio;
+ struct bio *bio;
+ unsigned long flags;
+ conf_t *conf = mddev_to_conf(mddev);
+ struct list_head *head = &conf->retry_list;
+ int unplug=0;
+ mdk_rdev_t *rdev;
+
+ md_check_recovery(mddev);
+ md_handle_safemode(mddev);
+
+ for (;;) {
+ char b[BDEVNAME_SIZE];
+ spin_lock_irqsave(&conf->device_lock, flags);
+ if (list_empty(head))
+ break;
+ r1_bio = list_entry(head->prev, r1bio_t, retry_list);
+ list_del(head->prev);
+ spin_unlock_irqrestore(&conf->device_lock, flags);
+
+ mddev = r1_bio->mddev;
+ conf = mddev_to_conf(mddev);
+ if (test_bit(R1BIO_IsSync, &r1_bio->state)) {
+ sync_request_write(mddev, r1_bio);
+ unplug = 1;
+ } else {
+ int disk;
+ bio = r1_bio->bios[r1_bio->read_disk];
+ if ((disk=read_balance(conf, r1_bio)) == -1) {
+ printk(KERN_ALERT "raid1: %s: unrecoverable I/O"
+ " read error for block %llu\n",
+ bdevname(bio->bi_bdev,b),
+ (unsigned long long)r1_bio->sector);
+ raid_end_bio_io(r1_bio);
+ } else {
+ r1_bio->bios[r1_bio->read_disk] = NULL;
+ r1_bio->read_disk = disk;
+ bio_put(bio);
+ bio = bio_clone(r1_bio->master_bio, GFP_NOIO);
+ r1_bio->bios[r1_bio->read_disk] = bio;
+ rdev = conf->mirrors[disk].rdev;
+ if (printk_ratelimit())
+ printk(KERN_ERR "raid1: %s: redirecting sector %llu to"
+ " another mirror\n",
+ bdevname(rdev->bdev,b),
+ (unsigned long long)r1_bio->sector);
+ bio->bi_sector = r1_bio->sector + rdev->data_offset;
+ bio->bi_bdev = rdev->bdev;
+ bio->bi_end_io = raid1_end_read_request;
+ bio->bi_rw = READ;
+ bio->bi_private = r1_bio;
+ unplug = 1;
+ generic_make_request(bio);
+ }
+ }
+ }
+ spin_unlock_irqrestore(&conf->device_lock, flags);
+ if (unplug)
+ unplug_slaves(mddev);
+}
+
+
+static int init_resync(conf_t *conf)
+{
+ int buffs;
+
+ buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
+ if (conf->r1buf_pool)
+ BUG();
+ conf->r1buf_pool = mempool_create(buffs, r1buf_pool_alloc, r1buf_pool_free,
+ conf->poolinfo);
+ if (!conf->r1buf_pool)
+ return -ENOMEM;
+ conf->next_resync = 0;
+ return 0;
+}
+
+/*
+ * perform a "sync" on one "block"
+ *
+ * We need to make sure that no normal I/O request - particularly write
+ * requests - conflict with active sync requests.
+ *
+ * This is achieved by tracking pending requests and a 'barrier' concept
+ * that can be installed to exclude normal IO requests.
+ */
+
+static int sync_request(mddev_t *mddev, sector_t sector_nr, int go_faster)
+{
+ conf_t *conf = mddev_to_conf(mddev);
+ mirror_info_t *mirror;
+ r1bio_t *r1_bio;
+ struct bio *bio;
+ sector_t max_sector, nr_sectors;
+ int disk;
+ int i;
+ int write_targets = 0;
+
+ if (!conf->r1buf_pool)
+ if (init_resync(conf))
+ return -ENOMEM;
+
+ max_sector = mddev->size << 1;
+ if (sector_nr >= max_sector) {
+ close_sync(conf);
+ return 0;
+ }
+
+ /*
+ * If there is non-resync activity waiting for us then
+ * put in a delay to throttle resync.
+ */
+ if (!go_faster && waitqueue_active(&conf->wait_resume))
+ msleep_interruptible(1000);
+ device_barrier(conf, sector_nr + RESYNC_SECTORS);
+
+ /*
+ * If reconstructing, and >1 working disc,
+ * could dedicate one to rebuild and others to
+ * service read requests ..
+ */
+ disk = conf->last_used;
+ /* make sure disk is operational */
+
+ while (conf->mirrors[disk].rdev == NULL ||
+ !conf->mirrors[disk].rdev->in_sync) {
+ if (disk <= 0)
+ disk = conf->raid_disks;
+ disk--;
+ if (disk == conf->last_used)
+ break;
+ }
+ conf->last_used = disk;
+ atomic_inc(&conf->mirrors[disk].rdev->nr_pending);
+
+
+ mirror = conf->mirrors + disk;
+
+ r1_bio = mempool_alloc(conf->r1buf_pool, GFP_NOIO);
+
+ spin_lock_irq(&conf->resync_lock);
+ conf->nr_pending++;
+ spin_unlock_irq(&conf->resync_lock);
+
+ r1_bio->mddev = mddev;
+ r1_bio->sector = sector_nr;
+ set_bit(R1BIO_IsSync, &r1_bio->state);
+ r1_bio->read_disk = disk;
+
+ for (i=0; i < conf->raid_disks; i++) {
+ bio = r1_bio->bios[i];
+
+ /* take from bio_init */
+ bio->bi_next = NULL;
+ bio->bi_flags |= 1 << BIO_UPTODATE;
+ bio->bi_rw = 0;
+ bio->bi_vcnt = 0;
+ bio->bi_idx = 0;
+ bio->bi_phys_segments = 0;
+ bio->bi_hw_segments = 0;
+ bio->bi_size = 0;
+ bio->bi_end_io = NULL;
+ bio->bi_private = NULL;
+
+ if (i == disk) {
+ bio->bi_rw = READ;
+ bio->bi_end_io = end_sync_read;
+ } else if (conf->mirrors[i].rdev &&
+ !conf->mirrors[i].rdev->faulty &&
+ (!conf->mirrors[i].rdev->in_sync ||
+ sector_nr + RESYNC_SECTORS > mddev->recovery_cp)) {
+ bio->bi_rw = WRITE;
+ bio->bi_end_io = end_sync_write;
+ write_targets ++;
+ } else
+ continue;
+ bio->bi_sector = sector_nr + conf->mirrors[i].rdev->data_offset;
+ bio->bi_bdev = conf->mirrors[i].rdev->bdev;
+ bio->bi_private = r1_bio;
+ }
+ if (write_targets == 0) {
+ /* There is nowhere to write, so all non-sync
+ * drives must be failed - so we are finished
+ */
+ int rv = max_sector - sector_nr;
+ md_done_sync(mddev, rv, 1);
+ put_buf(r1_bio);
+ rdev_dec_pending(conf->mirrors[disk].rdev, mddev);
+ return rv;
+ }
+
+ nr_sectors = 0;
+ do {
+ struct page *page;
+ int len = PAGE_SIZE;
+ if (sector_nr + (len>>9) > max_sector)
+ len = (max_sector - sector_nr) << 9;
+ if (len == 0)
+ break;
+ for (i=0 ; i < conf->raid_disks; i++) {
+ bio = r1_bio->bios[i];
+ if (bio->bi_end_io) {
+ page = r1_bio->bios[0]->bi_io_vec[bio->bi_vcnt].bv_page;
+ if (bio_add_page(bio, page, len, 0) == 0) {
+ /* stop here */
+ r1_bio->bios[0]->bi_io_vec[bio->bi_vcnt].bv_page = page;
+ while (i > 0) {
+ i--;
+ bio = r1_bio->bios[i];
+ if (bio->bi_end_io==NULL) continue;
+ /* remove last page from this bio */
+ bio->bi_vcnt--;
+ bio->bi_size -= len;
+ bio->bi_flags &= ~(1<< BIO_SEG_VALID);
+ }
+ goto bio_full;
+ }
+ }
+ }
+ nr_sectors += len>>9;
+ sector_nr += len>>9;
+ } while (r1_bio->bios[disk]->bi_vcnt < RESYNC_PAGES);
+ bio_full:
+ bio = r1_bio->bios[disk];
+ r1_bio->sectors = nr_sectors;
+
+ md_sync_acct(mirror->rdev->bdev, nr_sectors);
+
+ generic_make_request(bio);
+
+ return nr_sectors;
+}
+
+static int run(mddev_t *mddev)
+{
+ conf_t *conf;
+ int i, j, disk_idx;
+ mirror_info_t *disk;
+ mdk_rdev_t *rdev;
+ struct list_head *tmp;
+
+ if (mddev->level != 1) {
+ printk("raid1: %s: raid level not set to mirroring (%d)\n",
+ mdname(mddev), mddev->level);
+ goto out;
+ }
+ /*
+ * copy the already verified devices into our private RAID1
+ * bookkeeping area. [whatever we allocate in run(),
+ * should be freed in stop()]
+ */
+ conf = kmalloc(sizeof(conf_t), GFP_KERNEL);
+ mddev->private = conf;
+ if (!conf)
+ goto out_no_mem;
+
+ memset(conf, 0, sizeof(*conf));
+ conf->mirrors = kmalloc(sizeof(struct mirror_info)*mddev->raid_disks,
+ GFP_KERNEL);
+ if (!conf->mirrors)
+ goto out_no_mem;
+
+ memset(conf->mirrors, 0, sizeof(struct mirror_info)*mddev->raid_disks);
+
+ conf->poolinfo = kmalloc(sizeof(*conf->poolinfo), GFP_KERNEL);
+ if (!conf->poolinfo)
+ goto out_no_mem;
+ conf->poolinfo->mddev = mddev;
+ conf->poolinfo->raid_disks = mddev->raid_disks;
+ conf->r1bio_pool = mempool_create(NR_RAID1_BIOS, r1bio_pool_alloc,
+ r1bio_pool_free,
+ conf->poolinfo);
+ if (!conf->r1bio_pool)
+ goto out_no_mem;
+
+ mddev->queue->unplug_fn = raid1_unplug;
+
+ mddev->queue->issue_flush_fn = raid1_issue_flush;
+
+ ITERATE_RDEV(mddev, rdev, tmp) {
+ disk_idx = rdev->raid_disk;
+ if (disk_idx >= mddev->raid_disks
+ || disk_idx < 0)
+ continue;
+ disk = conf->mirrors + disk_idx;
+
+ disk->rdev = rdev;
+
+ blk_queue_stack_limits(mddev->queue,
+ rdev->bdev->bd_disk->queue);
+ /* as we don't honour merge_bvec_fn, we must never risk
+ * violating it, so limit ->max_sector to one PAGE, as
+ * a one page request is never in violation.
+ */
+ if (rdev->bdev->bd_disk->queue->merge_bvec_fn &&
+ mddev->queue->max_sectors > (PAGE_SIZE>>9))
+ blk_queue_max_sectors(mddev->queue, PAGE_SIZE>>9);
+
+ disk->head_position = 0;
+ if (!rdev->faulty && rdev->in_sync)
+ conf->working_disks++;
+ }
+ conf->raid_disks = mddev->raid_disks;
+ conf->mddev = mddev;
+ spin_lock_init(&conf->device_lock);
+ INIT_LIST_HEAD(&conf->retry_list);
+ if (conf->working_disks == 1)
+ mddev->recovery_cp = MaxSector;
+
+ spin_lock_init(&conf->resync_lock);
+ init_waitqueue_head(&conf->wait_idle);
+ init_waitqueue_head(&conf->wait_resume);
+
+ if (!conf->working_disks) {
+ printk(KERN_ERR "raid1: no operational mirrors for %s\n",
+ mdname(mddev));
+ goto out_free_conf;
+ }
+
+ mddev->degraded = 0;
+ for (i = 0; i < conf->raid_disks; i++) {
+
+ disk = conf->mirrors + i;
+
+ if (!disk->rdev) {
+ disk->head_position = 0;
+ mddev->degraded++;
+ }
+ }
+
+ /*
+ * find the first working one and use it as a starting point
+ * to read balancing.
+ */
+ for (j = 0; j < conf->raid_disks &&
+ (!conf->mirrors[j].rdev ||
+ !conf->mirrors[j].rdev->in_sync) ; j++)
+ /* nothing */;
+ conf->last_used = j;
+
+
+
+ {
+ mddev->thread = md_register_thread(raid1d, mddev, "%s_raid1");
+ if (!mddev->thread) {
+ printk(KERN_ERR
+ "raid1: couldn't allocate thread for %s\n",
+ mdname(mddev));
+ goto out_free_conf;
+ }
+ }
+ printk(KERN_INFO
+ "raid1: raid set %s active with %d out of %d mirrors\n",
+ mdname(mddev), mddev->raid_disks - mddev->degraded,
+ mddev->raid_disks);
+ /*
+ * Ok, everything is just fine now
+ */
+ mddev->array_size = mddev->size;
+
+ return 0;
+
+out_no_mem:
+ printk(KERN_ERR "raid1: couldn't allocate memory for %s\n",
+ mdname(mddev));
+
+out_free_conf:
+ if (conf) {
+ if (conf->r1bio_pool)
+ mempool_destroy(conf->r1bio_pool);
+ if (conf->mirrors)
+ kfree(conf->mirrors);
+ if (conf->poolinfo)
+ kfree(conf->poolinfo);
+ kfree(conf);
+ mddev->private = NULL;
+ }
+out:
+ return -EIO;
+}
+
+static int stop(mddev_t *mddev)
+{
+ conf_t *conf = mddev_to_conf(mddev);
+
+ md_unregister_thread(mddev->thread);
+ mddev->thread = NULL;
+ blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
+ if (conf->r1bio_pool)
+ mempool_destroy(conf->r1bio_pool);
+ if (conf->mirrors)
+ kfree(conf->mirrors);
+ if (conf->poolinfo)
+ kfree(conf->poolinfo);
+ kfree(conf);
+ mddev->private = NULL;
+ return 0;
+}
+
+static int raid1_resize(mddev_t *mddev, sector_t sectors)
+{
+ /* no resync is happening, and there is enough space
+ * on all devices, so we can resize.
+ * We need to make sure resync covers any new space.
+ * If the array is shrinking we should possibly wait until
+ * any io in the removed space completes, but it hardly seems
+ * worth it.
+ */
+ mddev->array_size = sectors>>1;
+ set_capacity(mddev->gendisk, mddev->array_size << 1);
+ mddev->changed = 1;
+ if (mddev->array_size > mddev->size && mddev->recovery_cp == MaxSector) {
+ mddev->recovery_cp = mddev->size << 1;
+ set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+ }
+ mddev->size = mddev->array_size;
+ return 0;
+}
+
+static int raid1_reshape(mddev_t *mddev, int raid_disks)
+{
+ /* We need to:
+ * 1/ resize the r1bio_pool
+ * 2/ resize conf->mirrors
+ *
+ * We allocate a new r1bio_pool if we can.
+ * Then raise a device barrier and wait until all IO stops.
+ * Then resize conf->mirrors and swap in the new r1bio pool.
+ */
+ mempool_t *newpool, *oldpool;
+ struct pool_info *newpoolinfo;
+ mirror_info_t *newmirrors;
+ conf_t *conf = mddev_to_conf(mddev);
+
+ int d;
+
+ for (d= raid_disks; d < conf->raid_disks; d++)
+ if (conf->mirrors[d].rdev)
+ return -EBUSY;
+
+ newpoolinfo = kmalloc(sizeof(*newpoolinfo), GFP_KERNEL);
+ if (!newpoolinfo)
+ return -ENOMEM;
+ newpoolinfo->mddev = mddev;
+ newpoolinfo->raid_disks = raid_disks;
+
+ newpool = mempool_create(NR_RAID1_BIOS, r1bio_pool_alloc,
+ r1bio_pool_free, newpoolinfo);
+ if (!newpool) {
+ kfree(newpoolinfo);
+ return -ENOMEM;
+ }
+ newmirrors = kmalloc(sizeof(struct mirror_info) * raid_disks, GFP_KERNEL);
+ if (!newmirrors) {
+ kfree(newpoolinfo);
+ mempool_destroy(newpool);
+ return -ENOMEM;
+ }
+ memset(newmirrors, 0, sizeof(struct mirror_info)*raid_disks);
+
+ spin_lock_irq(&conf->resync_lock);
+ conf->barrier++;
+ wait_event_lock_irq(conf->wait_idle, !conf->nr_pending,
+ conf->resync_lock, unplug_slaves(mddev));
+ spin_unlock_irq(&conf->resync_lock);
+
+ /* ok, everything is stopped */
+ oldpool = conf->r1bio_pool;
+ conf->r1bio_pool = newpool;
+ for (d=0; d < raid_disks && d < conf->raid_disks; d++)
+ newmirrors[d] = conf->mirrors[d];
+ kfree(conf->mirrors);
+ conf->mirrors = newmirrors;
+ kfree(conf->poolinfo);
+ conf->poolinfo = newpoolinfo;
+
+ mddev->degraded += (raid_disks - conf->raid_disks);
+ conf->raid_disks = mddev->raid_disks = raid_disks;
+
+ spin_lock_irq(&conf->resync_lock);
+ conf->barrier--;
+ spin_unlock_irq(&conf->resync_lock);
+ wake_up(&conf->wait_resume);
+ wake_up(&conf->wait_idle);
+
+
+ set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+ md_wakeup_thread(mddev->thread);
+
+ mempool_destroy(oldpool);
+ return 0;
+}
+
+
+static mdk_personality_t raid1_personality =
+{
+ .name = "raid1",
+ .owner = THIS_MODULE,
+ .make_request = make_request,
+ .run = run,
+ .stop = stop,
+ .status = status,
+ .error_handler = error,
+ .hot_add_disk = raid1_add_disk,
+ .hot_remove_disk= raid1_remove_disk,
+ .spare_active = raid1_spare_active,
+ .sync_request = sync_request,
+ .resize = raid1_resize,
+ .reshape = raid1_reshape,
+};
+
+static int __init raid_init(void)
+{
+ return register_md_personality(RAID1, &raid1_personality);
+}
+
+static void raid_exit(void)
+{
+ unregister_md_personality(RAID1);
+}
+
+module_init(raid_init);
+module_exit(raid_exit);
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("md-personality-3"); /* RAID1 */
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
new file mode 100644
index 000000000000..b100bfe4fdca
--- /dev/null
+++ b/drivers/md/raid10.c
@@ -0,0 +1,1787 @@
+/*
+ * raid10.c : Multiple Devices driver for Linux
+ *
+ * Copyright (C) 2000-2004 Neil Brown
+ *
+ * RAID-10 support for md.
+ *
+ * Base on code in raid1.c. See raid1.c for futher copyright information.
+ *
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * (for example /usr/src/linux/COPYING); if not, write to the Free
+ * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/raid/raid10.h>
+
+/*
+ * RAID10 provides a combination of RAID0 and RAID1 functionality.
+ * The layout of data is defined by
+ * chunk_size
+ * raid_disks
+ * near_copies (stored in low byte of layout)
+ * far_copies (stored in second byte of layout)
+ *
+ * The data to be stored is divided into chunks using chunksize.
+ * Each device is divided into far_copies sections.
+ * In each section, chunks are laid out in a style similar to raid0, but
+ * near_copies copies of each chunk is stored (each on a different drive).
+ * The starting device for each section is offset near_copies from the starting
+ * device of the previous section.
+ * Thus there are (near_copies*far_copies) of each chunk, and each is on a different
+ * drive.
+ * near_copies and far_copies must be at least one, and their product is at most
+ * raid_disks.
+ */
+
+/*
+ * Number of guaranteed r10bios in case of extreme VM load:
+ */
+#define NR_RAID10_BIOS 256
+
+static void unplug_slaves(mddev_t *mddev);
+
+static void * r10bio_pool_alloc(unsigned int __nocast gfp_flags, void *data)
+{
+ conf_t *conf = data;
+ r10bio_t *r10_bio;
+ int size = offsetof(struct r10bio_s, devs[conf->copies]);
+
+ /* allocate a r10bio with room for raid_disks entries in the bios array */
+ r10_bio = kmalloc(size, gfp_flags);
+ if (r10_bio)
+ memset(r10_bio, 0, size);
+ else
+ unplug_slaves(conf->mddev);
+
+ return r10_bio;
+}
+
+static void r10bio_pool_free(void *r10_bio, void *data)
+{
+ kfree(r10_bio);
+}
+
+#define RESYNC_BLOCK_SIZE (64*1024)
+//#define RESYNC_BLOCK_SIZE PAGE_SIZE
+#define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)
+#define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
+#define RESYNC_WINDOW (2048*1024)
+
+/*
+ * When performing a resync, we need to read and compare, so
+ * we need as many pages are there are copies.
+ * When performing a recovery, we need 2 bios, one for read,
+ * one for write (we recover only one drive per r10buf)
+ *
+ */
+static void * r10buf_pool_alloc(unsigned int __nocast gfp_flags, void *data)
+{
+ conf_t *conf = data;
+ struct page *page;
+ r10bio_t *r10_bio;
+ struct bio *bio;
+ int i, j;
+ int nalloc;
+
+ r10_bio = r10bio_pool_alloc(gfp_flags, conf);
+ if (!r10_bio) {
+ unplug_slaves(conf->mddev);
+ return NULL;
+ }
+
+ if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery))
+ nalloc = conf->copies; /* resync */
+ else
+ nalloc = 2; /* recovery */
+
+ /*
+ * Allocate bios.
+ */
+ for (j = nalloc ; j-- ; ) {
+ bio = bio_alloc(gfp_flags, RESYNC_PAGES);
+ if (!bio)
+ goto out_free_bio;
+ r10_bio->devs[j].bio = bio;
+ }
+ /*
+ * Allocate RESYNC_PAGES data pages and attach them
+ * where needed.
+ */
+ for (j = 0 ; j < nalloc; j++) {
+ bio = r10_bio->devs[j].bio;
+ for (i = 0; i < RESYNC_PAGES; i++) {
+ page = alloc_page(gfp_flags);
+ if (unlikely(!page))
+ goto out_free_pages;
+
+ bio->bi_io_vec[i].bv_page = page;
+ }
+ }
+
+ return r10_bio;
+
+out_free_pages:
+ for ( ; i > 0 ; i--)
+ __free_page(bio->bi_io_vec[i-1].bv_page);
+ while (j--)
+ for (i = 0; i < RESYNC_PAGES ; i++)
+ __free_page(r10_bio->devs[j].bio->bi_io_vec[i].bv_page);
+ j = -1;
+out_free_bio:
+ while ( ++j < nalloc )
+ bio_put(r10_bio->devs[j].bio);
+ r10bio_pool_free(r10_bio, conf);
+ return NULL;
+}
+
+static void r10buf_pool_free(void *__r10_bio, void *data)
+{
+ int i;
+ conf_t *conf = data;
+ r10bio_t *r10bio = __r10_bio;
+ int j;
+
+ for (j=0; j < conf->copies; j++) {
+ struct bio *bio = r10bio->devs[j].bio;
+ if (bio) {
+ for (i = 0; i < RESYNC_PAGES; i++) {
+ __free_page(bio->bi_io_vec[i].bv_page);
+ bio->bi_io_vec[i].bv_page = NULL;
+ }
+ bio_put(bio);
+ }
+ }
+ r10bio_pool_free(r10bio, conf);
+}
+
+static void put_all_bios(conf_t *conf, r10bio_t *r10_bio)
+{
+ int i;
+
+ for (i = 0; i < conf->copies; i++) {
+ struct bio **bio = & r10_bio->devs[i].bio;
+ if (*bio)
+ bio_put(*bio);
+ *bio = NULL;
+ }
+}
+
+static inline void free_r10bio(r10bio_t *r10_bio)
+{
+ unsigned long flags;
+
+ conf_t *conf = mddev_to_conf(r10_bio->mddev);
+
+ /*
+ * Wake up any possible resync thread that waits for the device
+ * to go idle.
+ */
+ spin_lock_irqsave(&conf->resync_lock, flags);
+ if (!--conf->nr_pending) {
+ wake_up(&conf->wait_idle);
+ wake_up(&conf->wait_resume);
+ }
+ spin_unlock_irqrestore(&conf->resync_lock, flags);
+
+ put_all_bios(conf, r10_bio);
+ mempool_free(r10_bio, conf->r10bio_pool);
+}
+
+static inline void put_buf(r10bio_t *r10_bio)
+{
+ conf_t *conf = mddev_to_conf(r10_bio->mddev);
+ unsigned long flags;
+
+ mempool_free(r10_bio, conf->r10buf_pool);
+
+ spin_lock_irqsave(&conf->resync_lock, flags);
+ if (!conf->barrier)
+ BUG();
+ --conf->barrier;
+ wake_up(&conf->wait_resume);
+ wake_up(&conf->wait_idle);
+
+ if (!--conf->nr_pending) {
+ wake_up(&conf->wait_idle);
+ wake_up(&conf->wait_resume);
+ }
+ spin_unlock_irqrestore(&conf->resync_lock, flags);
+}
+
+static void reschedule_retry(r10bio_t *r10_bio)
+{
+ unsigned long flags;
+ mddev_t *mddev = r10_bio->mddev;
+ conf_t *conf = mddev_to_conf(mddev);
+
+ spin_lock_irqsave(&conf->device_lock, flags);
+ list_add(&r10_bio->retry_list, &conf->retry_list);
+ spin_unlock_irqrestore(&conf->device_lock, flags);
+
+ md_wakeup_thread(mddev->thread);
+}
+
+/*
+ * raid_end_bio_io() is called when we have finished servicing a mirrored
+ * operation and are ready to return a success/failure code to the buffer
+ * cache layer.
+ */
+static void raid_end_bio_io(r10bio_t *r10_bio)
+{
+ struct bio *bio = r10_bio->master_bio;
+
+ bio_endio(bio, bio->bi_size,
+ test_bit(R10BIO_Uptodate, &r10_bio->state) ? 0 : -EIO);
+ free_r10bio(r10_bio);
+}
+
+/*
+ * Update disk head position estimator based on IRQ completion info.
+ */
+static inline void update_head_pos(int slot, r10bio_t *r10_bio)
+{
+ conf_t *conf = mddev_to_conf(r10_bio->mddev);
+
+ conf->mirrors[r10_bio->devs[slot].devnum].head_position =
+ r10_bio->devs[slot].addr + (r10_bio->sectors);
+}
+
+static int raid10_end_read_request(struct bio *bio, unsigned int bytes_done, int error)
+{
+ int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
+ r10bio_t * r10_bio = (r10bio_t *)(bio->bi_private);
+ int slot, dev;
+ conf_t *conf = mddev_to_conf(r10_bio->mddev);
+
+ if (bio->bi_size)
+ return 1;
+
+ slot = r10_bio->read_slot;
+ dev = r10_bio->devs[slot].devnum;
+ /*
+ * this branch is our 'one mirror IO has finished' event handler:
+ */
+ if (!uptodate)
+ md_error(r10_bio->mddev, conf->mirrors[dev].rdev);
+ else
+ /*
+ * Set R10BIO_Uptodate in our master bio, so that
+ * we will return a good error code to the higher
+ * levels even if IO on some other mirrored buffer fails.
+ *
+ * The 'master' represents the composite IO operation to
+ * user-side. So if something waits for IO, then it will
+ * wait for the 'master' bio.
+ */
+ set_bit(R10BIO_Uptodate, &r10_bio->state);
+
+ update_head_pos(slot, r10_bio);
+
+ /*
+ * we have only one bio on the read side
+ */
+ if (uptodate)
+ raid_end_bio_io(r10_bio);
+ else {
+ /*
+ * oops, read error:
+ */
+ char b[BDEVNAME_SIZE];
+ if (printk_ratelimit())
+ printk(KERN_ERR "raid10: %s: rescheduling sector %llu\n",
+ bdevname(conf->mirrors[dev].rdev->bdev,b), (unsigned long long)r10_bio->sector);
+ reschedule_retry(r10_bio);
+ }
+
+ rdev_dec_pending(conf->mirrors[dev].rdev, conf->mddev);
+ return 0;
+}
+
+static int raid10_end_write_request(struct bio *bio, unsigned int bytes_done, int error)
+{
+ int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
+ r10bio_t * r10_bio = (r10bio_t *)(bio->bi_private);
+ int slot, dev;
+ conf_t *conf = mddev_to_conf(r10_bio->mddev);
+
+ if (bio->bi_size)
+ return 1;
+
+ for (slot = 0; slot < conf->copies; slot++)
+ if (r10_bio->devs[slot].bio == bio)
+ break;
+ dev = r10_bio->devs[slot].devnum;
+
+ /*
+ * this branch is our 'one mirror IO has finished' event handler:
+ */
+ if (!uptodate)
+ md_error(r10_bio->mddev, conf->mirrors[dev].rdev);
+ else
+ /*
+ * Set R10BIO_Uptodate in our master bio, so that
+ * we will return a good error code for to the higher
+ * levels even if IO on some other mirrored buffer fails.
+ *
+ * The 'master' represents the composite IO operation to
+ * user-side. So if something waits for IO, then it will
+ * wait for the 'master' bio.
+ */
+ set_bit(R10BIO_Uptodate, &r10_bio->state);
+
+ update_head_pos(slot, r10_bio);
+
+ /*
+ *
+ * Let's see if all mirrored write operations have finished
+ * already.
+ */
+ if (atomic_dec_and_test(&r10_bio->remaining)) {
+ md_write_end(r10_bio->mddev);
+ raid_end_bio_io(r10_bio);
+ }
+
+ rdev_dec_pending(conf->mirrors[dev].rdev, conf->mddev);
+ return 0;
+}
+
+
+/*
+ * RAID10 layout manager
+ * Aswell as the chunksize and raid_disks count, there are two
+ * parameters: near_copies and far_copies.
+ * near_copies * far_copies must be <= raid_disks.
+ * Normally one of these will be 1.
+ * If both are 1, we get raid0.
+ * If near_copies == raid_disks, we get raid1.
+ *
+ * Chunks are layed out in raid0 style with near_copies copies of the
+ * first chunk, followed by near_copies copies of the next chunk and
+ * so on.
+ * If far_copies > 1, then after 1/far_copies of the array has been assigned
+ * as described above, we start again with a device offset of near_copies.
+ * So we effectively have another copy of the whole array further down all
+ * the drives, but with blocks on different drives.
+ * With this layout, and block is never stored twice on the one device.
+ *
+ * raid10_find_phys finds the sector offset of a given virtual sector
+ * on each device that it is on. If a block isn't on a device,
+ * that entry in the array is set to MaxSector.
+ *
+ * raid10_find_virt does the reverse mapping, from a device and a
+ * sector offset to a virtual address
+ */
+
+static void raid10_find_phys(conf_t *conf, r10bio_t *r10bio)
+{
+ int n,f;
+ sector_t sector;
+ sector_t chunk;
+ sector_t stripe;
+ int dev;
+
+ int slot = 0;
+
+ /* now calculate first sector/dev */
+ chunk = r10bio->sector >> conf->chunk_shift;
+ sector = r10bio->sector & conf->chunk_mask;
+
+ chunk *= conf->near_copies;
+ stripe = chunk;
+ dev = sector_div(stripe, conf->raid_disks);
+
+ sector += stripe << conf->chunk_shift;
+
+ /* and calculate all the others */
+ for (n=0; n < conf->near_copies; n++) {
+ int d = dev;
+ sector_t s = sector;
+ r10bio->devs[slot].addr = sector;
+ r10bio->devs[slot].devnum = d;
+ slot++;
+
+ for (f = 1; f < conf->far_copies; f++) {
+ d += conf->near_copies;
+ if (d >= conf->raid_disks)
+ d -= conf->raid_disks;
+ s += conf->stride;
+ r10bio->devs[slot].devnum = d;
+ r10bio->devs[slot].addr = s;
+ slot++;
+ }
+ dev++;
+ if (dev >= conf->raid_disks) {
+ dev = 0;
+ sector += (conf->chunk_mask + 1);
+ }
+ }
+ BUG_ON(slot != conf->copies);
+}
+
+static sector_t raid10_find_virt(conf_t *conf, sector_t sector, int dev)
+{
+ sector_t offset, chunk, vchunk;
+
+ while (sector > conf->stride) {
+ sector -= conf->stride;
+ if (dev < conf->near_copies)
+ dev += conf->raid_disks - conf->near_copies;
+ else
+ dev -= conf->near_copies;
+ }
+
+ offset = sector & conf->chunk_mask;
+ chunk = sector >> conf->chunk_shift;
+ vchunk = chunk * conf->raid_disks + dev;
+ sector_div(vchunk, conf->near_copies);
+ return (vchunk << conf->chunk_shift) + offset;
+}
+
+/**
+ * raid10_mergeable_bvec -- tell bio layer if a two requests can be merged
+ * @q: request queue
+ * @bio: the buffer head that's been built up so far
+ * @biovec: the request that could be merged to it.
+ *
+ * Return amount of bytes we can accept at this offset
+ * If near_copies == raid_disk, there are no striping issues,
+ * but in that case, the function isn't called at all.
+ */
+static int raid10_mergeable_bvec(request_queue_t *q, struct bio *bio,
+ struct bio_vec *bio_vec)
+{
+ mddev_t *mddev = q->queuedata;
+ sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev);
+ int max;
+ unsigned int chunk_sectors = mddev->chunk_size >> 9;
+ unsigned int bio_sectors = bio->bi_size >> 9;
+
+ max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
+ if (max < 0) max = 0; /* bio_add cannot handle a negative return */
+ if (max <= bio_vec->bv_len && bio_sectors == 0)
+ return bio_vec->bv_len;
+ else
+ return max;
+}
+
+/*
+ * This routine returns the disk from which the requested read should
+ * be done. There is a per-array 'next expected sequential IO' sector
+ * number - if this matches on the next IO then we use the last disk.
+ * There is also a per-disk 'last know head position' sector that is
+ * maintained from IRQ contexts, both the normal and the resync IO
+ * completion handlers update this position correctly. If there is no
+ * perfect sequential match then we pick the disk whose head is closest.
+ *
+ * If there are 2 mirrors in the same 2 devices, performance degrades
+ * because position is mirror, not device based.
+ *
+ * The rdev for the device selected will have nr_pending incremented.
+ */
+
+/*
+ * FIXME: possibly should rethink readbalancing and do it differently
+ * depending on near_copies / far_copies geometry.
+ */
+static int read_balance(conf_t *conf, r10bio_t *r10_bio)
+{
+ const unsigned long this_sector = r10_bio->sector;
+ int disk, slot, nslot;
+ const int sectors = r10_bio->sectors;
+ sector_t new_distance, current_distance;
+
+ raid10_find_phys(conf, r10_bio);
+ rcu_read_lock();
+ /*
+ * Check if we can balance. We can balance on the whole
+ * device if no resync is going on, or below the resync window.
+ * We take the first readable disk when above the resync window.
+ */
+ if (conf->mddev->recovery_cp < MaxSector
+ && (this_sector + sectors >= conf->next_resync)) {
+ /* make sure that disk is operational */
+ slot = 0;
+ disk = r10_bio->devs[slot].devnum;
+
+ while (!conf->mirrors[disk].rdev ||
+ !conf->mirrors[disk].rdev->in_sync) {
+ slot++;
+ if (slot == conf->copies) {
+ slot = 0;
+ disk = -1;
+ break;
+ }
+ disk = r10_bio->devs[slot].devnum;
+ }
+ goto rb_out;
+ }
+
+
+ /* make sure the disk is operational */
+ slot = 0;
+ disk = r10_bio->devs[slot].devnum;
+ while (!conf->mirrors[disk].rdev ||
+ !conf->mirrors[disk].rdev->in_sync) {
+ slot ++;
+ if (slot == conf->copies) {
+ disk = -1;
+ goto rb_out;
+ }
+ disk = r10_bio->devs[slot].devnum;
+ }
+
+
+ current_distance = abs(this_sector - conf->mirrors[disk].head_position);
+
+ /* Find the disk whose head is closest */
+
+ for (nslot = slot; nslot < conf->copies; nslot++) {
+ int ndisk = r10_bio->devs[nslot].devnum;
+
+
+ if (!conf->mirrors[ndisk].rdev ||
+ !conf->mirrors[ndisk].rdev->in_sync)
+ continue;
+
+ if (!atomic_read(&conf->mirrors[ndisk].rdev->nr_pending)) {
+ disk = ndisk;
+ slot = nslot;
+ break;
+ }
+ new_distance = abs(r10_bio->devs[nslot].addr -
+ conf->mirrors[ndisk].head_position);
+ if (new_distance < current_distance) {
+ current_distance = new_distance;
+ disk = ndisk;
+ slot = nslot;
+ }
+ }
+
+rb_out:
+ r10_bio->read_slot = slot;
+/* conf->next_seq_sect = this_sector + sectors;*/
+
+ if (disk >= 0 && conf->mirrors[disk].rdev)
+ atomic_inc(&conf->mirrors[disk].rdev->nr_pending);
+ rcu_read_unlock();
+
+ return disk;
+}
+
+static void unplug_slaves(mddev_