summaryrefslogtreecommitdiff
path: root/drivers/md/dm-crypt.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/md/dm-crypt.c')
-rw-r--r--drivers/md/dm-crypt.c81
1 files changed, 54 insertions, 27 deletions
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 1b7a97cc3779..348b4b26c272 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -214,7 +214,8 @@ struct crypt_config {
unsigned int integrity_tag_size;
unsigned int integrity_iv_size;
- unsigned int on_disk_tag_size;
+ unsigned int used_tag_size;
+ unsigned int tuple_size;
/*
* pool for per bio private data, crypto requests,
@@ -241,6 +242,31 @@ static unsigned int dm_crypt_clients_n;
static volatile unsigned long dm_crypt_pages_per_client;
#define DM_CRYPT_MEMORY_PERCENT 2
#define DM_CRYPT_MIN_PAGES_PER_CLIENT (BIO_MAX_VECS * 16)
+#define DM_CRYPT_DEFAULT_MAX_READ_SIZE 131072
+#define DM_CRYPT_DEFAULT_MAX_WRITE_SIZE 131072
+
+static unsigned int max_read_size = 0;
+module_param(max_read_size, uint, 0644);
+MODULE_PARM_DESC(max_read_size, "Maximum size of a read request");
+static unsigned int max_write_size = 0;
+module_param(max_write_size, uint, 0644);
+MODULE_PARM_DESC(max_write_size, "Maximum size of a write request");
+static unsigned get_max_request_size(struct crypt_config *cc, bool wrt)
+{
+ unsigned val, sector_align;
+ val = !wrt ? READ_ONCE(max_read_size) : READ_ONCE(max_write_size);
+ if (likely(!val))
+ val = !wrt ? DM_CRYPT_DEFAULT_MAX_READ_SIZE : DM_CRYPT_DEFAULT_MAX_WRITE_SIZE;
+ if (wrt || cc->used_tag_size) {
+ if (unlikely(val > BIO_MAX_VECS << PAGE_SHIFT))
+ val = BIO_MAX_VECS << PAGE_SHIFT;
+ }
+ sector_align = max(bdev_logical_block_size(cc->dev->bdev), (unsigned)cc->sector_size);
+ val = round_down(val, sector_align);
+ if (unlikely(!val))
+ val = sector_align;
+ return val >> SECTOR_SHIFT;
+}
static void crypt_endio(struct bio *clone);
static void kcryptd_queue_crypt(struct dm_crypt_io *io);
@@ -1151,14 +1177,14 @@ static int dm_crypt_integrity_io_alloc(struct dm_crypt_io *io, struct bio *bio)
unsigned int tag_len;
int ret;
- if (!bio_sectors(bio) || !io->cc->on_disk_tag_size)
+ if (!bio_sectors(bio) || !io->cc->tuple_size)
return 0;
bip = bio_integrity_alloc(bio, GFP_NOIO, 1);
if (IS_ERR(bip))
return PTR_ERR(bip);
- tag_len = io->cc->on_disk_tag_size * (bio_sectors(bio) >> io->cc->sector_shift);
+ tag_len = io->cc->tuple_size * (bio_sectors(bio) >> io->cc->sector_shift);
bip->bip_iter.bi_sector = io->cc->start + io->sector;
@@ -1176,24 +1202,24 @@ static int crypt_integrity_ctr(struct crypt_config *cc, struct dm_target *ti)
struct blk_integrity *bi = blk_get_integrity(cc->dev->bdev->bd_disk);
struct mapped_device *md = dm_table_get_md(ti->table);
- /* From now we require underlying device with our integrity profile */
- if (!bi || strcasecmp(bi->profile->name, "DM-DIF-EXT-TAG")) {
+ /* We require an underlying device with non-PI metadata */
+ if (!bi || bi->csum_type != BLK_INTEGRITY_CSUM_NONE) {
ti->error = "Integrity profile not supported.";
return -EINVAL;
}
- if (bi->tag_size != cc->on_disk_tag_size ||
- bi->tuple_size != cc->on_disk_tag_size) {
+ if (bi->tuple_size < cc->used_tag_size) {
ti->error = "Integrity profile tag size mismatch.";
return -EINVAL;
}
+ cc->tuple_size = bi->tuple_size;
if (1 << bi->interval_exp != cc->sector_size) {
ti->error = "Integrity profile sector size mismatch.";
return -EINVAL;
}
if (crypt_integrity_aead(cc)) {
- cc->integrity_tag_size = cc->on_disk_tag_size - cc->integrity_iv_size;
+ cc->integrity_tag_size = cc->used_tag_size - cc->integrity_iv_size;
DMDEBUG("%s: Integrity AEAD, tag size %u, IV size %u.", dm_device_name(md),
cc->integrity_tag_size, cc->integrity_iv_size);
@@ -1205,7 +1231,7 @@ static int crypt_integrity_ctr(struct crypt_config *cc, struct dm_target *ti)
DMDEBUG("%s: Additional per-sector space %u bytes for IV.", dm_device_name(md),
cc->integrity_iv_size);
- if ((cc->integrity_tag_size + cc->integrity_iv_size) != bi->tag_size) {
+ if ((cc->integrity_tag_size + cc->integrity_iv_size) > cc->tuple_size) {
ti->error = "Not enough space for integrity tag in the profile.";
return -EINVAL;
}
@@ -1284,7 +1310,7 @@ static void *tag_from_dmreq(struct crypt_config *cc,
struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);
return &io->integrity_metadata[*org_tag_of_dmreq(cc, dmreq) *
- cc->on_disk_tag_size];
+ cc->tuple_size];
}
static void *iv_tag_from_dmreq(struct crypt_config *cc,
@@ -1365,9 +1391,9 @@ static int crypt_convert_block_aead(struct crypt_config *cc,
aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out,
cc->sector_size, iv);
r = crypto_aead_encrypt(req);
- if (cc->integrity_tag_size + cc->integrity_iv_size != cc->on_disk_tag_size)
+ if (cc->integrity_tag_size + cc->integrity_iv_size != cc->tuple_size)
memset(tag + cc->integrity_tag_size + cc->integrity_iv_size, 0,
- cc->on_disk_tag_size - (cc->integrity_tag_size + cc->integrity_iv_size));
+ cc->tuple_size - (cc->integrity_tag_size + cc->integrity_iv_size));
} else {
aead_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out,
cc->sector_size + cc->integrity_tag_size, iv);
@@ -1797,7 +1823,7 @@ static void crypt_dec_pending(struct dm_crypt_io *io)
return;
if (likely(!io->ctx.aead_recheck) && unlikely(io->ctx.aead_failed) &&
- cc->on_disk_tag_size && bio_data_dir(base_bio) == READ) {
+ cc->used_tag_size && bio_data_dir(base_bio) == READ) {
io->ctx.aead_recheck = true;
io->ctx.aead_failed = false;
io->error = 0;
@@ -3181,7 +3207,7 @@ static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **ar
ti->error = "Invalid integrity arguments";
return -EINVAL;
}
- cc->on_disk_tag_size = val;
+ cc->used_tag_size = val;
sval = strchr(opt_string + strlen("integrity:"), ':') + 1;
if (!strcasecmp(sval, "aead")) {
set_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags);
@@ -3393,12 +3419,12 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
if (ret)
goto bad;
- cc->tag_pool_max_sectors = POOL_ENTRY_SIZE / cc->on_disk_tag_size;
+ cc->tag_pool_max_sectors = POOL_ENTRY_SIZE / cc->tuple_size;
if (!cc->tag_pool_max_sectors)
cc->tag_pool_max_sectors = 1;
ret = mempool_init_kmalloc_pool(&cc->tag_pool, MIN_IOS,
- cc->tag_pool_max_sectors * cc->on_disk_tag_size);
+ cc->tag_pool_max_sectors * cc->tuple_size);
if (ret) {
ti->error = "Cannot allocate integrity tags mempool";
goto bad;
@@ -3474,6 +3500,7 @@ static int crypt_map(struct dm_target *ti, struct bio *bio)
{
struct dm_crypt_io *io;
struct crypt_config *cc = ti->private;
+ unsigned max_sectors;
/*
* If bio is REQ_PREFLUSH or REQ_OP_DISCARD, just bypass crypt queues.
@@ -3492,9 +3519,9 @@ static int crypt_map(struct dm_target *ti, struct bio *bio)
/*
* Check if bio is too large, split as needed.
*/
- if (unlikely(bio->bi_iter.bi_size > (BIO_MAX_VECS << PAGE_SHIFT)) &&
- (bio_data_dir(bio) == WRITE || cc->on_disk_tag_size))
- dm_accept_partial_bio(bio, ((BIO_MAX_VECS << PAGE_SHIFT) >> SECTOR_SHIFT));
+ max_sectors = get_max_request_size(cc, bio_data_dir(bio) == WRITE);
+ if (unlikely(bio_sectors(bio) > max_sectors))
+ dm_accept_partial_bio(bio, max_sectors);
/*
* Ensure that bio is a multiple of internal sector encryption size
@@ -3509,8 +3536,8 @@ static int crypt_map(struct dm_target *ti, struct bio *bio)
io = dm_per_bio_data(bio, cc->per_bio_data_size);
crypt_io_init(io, cc, bio, dm_target_offset(ti, bio->bi_iter.bi_sector));
- if (cc->on_disk_tag_size) {
- unsigned int tag_len = cc->on_disk_tag_size * (bio_sectors(bio) >> cc->sector_shift);
+ if (cc->tuple_size) {
+ unsigned int tag_len = cc->tuple_size * (bio_sectors(bio) >> cc->sector_shift);
if (unlikely(tag_len > KMALLOC_MAX_SIZE))
io->integrity_metadata = NULL;
@@ -3582,7 +3609,7 @@ static void crypt_status(struct dm_target *ti, status_type_t type,
num_feature_args += test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags);
num_feature_args += cc->sector_size != (1 << SECTOR_SHIFT);
num_feature_args += test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags);
- if (cc->on_disk_tag_size)
+ if (cc->used_tag_size)
num_feature_args++;
if (num_feature_args) {
DMEMIT(" %d", num_feature_args);
@@ -3598,8 +3625,8 @@ static void crypt_status(struct dm_target *ti, status_type_t type,
DMEMIT(" no_read_workqueue");
if (test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags))
DMEMIT(" no_write_workqueue");
- if (cc->on_disk_tag_size)
- DMEMIT(" integrity:%u:%s", cc->on_disk_tag_size, cc->cipher_auth);
+ if (cc->used_tag_size)
+ DMEMIT(" integrity:%u:%s", cc->used_tag_size, cc->cipher_auth);
if (cc->sector_size != (1 << SECTOR_SHIFT))
DMEMIT(" sector_size:%d", cc->sector_size);
if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
@@ -3621,9 +3648,9 @@ static void crypt_status(struct dm_target *ti, status_type_t type,
DMEMIT(",iv_large_sectors=%c", test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags) ?
'y' : 'n');
- if (cc->on_disk_tag_size)
+ if (cc->used_tag_size)
DMEMIT(",integrity_tag_size=%u,cipher_auth=%s",
- cc->on_disk_tag_size, cc->cipher_auth);
+ cc->used_tag_size, cc->cipher_auth);
if (cc->sector_size != (1 << SECTOR_SHIFT))
DMEMIT(",sector_size=%d", cc->sector_size);
if (cc->cipher_string)
@@ -3731,7 +3758,7 @@ static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits)
static struct target_type crypt_target = {
.name = "crypt",
- .version = {1, 26, 0},
+ .version = {1, 27, 0},
.module = THIS_MODULE,
.ctr = crypt_ctr,
.dtr = crypt_dtr,
generated by cgit v1.2.3 (git 2.46.0) at 2025-09-08 06:09:54 +0000