diff options
Diffstat (limited to 'libbcachefs/compress.c')
-rw-r--r-- | libbcachefs/compress.c | 396 |
1 files changed, 245 insertions, 151 deletions
diff --git a/libbcachefs/compress.c b/libbcachefs/compress.c index 64079981..7726cfd8 100644 --- a/libbcachefs/compress.c +++ b/libbcachefs/compress.c @@ -8,6 +8,7 @@ #include "lz4.h" #include <linux/lz4.h> #include <linux/zlib.h> +#include <linux/zstd.h> /* Bounce buffer: */ struct bbuf { @@ -151,6 +152,7 @@ static int __bio_uncompress(struct bch_fs *c, struct bio *src, struct bbuf src_data = { NULL }; size_t src_len = src->bi_iter.bi_size; size_t dst_len = crc.uncompressed_size << 9; + void *workspace; int ret; src_data = bio_map_or_bounce(c, src, READ); @@ -159,57 +161,64 @@ static int __bio_uncompress(struct bch_fs *c, struct bio *src, case BCH_COMPRESSION_LZ4_OLD: ret = bch2_lz4_decompress(src_data.b, &src_len, dst_data, dst_len); - if (ret) { - ret = -EIO; + if (ret) goto err; - } break; case BCH_COMPRESSION_LZ4: ret = LZ4_decompress_safe_partial(src_data.b, dst_data, src_len, dst_len, dst_len); - if (ret != dst_len) { - ret = -EIO; + if (ret != dst_len) goto err; - } break; case BCH_COMPRESSION_GZIP: { - void *workspace; - z_stream strm; - - workspace = kmalloc(zlib_inflate_workspacesize(), - GFP_NOIO|__GFP_NOWARN); - if (!workspace) { - mutex_lock(&c->zlib_workspace_lock); - workspace = c->zlib_workspace; - } + z_stream strm = { + .next_in = src_data.b, + .avail_in = src_len, + .next_out = dst_data, + .avail_out = dst_len, + }; + + workspace = mempool_alloc(&c->decompress_workspace, GFP_NOIO); - strm.next_in = src_data.b; - strm.avail_in = src_len; - strm.next_out = dst_data; - strm.avail_out = dst_len; zlib_set_workspace(&strm, workspace); zlib_inflateInit2(&strm, -MAX_WBITS); - ret = zlib_inflate(&strm, Z_FINISH); - if (workspace == c->zlib_workspace) - mutex_unlock(&c->zlib_workspace_lock); - else - kfree(workspace); + mempool_free(workspace, &c->decompress_workspace); - if (ret != Z_STREAM_END) { - ret = -EIO; + if (ret != Z_STREAM_END) + goto err; + break; + } + case BCH_COMPRESSION_ZSTD: { + ZSTD_DCtx *ctx; + size_t len; + + workspace = mempool_alloc(&c->decompress_workspace, GFP_NOIO); + ctx = ZSTD_initDCtx(workspace, ZSTD_DCtxWorkspaceBound()); + + src_len = le32_to_cpup(src_data.b); + + len = ZSTD_decompressDCtx(ctx, + dst_data, dst_len, + src_data.b + 4, src_len); + + mempool_free(workspace, &c->decompress_workspace); + + if (len != dst_len) goto err; - } break; } default: BUG(); } ret = 0; -err: +out: bio_unmap_or_unbounce(c, src_data); return ret; +err: + ret = -EIO; + goto out; } int bch2_bio_uncompress_inplace(struct bch_fs *c, struct bio *bio, @@ -282,113 +291,129 @@ err: return ret; } +static int attempt_compress(struct bch_fs *c, + void *workspace, + void *dst, size_t dst_len, + void *src, size_t src_len, + unsigned compression_type) +{ + switch (compression_type) { + case BCH_COMPRESSION_LZ4: { + int len = src_len; + int ret = LZ4_compress_destSize( + src, dst, + &len, dst_len, + workspace); + + if (len < src_len) + return -len; + + return ret; + } + case BCH_COMPRESSION_GZIP: { + z_stream strm = { + .next_in = src, + .avail_in = src_len, + .next_out = dst, + .avail_out = dst_len, + }; + + zlib_set_workspace(&strm, workspace); + zlib_deflateInit2(&strm, Z_DEFAULT_COMPRESSION, + Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL, + Z_DEFAULT_STRATEGY); + + if (zlib_deflate(&strm, Z_FINISH) != Z_STREAM_END) + return 0; + + if (zlib_deflateEnd(&strm) != Z_OK) + return 0; + + return strm.total_out; + } + case BCH_COMPRESSION_ZSTD: { + ZSTD_CCtx *ctx = ZSTD_initCCtx(workspace, + ZSTD_CCtxWorkspaceBound(c->zstd_params.cParams)); + + size_t len = ZSTD_compressCCtx(ctx, + dst + 4, dst_len - 4, + src, src_len, + c->zstd_params); + if (ZSTD_isError(len)) + return 0; + + *((__le32 *) dst) = cpu_to_le32(len); + return len + 4; + } + default: + BUG(); + } +} + static unsigned __bio_compress(struct bch_fs *c, struct bio *dst, size_t *dst_len, struct bio *src, size_t *src_len, unsigned compression_type) { struct bbuf src_data = { NULL }, dst_data = { NULL }; + void *workspace; unsigned pad; int ret = 0; /* If it's only one block, don't bother trying to compress: */ if (bio_sectors(src) <= c->opts.block_size) - goto err; + return 0; dst_data = bio_map_or_bounce(c, dst, WRITE); src_data = bio_map_or_bounce(c, src, READ); - switch (compression_type) { - case BCH_COMPRESSION_LZ4_OLD: - compression_type = BCH_COMPRESSION_LZ4; + workspace = mempool_alloc(&c->compress_workspace[compression_type], GFP_NOIO); - case BCH_COMPRESSION_LZ4: { - void *workspace; - int len = src->bi_iter.bi_size; - - workspace = mempool_alloc(&c->lz4_workspace_pool, GFP_NOIO); - - while (1) { - if (len <= block_bytes(c)) { - ret = 0; - break; - } - - ret = LZ4_compress_destSize( - src_data.b, dst_data.b, - &len, dst->bi_iter.bi_size, - workspace); - if (ret >= len) { - /* uncompressible: */ - ret = 0; - break; - } - - if (!(len & (block_bytes(c) - 1))) - break; - len = round_down(len, block_bytes(c)); - } - mempool_free(workspace, &c->lz4_workspace_pool); + *src_len = src->bi_iter.bi_size; + *dst_len = dst->bi_iter.bi_size; - if (!ret) - goto err; - - *src_len = len; - *dst_len = ret; - ret = 0; - break; - } - case BCH_COMPRESSION_GZIP: { - void *workspace; - z_stream strm; - - workspace = kmalloc(zlib_deflate_workspacesize(MAX_WBITS, - DEF_MEM_LEVEL), - GFP_NOIO|__GFP_NOWARN); - if (!workspace) { - mutex_lock(&c->zlib_workspace_lock); - workspace = c->zlib_workspace; + /* + * XXX: this algorithm sucks when the compression code doesn't tell us + * how much would fit, like LZ4 does: + */ + while (1) { + if (*src_len <= block_bytes(c)) { + ret = -1; + break; } - strm.next_in = src_data.b; - strm.avail_in = min(src->bi_iter.bi_size, - dst->bi_iter.bi_size); - strm.next_out = dst_data.b; - strm.avail_out = dst->bi_iter.bi_size; - zlib_set_workspace(&strm, workspace); - zlib_deflateInit2(&strm, Z_DEFAULT_COMPRESSION, - Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL, - Z_DEFAULT_STRATEGY); - - ret = zlib_deflate(&strm, Z_FINISH); - if (ret != Z_STREAM_END) { - ret = -EIO; - goto zlib_err; + ret = attempt_compress(c, workspace, + dst_data.b, *dst_len, + src_data.b, *src_len, + compression_type); + if (ret > 0) { + *dst_len = ret; + ret = 0; + break; } - ret = zlib_deflateEnd(&strm); - if (ret != Z_OK) { - ret = -EIO; - goto zlib_err; + /* Didn't fit: should we retry with a smaller amount? */ + if (*src_len <= *dst_len) { + ret = -1; + break; } - ret = 0; -zlib_err: - if (workspace == c->zlib_workspace) - mutex_unlock(&c->zlib_workspace_lock); + /* + * If ret is negative, it's a hint as to how much data would fit + */ + BUG_ON(-ret >= *src_len); + + if (ret < 0) + *src_len = -ret; else - kfree(workspace); + *src_len -= (*src_len - *dst_len) / 2; + *src_len = round_down(*src_len, block_bytes(c)); + } - if (ret) - goto err; + mempool_free(workspace, &c->compress_workspace[compression_type]); - *dst_len = strm.total_out; - *src_len = strm.total_in; - break; - } - default: - BUG(); - } + if (ret) + goto err; /* Didn't get smaller: */ if (round_up(*dst_len, block_bytes(c)) >= *src_len) @@ -429,6 +454,9 @@ unsigned bch2_bio_compress(struct bch_fs *c, /* Don't generate a bigger output than input: */ dst->bi_iter.bi_size = min(dst->bi_iter.bi_size, src->bi_iter.bi_size); + if (compression_type == BCH_COMPRESSION_LZ4_OLD) + compression_type = BCH_COMPRESSION_LZ4; + compression_type = __bio_compress(c, dst, dst_len, src, src_len, compression_type); @@ -437,81 +465,147 @@ unsigned bch2_bio_compress(struct bch_fs *c, return compression_type; } +#define BCH_FEATURE_NONE 0 + +static const unsigned bch2_compression_opt_to_feature[] = { +#define x(t) [BCH_COMPRESSION_OPT_##t] = BCH_FEATURE_##t, + BCH_COMPRESSION_TYPES() +#undef x +}; + +#undef BCH_FEATURE_NONE + /* doesn't write superblock: */ int bch2_check_set_has_compressed_data(struct bch_fs *c, unsigned compression_type) { - switch (compression_type) { - case BCH_COMPRESSION_OPT_NONE: - return 0; - case BCH_COMPRESSION_OPT_LZ4: - if (bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4)) - return 0; + unsigned f; + int ret = 0; - bch2_sb_set_feature(c->disk_sb, BCH_FEATURE_LZ4); - break; - case BCH_COMPRESSION_OPT_GZIP: - if (bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP)) - return 0; + pr_verbose_init(c->opts, ""); - bch2_sb_set_feature(c->disk_sb, BCH_FEATURE_GZIP); - break; - default: - BUG(); - } + BUG_ON(compression_type >= ARRAY_SIZE(bch2_compression_opt_to_feature)); + + if (!compression_type) + goto out; - return bch2_fs_compress_init(c); + f = bch2_compression_opt_to_feature[compression_type]; + if (bch2_sb_test_feature(c->disk_sb, f)) + goto out; + + bch2_sb_set_feature(c->disk_sb, f); + ret = bch2_fs_compress_init(c); +out: + pr_verbose_init(c->opts, "ret %i", ret); + return ret; } void bch2_fs_compress_exit(struct bch_fs *c) { - vfree(c->zlib_workspace); - mempool_exit(&c->lz4_workspace_pool); + unsigned i; + + mempool_exit(&c->decompress_workspace); + for (i = 0; i < ARRAY_SIZE(c->compress_workspace); i++) + mempool_exit(&c->compress_workspace[i]); mempool_exit(&c->compression_bounce[WRITE]); mempool_exit(&c->compression_bounce[READ]); } -#define COMPRESSION_WORKSPACE_SIZE \ - max_t(size_t, zlib_inflate_workspacesize(), \ - zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL)) +static void *mempool_kvpmalloc(gfp_t gfp_mask, void *pool_data) +{ + size_t size = (size_t)pool_data; + return kvpmalloc(size, gfp_mask); +} + +void mempool_kvpfree(void *element, void *pool_data) +{ + size_t size = (size_t)pool_data; + kvpfree(element, size); +} + +static int mempool_init_kvpmalloc_pool(mempool_t *pool, int min_nr, size_t size) +{ + return !mempool_initialized(pool) + ? mempool_init(pool, min_nr, mempool_kvpmalloc, + mempool_kvpfree, (void *) size) + : 0; +} int bch2_fs_compress_init(struct bch_fs *c) { - unsigned order = get_order(c->sb.encoded_extent_max << 9); - int ret; + size_t max_extent = c->sb.encoded_extent_max << 9; + size_t order = get_order(max_extent); + size_t decompress_workspace_size = 0; + bool decompress_workspace_needed; + ZSTD_parameters params = ZSTD_getParams(0, max_extent, 0); + struct { + unsigned feature; + unsigned type; + size_t compress_workspace; + size_t decompress_workspace; + } compression_types[] = { + { BCH_FEATURE_LZ4, BCH_COMPRESSION_LZ4, LZ4_MEM_COMPRESS, 0 }, + { BCH_FEATURE_GZIP, BCH_COMPRESSION_GZIP, + zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL), + zlib_inflate_workspacesize(), }, + { BCH_FEATURE_ZSTD, BCH_COMPRESSION_ZSTD, + ZSTD_CCtxWorkspaceBound(params.cParams), + ZSTD_DCtxWorkspaceBound() }, + }, *i; + int ret = 0; - if (!bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4) && - !bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP)) - return 0; + pr_verbose_init(c->opts, ""); + + c->zstd_params = params; + + for (i = compression_types; + i < compression_types + ARRAY_SIZE(compression_types); + i++) + if (bch2_sb_test_feature(c->disk_sb, i->feature)) + goto have_compressed; + + goto out; +have_compressed: if (!mempool_initialized(&c->compression_bounce[READ])) { ret = mempool_init_page_pool(&c->compression_bounce[READ], 1, order); if (ret) - return ret; + goto out; } if (!mempool_initialized(&c->compression_bounce[WRITE])) { ret = mempool_init_page_pool(&c->compression_bounce[WRITE], 1, order); if (ret) - return ret; + goto out; } - if (!mempool_initialized(&c->lz4_workspace_pool) && - bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4)) { - ret = mempool_init_kmalloc_pool(&c->lz4_workspace_pool, - 1, LZ4_MEM_COMPRESS); - if (ret) - return ret; - } + for (i = compression_types; + i < compression_types + ARRAY_SIZE(compression_types); + i++) { + decompress_workspace_size = + max(decompress_workspace_size, i->decompress_workspace); - if (!c->zlib_workspace && - bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP)) { - c->zlib_workspace = vmalloc(COMPRESSION_WORKSPACE_SIZE); - if (!c->zlib_workspace) - return -ENOMEM; + if (!bch2_sb_test_feature(c->disk_sb, i->feature)) + continue; + + if (i->decompress_workspace) + decompress_workspace_needed = true; + + ret = mempool_init_kvpmalloc_pool( + &c->compress_workspace[i->type], + 1, i->compress_workspace); + if (ret) + goto out; } - return 0; + ret = mempool_init_kmalloc_pool( + &c->decompress_workspace, + 1, decompress_workspace_size); + if (ret) + goto out; +out: + pr_verbose_init(c->opts, "ret %i", ret); + return ret; } |