summaryrefslogtreecommitdiff
path: root/libbcache/compress.c
diff options
context:
space:
mode:
Diffstat (limited to 'libbcache/compress.c')
-rw-r--r--libbcache/compress.c500
1 files changed, 0 insertions, 500 deletions
diff --git a/libbcache/compress.c b/libbcache/compress.c
deleted file mode 100644
index d9a64c38..00000000
--- a/libbcache/compress.c
+++ /dev/null
@@ -1,500 +0,0 @@
-#include "bcache.h"
-#include "compress.h"
-#include "extents.h"
-#include "io.h"
-#include "super-io.h"
-
-#include <linux/lz4.h>
-#include <linux/zlib.h>
-
-enum bounced {
- BOUNCED_CONTIG,
- BOUNCED_MAPPED,
- BOUNCED_KMALLOCED,
- BOUNCED_VMALLOCED,
- BOUNCED_MEMPOOLED,
-};
-
-static void *__bounce_alloc(struct bch_fs *c, unsigned size,
- unsigned *bounced, int direction)
-{
- void *data;
-
- *bounced = BOUNCED_KMALLOCED;
- data = kmalloc(size, GFP_NOIO|__GFP_NOWARN);
- if (data)
- return data;
-
- *bounced = BOUNCED_MEMPOOLED;
- data = mempool_alloc(&c->compression_bounce[direction], GFP_NOWAIT);
- if (data)
- return page_address(data);
-
- *bounced = BOUNCED_VMALLOCED;
- data = vmalloc(size);
- if (data)
- return data;
-
- *bounced = BOUNCED_MEMPOOLED;
- data = mempool_alloc(&c->compression_bounce[direction], GFP_NOIO);
- return page_address(data);
-}
-
-static void *__bio_map_or_bounce(struct bch_fs *c,
- struct bio *bio, struct bvec_iter start,
- unsigned *bounced, int direction)
-{
- struct bio_vec bv;
- struct bvec_iter iter;
- unsigned nr_pages = 0;
- struct page *stack_pages[16];
- struct page **pages = NULL;
- bool first = true;
- unsigned prev_end = PAGE_SIZE;
- void *data;
-
- BUG_ON(bvec_iter_sectors(start) > BCH_ENCODED_EXTENT_MAX);
-
-#ifndef CONFIG_HIGHMEM
- *bounced = BOUNCED_CONTIG;
-
- __bio_for_each_contig_segment(bv, bio, iter, start) {
- if (bv.bv_len == start.bi_size)
- return page_address(bv.bv_page) + bv.bv_offset;
- }
-#endif
- *bounced = BOUNCED_MAPPED;
-
- __bio_for_each_segment(bv, bio, iter, start) {
- if ((!first && bv.bv_offset) ||
- prev_end != PAGE_SIZE)
- goto bounce;
-
- prev_end = bv.bv_offset + bv.bv_len;
- nr_pages++;
- }
-
- BUG_ON(DIV_ROUND_UP(start.bi_size, PAGE_SIZE) > nr_pages);
-
- pages = nr_pages > ARRAY_SIZE(stack_pages)
- ? kmalloc_array(nr_pages, sizeof(struct page *), GFP_NOIO)
- : stack_pages;
- if (!pages)
- goto bounce;
-
- nr_pages = 0;
- __bio_for_each_segment(bv, bio, iter, start)
- pages[nr_pages++] = bv.bv_page;
-
- data = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
- if (pages != stack_pages)
- kfree(pages);
-
- return data + bio_iter_offset(bio, start);
-bounce:
- data = __bounce_alloc(c, start.bi_size, bounced, direction);
-
- if (direction == READ)
- memcpy_from_bio(data, bio, start);
-
- return data;
-}
-
-static void *bio_map_or_bounce(struct bch_fs *c, struct bio *bio,
- unsigned *bounced, int direction)
-{
- return __bio_map_or_bounce(c, bio, bio->bi_iter, bounced, direction);
-}
-
-static void bio_unmap_or_unbounce(struct bch_fs *c, void *data,
- unsigned bounced, int direction)
-{
- if (!data)
- return;
-
- switch (bounced) {
- case BOUNCED_MAPPED:
- vunmap((void *) ((unsigned long) data & PAGE_MASK));
- return;
- case BOUNCED_KMALLOCED:
- kfree(data);
- return;
- case BOUNCED_VMALLOCED:
- vfree(data);
- return;
- case BOUNCED_MEMPOOLED:
- mempool_free(virt_to_page(data), &c->compression_bounce[direction]);
- return;
- }
-}
-
-static inline void zlib_set_workspace(z_stream *strm, void *workspace)
-{
-#ifdef __KERNEL__
- strm->workspace = workspace;
-#endif
-}
-
-static int __bio_uncompress(struct bch_fs *c, struct bio *src,
- void *dst_data, struct bch_extent_crc128 crc)
-{
- void *src_data = NULL;
- unsigned src_bounced;
- size_t src_len = src->bi_iter.bi_size;
- size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
- int ret;
-
- src_data = bio_map_or_bounce(c, src, &src_bounced, READ);
-
- switch (crc.compression_type) {
- case BCH_COMPRESSION_LZ4:
- ret = lz4_decompress(src_data, &src_len,
- dst_data, dst_len);
- if (ret) {
- ret = -EIO;
- goto err;
- }
- break;
- case BCH_COMPRESSION_GZIP: {
- void *workspace;
- z_stream strm;
-
- workspace = kmalloc(zlib_inflate_workspacesize(),
- GFP_NOIO|__GFP_NOWARN);
- if (!workspace) {
- mutex_lock(&c->zlib_workspace_lock);
- workspace = c->zlib_workspace;
- }
-
- strm.next_in = src_data;
- strm.avail_in = src_len;
- strm.next_out = dst_data;
- strm.avail_out = dst_len;
- zlib_set_workspace(&strm, workspace);
- zlib_inflateInit2(&strm, -MAX_WBITS);
-
- ret = zlib_inflate(&strm, Z_FINISH);
-
- if (workspace == c->zlib_workspace)
- mutex_unlock(&c->zlib_workspace_lock);
- else
- kfree(workspace);
-
- if (ret != Z_STREAM_END) {
- ret = -EIO;
- goto err;
- }
- break;
- }
- default:
- BUG();
- }
- ret = 0;
-err:
- bio_unmap_or_unbounce(c, src_data, src_bounced, READ);
- return ret;
-}
-
-int bch_bio_uncompress_inplace(struct bch_fs *c, struct bio *bio,
- unsigned live_data_sectors,
- struct bch_extent_crc128 crc)
-{
- void *dst_data = NULL;
- size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
- int ret = -ENOMEM;
-
- BUG_ON(DIV_ROUND_UP(live_data_sectors, PAGE_SECTORS) > bio->bi_max_vecs);
-
- /* XXX mempoolify */
- dst_data = kmalloc(dst_len, GFP_NOIO|__GFP_NOWARN);
- if (!dst_data) {
- dst_data = vmalloc(dst_len);
- if (!dst_data)
- goto err;
- }
-
- ret = __bio_uncompress(c, bio, dst_data, crc);
- if (ret)
- goto err;
-
- while (bio->bi_vcnt < DIV_ROUND_UP(live_data_sectors, PAGE_SECTORS)) {
- struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt];
-
- bv->bv_page = alloc_page(GFP_NOIO);
- if (!bv->bv_page)
- goto use_mempool;
-
- bv->bv_len = PAGE_SIZE;
- bv->bv_offset = 0;
- bio->bi_vcnt++;
- }
-
- bio->bi_iter.bi_size = live_data_sectors << 9;
-copy_data:
- memcpy_to_bio(bio, bio->bi_iter, dst_data + (crc.offset << 9));
-err:
- kvfree(dst_data);
- return ret;
-use_mempool:
- /*
- * We already allocated from mempool, we can't allocate from it again
- * without freeing the pages we already allocated or else we could
- * deadlock:
- */
-
- bch_bio_free_pages_pool(c, bio);
- bch_bio_alloc_pages_pool(c, bio, live_data_sectors << 9);
- goto copy_data;
-}
-
-int bch_bio_uncompress(struct bch_fs *c, struct bio *src,
- struct bio *dst, struct bvec_iter dst_iter,
- struct bch_extent_crc128 crc)
-{
- void *dst_data = NULL;
- unsigned dst_bounced;
- size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
- int ret = -ENOMEM;
-
- dst_data = dst_len == dst_iter.bi_size
- ? __bio_map_or_bounce(c, dst, dst_iter, &dst_bounced, WRITE)
- : __bounce_alloc(c, dst_len, &dst_bounced, WRITE);
-
- ret = __bio_uncompress(c, src, dst_data, crc);
- if (ret)
- goto err;
-
- if (dst_bounced)
- memcpy_to_bio(dst, dst_iter, dst_data + (crc.offset << 9));
-err:
- bio_unmap_or_unbounce(c, dst_data, dst_bounced, WRITE);
- return ret;
-}
-
-static int __bio_compress(struct bch_fs *c,
- struct bio *dst, size_t *dst_len,
- struct bio *src, size_t *src_len,
- unsigned compression_type)
-{
- void *src_data = NULL, *dst_data = NULL;
- unsigned src_bounced, dst_bounced, pad;
- int ret = -1;
-
- dst_data = bio_map_or_bounce(c, dst, &dst_bounced, WRITE);
- src_data = bio_map_or_bounce(c, src, &src_bounced, READ);
-
- switch (compression_type) {
- case BCH_COMPRESSION_LZ4: {
- void *workspace;
-
- *dst_len = dst->bi_iter.bi_size;
- *src_len = src->bi_iter.bi_size;
-
- workspace = mempool_alloc(&c->lz4_workspace_pool, GFP_NOIO);
-
- while (*src_len > block_bytes(c) &&
- (ret = lz4_compress(src_data, *src_len,
- dst_data, dst_len,
- workspace))) {
- /*
- * On error, the compressed data was bigger than
- * dst_len, and -ret is the amount of data we were able
- * to compress - round down to nearest block and try
- * again:
- */
- BUG_ON(ret > 0);
- BUG_ON(-ret >= *src_len);
-
- *src_len = round_down(-ret, block_bytes(c));
- }
-
- mempool_free(workspace, &c->lz4_workspace_pool);
-
- if (ret)
- goto err;
- break;
- }
- case BCH_COMPRESSION_GZIP: {
- void *workspace;
- z_stream strm;
-
- workspace = kmalloc(zlib_deflate_workspacesize(MAX_WBITS,
- DEF_MEM_LEVEL),
- GFP_NOIO|__GFP_NOWARN);
- if (!workspace) {
- mutex_lock(&c->zlib_workspace_lock);
- workspace = c->zlib_workspace;
- }
-
- strm.next_in = src_data;
- strm.avail_in = min(src->bi_iter.bi_size,
- dst->bi_iter.bi_size);
- strm.next_out = dst_data;
- strm.avail_out = dst->bi_iter.bi_size;
- zlib_set_workspace(&strm, workspace);
- zlib_deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
- Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL,
- Z_DEFAULT_STRATEGY);
-
- ret = zlib_deflate(&strm, Z_FINISH);
- if (ret != Z_STREAM_END) {
- ret = -EIO;
- goto zlib_err;
- }
-
- ret = zlib_deflateEnd(&strm);
- if (ret != Z_OK) {
- ret = -EIO;
- goto zlib_err;
- }
-
- ret = 0;
-zlib_err:
- if (workspace == c->zlib_workspace)
- mutex_unlock(&c->zlib_workspace_lock);
- else
- kfree(workspace);
-
- if (ret)
- goto err;
-
- *dst_len = strm.total_out;
- *src_len = strm.total_in;
- break;
- }
- default:
- BUG();
- }
-
- BUG_ON(!*dst_len);
- BUG_ON(*dst_len > dst->bi_iter.bi_size);
-
- BUG_ON(*src_len & (block_bytes(c) - 1));
- BUG_ON(*src_len > src->bi_iter.bi_size);
-
- /* Didn't get smaller: */
- if (round_up(*dst_len, block_bytes(c)) >= *src_len) {
- ret = -1;
- goto err;
- }
-
- pad = round_up(*dst_len, block_bytes(c)) - *dst_len;
-
- memset(dst_data + *dst_len, 0, pad);
- *dst_len += pad;
-
- if (dst_bounced)
- memcpy_to_bio(dst, dst->bi_iter, dst_data);
-err:
- bio_unmap_or_unbounce(c, src_data, src_bounced, READ);
- bio_unmap_or_unbounce(c, dst_data, dst_bounced, WRITE);
- return ret;
-}
-
-void bch_bio_compress(struct bch_fs *c,
- struct bio *dst, size_t *dst_len,
- struct bio *src, size_t *src_len,
- unsigned *compression_type)
-{
- unsigned orig_dst = dst->bi_iter.bi_size;
- unsigned orig_src = src->bi_iter.bi_size;
-
- /* Don't consume more than BCH_ENCODED_EXTENT_MAX from @src: */
- src->bi_iter.bi_size =
- min(src->bi_iter.bi_size, BCH_ENCODED_EXTENT_MAX << 9);
-
- /* Don't generate a bigger output than input: */
- dst->bi_iter.bi_size =
- min(dst->bi_iter.bi_size, src->bi_iter.bi_size);
-
- /* If it's only one block, don't bother trying to compress: */
- if (*compression_type != BCH_COMPRESSION_NONE &&
- bio_sectors(src) > c->sb.block_size &&
- !__bio_compress(c, dst, dst_len, src, src_len, *compression_type))
- goto out;
-
- /* If compressing failed (didn't get smaller), just copy: */
- *compression_type = BCH_COMPRESSION_NONE;
- *dst_len = *src_len = min(dst->bi_iter.bi_size, src->bi_iter.bi_size);
- bio_copy_data(dst, src);
-out:
- dst->bi_iter.bi_size = orig_dst;
- src->bi_iter.bi_size = orig_src;
-}
-
-/* doesn't write superblock: */
-int bch_check_set_has_compressed_data(struct bch_fs *c,
- unsigned compression_type)
-{
- switch (compression_type) {
- case BCH_COMPRESSION_NONE:
- return 0;
- case BCH_COMPRESSION_LZ4:
- if (bch_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4))
- return 0;
-
- bch_sb_set_feature(c->disk_sb, BCH_FEATURE_LZ4);
- break;
- case BCH_COMPRESSION_GZIP:
- if (bch_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP))
- return 0;
-
- bch_sb_set_feature(c->disk_sb, BCH_FEATURE_GZIP);
- break;
- }
-
- return bch_fs_compress_init(c);
-}
-
-void bch_fs_compress_exit(struct bch_fs *c)
-{
- vfree(c->zlib_workspace);
- mempool_exit(&c->lz4_workspace_pool);
- mempool_exit(&c->compression_bounce[WRITE]);
- mempool_exit(&c->compression_bounce[READ]);
-}
-
-#define COMPRESSION_WORKSPACE_SIZE \
- max_t(size_t, zlib_inflate_workspacesize(), \
- zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL))
-
-int bch_fs_compress_init(struct bch_fs *c)
-{
- unsigned order = get_order(BCH_ENCODED_EXTENT_MAX << 9);
- int ret;
-
- if (!bch_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4) &&
- !bch_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP))
- return 0;
-
- if (!mempool_initialized(&c->compression_bounce[READ])) {
- ret = mempool_init_page_pool(&c->compression_bounce[READ],
- 1, order);
- if (ret)
- return ret;
- }
-
- if (!mempool_initialized(&c->compression_bounce[WRITE])) {
- ret = mempool_init_page_pool(&c->compression_bounce[WRITE],
- 1, order);
- if (ret)
- return ret;
- }
-
- if (!mempool_initialized(&c->lz4_workspace_pool) &&
- bch_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4)) {
- ret = mempool_init_kmalloc_pool(&c->lz4_workspace_pool,
- 1, LZ4_MEM_COMPRESS);
- if (ret)
- return ret;
- }
-
- if (!c->zlib_workspace &&
- bch_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP)) {
- c->zlib_workspace = vmalloc(COMPRESSION_WORKSPACE_SIZE);
- if (!c->zlib_workspace)
- return -ENOMEM;
- }
-
- return 0;
-}