mirror of
https://github.com/facebook/zstd.git
synced 2025-07-30 22:23:13 +03:00
[linux] Write all the patch summaries
This commit is contained in:
@ -1,6 +1,6 @@
|
|||||||
From 36b990a005e68a30495e2b2981e30c5ce97a8591 Mon Sep 17 00:00:00 2001
|
From e75beb7c2e05550b2846e31ad8a0082c188504da Mon Sep 17 00:00:00 2001
|
||||||
From: Nick Terrell <terrelln@fb.com>
|
From: Nick Terrell <terrelln@fb.com>
|
||||||
Date: Mon, 12 Jun 2017 12:13:48 -0700
|
Date: Wed, 21 Jun 2017 17:27:42 -0700
|
||||||
Subject: [PATCH 1/4] lib: Add xxhash module
|
Subject: [PATCH 1/4] lib: Add xxhash module
|
||||||
|
|
||||||
Adds xxhash kernel module with xxh32 and xxh64 hashes. xxhash is an
|
Adds xxhash kernel module with xxh32 and xxh64 hashes. xxhash is an
|
||||||
@ -324,10 +324,10 @@ index 0000000..9e1f42c
|
|||||||
+
|
+
|
||||||
+#endif /* XXHASH_H */
|
+#endif /* XXHASH_H */
|
||||||
diff --git a/lib/Kconfig b/lib/Kconfig
|
diff --git a/lib/Kconfig b/lib/Kconfig
|
||||||
index 260a80e..9db178f 100644
|
index 0c8b78a..b6009d7 100644
|
||||||
--- a/lib/Kconfig
|
--- a/lib/Kconfig
|
||||||
+++ b/lib/Kconfig
|
+++ b/lib/Kconfig
|
||||||
@@ -185,6 +185,9 @@ config CRC8
|
@@ -184,6 +184,9 @@ config CRC8
|
||||||
when they need to do cyclic redundancy check according CRC8
|
when they need to do cyclic redundancy check according CRC8
|
||||||
algorithm. Module will be called crc8.
|
algorithm. Module will be called crc8.
|
||||||
|
|
||||||
@ -338,10 +338,10 @@ index 260a80e..9db178f 100644
|
|||||||
bool
|
bool
|
||||||
depends on AUDIT && !AUDIT_ARCH
|
depends on AUDIT && !AUDIT_ARCH
|
||||||
diff --git a/lib/Makefile b/lib/Makefile
|
diff --git a/lib/Makefile b/lib/Makefile
|
||||||
index 50144a3..5644bad 100644
|
index 0166fbc..1338226 100644
|
||||||
--- a/lib/Makefile
|
--- a/lib/Makefile
|
||||||
+++ b/lib/Makefile
|
+++ b/lib/Makefile
|
||||||
@@ -93,6 +93,7 @@ obj-$(CONFIG_CRC32) += crc32.o
|
@@ -102,6 +102,7 @@ obj-$(CONFIG_CRC32_SELFTEST) += crc32test.o
|
||||||
obj-$(CONFIG_CRC7) += crc7.o
|
obj-$(CONFIG_CRC7) += crc7.o
|
||||||
obj-$(CONFIG_LIBCRC32C) += libcrc32c.o
|
obj-$(CONFIG_LIBCRC32C) += libcrc32c.o
|
||||||
obj-$(CONFIG_CRC8) += crc8.o
|
obj-$(CONFIG_CRC8) += crc8.o
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
From 536a2c59c5bc58f526e69dd0a35b83d39508ae74 Mon Sep 17 00:00:00 2001
|
From b52ae824ae6c0f7c7786380b34da9daaa54bfc26 Mon Sep 17 00:00:00 2001
|
||||||
From: Nick Terrell <terrelln@fb.com>
|
From: Nick Terrell <terrelln@fb.com>
|
||||||
Date: Wed, 21 Jun 2017 14:56:21 -0700
|
Date: Wed, 21 Jun 2017 17:31:24 -0700
|
||||||
Subject: [PATCH 2/4] lib: Add zstd modules
|
Subject: [PATCH 2/4] lib: Add zstd modules
|
||||||
|
|
||||||
Add zstd compression and decompression kernel modules.
|
Add zstd compression and decompression kernel modules.
|
||||||
@ -1304,10 +1304,10 @@ index 0000000..249575e
|
|||||||
+
|
+
|
||||||
+#endif /* ZSTD_H */
|
+#endif /* ZSTD_H */
|
||||||
diff --git a/lib/Kconfig b/lib/Kconfig
|
diff --git a/lib/Kconfig b/lib/Kconfig
|
||||||
index 9db178f..8deab4f 100644
|
index b6009d7..f00ddab 100644
|
||||||
--- a/lib/Kconfig
|
--- a/lib/Kconfig
|
||||||
+++ b/lib/Kconfig
|
+++ b/lib/Kconfig
|
||||||
@@ -242,6 +242,14 @@ config LZ4HC_COMPRESS
|
@@ -241,6 +241,14 @@ config LZ4HC_COMPRESS
|
||||||
config LZ4_DECOMPRESS
|
config LZ4_DECOMPRESS
|
||||||
tristate
|
tristate
|
||||||
|
|
||||||
@ -1323,10 +1323,10 @@ index 9db178f..8deab4f 100644
|
|||||||
|
|
||||||
#
|
#
|
||||||
diff --git a/lib/Makefile b/lib/Makefile
|
diff --git a/lib/Makefile b/lib/Makefile
|
||||||
index 5644bad..aa630d4 100644
|
index 1338226..4fcef16 100644
|
||||||
--- a/lib/Makefile
|
--- a/lib/Makefile
|
||||||
+++ b/lib/Makefile
|
+++ b/lib/Makefile
|
||||||
@@ -107,6 +107,8 @@ obj-$(CONFIG_LZO_DECOMPRESS) += lzo/
|
@@ -116,6 +116,8 @@ obj-$(CONFIG_LZO_DECOMPRESS) += lzo/
|
||||||
obj-$(CONFIG_LZ4_COMPRESS) += lz4/
|
obj-$(CONFIG_LZ4_COMPRESS) += lz4/
|
||||||
obj-$(CONFIG_LZ4HC_COMPRESS) += lz4/
|
obj-$(CONFIG_LZ4HC_COMPRESS) += lz4/
|
||||||
obj-$(CONFIG_LZ4_DECOMPRESS) += lz4/
|
obj-$(CONFIG_LZ4_DECOMPRESS) += lz4/
|
||||||
|
@ -1,3 +1,83 @@
|
|||||||
|
From 599f8f2aaace3df939cb145368574a52268d82d0 Mon Sep 17 00:00:00 2001
|
||||||
|
From: Nick Terrell <terrelln@fb.com>
|
||||||
|
Date: Wed, 21 Jun 2017 17:31:39 -0700
|
||||||
|
Subject: [PATCH 3/4] btrfs: Add zstd support
|
||||||
|
|
||||||
|
Add zstd compression and decompression support to BtrFS. zstd at its
|
||||||
|
fastest level compresses almost as well as zlib, while offering much
|
||||||
|
faster compression and decompression, approaching lzo speeds.
|
||||||
|
|
||||||
|
I benchmarked btrfs with zstd compression against no compression, lzo
|
||||||
|
compression, and zlib compression. I benchmarked two scenarios. Copying
|
||||||
|
a set of files to btrfs, and then reading the files. Copying a tarball
|
||||||
|
to btrfs, extracting it to btrfs, and then reading the extracted files.
|
||||||
|
After every operation, I call `sync` and include the sync time.
|
||||||
|
Between every pair of operations I unmount and remount the filesystem
|
||||||
|
to avoid caching. The benchmark files can be found in the upstream
|
||||||
|
zstd source repository under
|
||||||
|
`contrib/linux-kernel/{btrfs-benchmark.sh,btrfs-extract-benchmark.sh}`
|
||||||
|
[1] [2].
|
||||||
|
|
||||||
|
I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
|
||||||
|
The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor,
|
||||||
|
16 GB of RAM, and a SSD.
|
||||||
|
|
||||||
|
The first compression benchmark is copying 10 copies of the unzipped
|
||||||
|
Silesia corpus [3] into a BtrFS filesystem mounted with
|
||||||
|
`-o compress-force=Method`. The decompression benchmark times how long
|
||||||
|
it takes to `tar` all 10 copies into `/dev/null`. The compression ratio is
|
||||||
|
measured by comparing the output of `df` and `du`. See the benchmark file
|
||||||
|
[1] for details. I benchmarked multiple zstd compression levels, although
|
||||||
|
the patch uses zstd level 1.
|
||||||
|
|
||||||
|
| Method | Ratio | Compression MB/s | Decompression speed |
|
||||||
|
|---------|-------|------------------|---------------------|
|
||||||
|
| None | 0.99 | 504 | 686 |
|
||||||
|
| lzo | 1.66 | 398 | 442 |
|
||||||
|
| zlib | 2.58 | 65 | 241 |
|
||||||
|
| zstd 1 | 2.57 | 260 | 383 |
|
||||||
|
| zstd 3 | 2.71 | 174 | 408 |
|
||||||
|
| zstd 6 | 2.87 | 70 | 398 |
|
||||||
|
| zstd 9 | 2.92 | 43 | 406 |
|
||||||
|
| zstd 12 | 2.93 | 21 | 408 |
|
||||||
|
| zstd 15 | 3.01 | 11 | 354 |
|
||||||
|
|
||||||
|
The next benchmark first copies `linux-4.11.6.tar` [4] to btrfs. Then it
|
||||||
|
measures the compression ratio, extracts the tar, and deletes the tar.
|
||||||
|
Then it measures the compression ratio again, and `tar`s the extracted
|
||||||
|
files into `/dev/null`. See the benchmark file [2] for details.
|
||||||
|
|
||||||
|
| Method | Tar Ratio | Extract Ratio | Copy (s) | Extract (s)| Read (s) |
|
||||||
|
|--------|-----------|---------------|----------|------------|----------|
|
||||||
|
| None | 0.97 | 0.78 | 0.981 | 5.501 | 8.807 |
|
||||||
|
| lzo | 2.06 | 1.38 | 1.631 | 8.458 | 8.585 |
|
||||||
|
| zlib | 3.40 | 1.86 | 7.750 | 21.544 | 11.744 |
|
||||||
|
| zstd 1 | 3.57 | 1.85 | 2.579 | 11.479 | 9.389 |
|
||||||
|
|
||||||
|
[1] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/btrfs-benchmark.sh
|
||||||
|
[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/btrfs-extract-benchmark.sh
|
||||||
|
[3] http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
|
||||||
|
[4] https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.11.6.tar.xz
|
||||||
|
|
||||||
|
zstd source repository: https://github.com/facebook/zstd
|
||||||
|
|
||||||
|
Signed-off-by: Nick Terrell <terrelln@fb.com>
|
||||||
|
---
|
||||||
|
fs/btrfs/Kconfig | 2 +
|
||||||
|
fs/btrfs/Makefile | 2 +-
|
||||||
|
fs/btrfs/compression.c | 1 +
|
||||||
|
fs/btrfs/compression.h | 6 +-
|
||||||
|
fs/btrfs/ctree.h | 1 +
|
||||||
|
fs/btrfs/disk-io.c | 2 +
|
||||||
|
fs/btrfs/ioctl.c | 6 +-
|
||||||
|
fs/btrfs/props.c | 6 +
|
||||||
|
fs/btrfs/super.c | 12 +-
|
||||||
|
fs/btrfs/sysfs.c | 2 +
|
||||||
|
fs/btrfs/zstd.c | 433 +++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
include/uapi/linux/btrfs.h | 8 +-
|
||||||
|
12 files changed, 469 insertions(+), 12 deletions(-)
|
||||||
|
create mode 100644 fs/btrfs/zstd.c
|
||||||
|
|
||||||
diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig
|
diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig
|
||||||
index 80e9c18..a26c63b 100644
|
index 80e9c18..a26c63b 100644
|
||||||
--- a/fs/btrfs/Kconfig
|
--- a/fs/btrfs/Kconfig
|
||||||
@ -25,7 +105,7 @@ index 128ce17..962a95a 100644
|
|||||||
reada.o backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \
|
reada.o backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \
|
||||||
uuid-tree.o props.o hash.o free-space-tree.o
|
uuid-tree.o props.o hash.o free-space-tree.o
|
||||||
diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
|
diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c
|
||||||
index c7721a6..66d4ced 100644
|
index 10e6b28..3beb0d0 100644
|
||||||
--- a/fs/btrfs/compression.c
|
--- a/fs/btrfs/compression.c
|
||||||
+++ b/fs/btrfs/compression.c
|
+++ b/fs/btrfs/compression.c
|
||||||
@@ -761,6 +761,7 @@ static struct {
|
@@ -761,6 +761,7 @@ static struct {
|
||||||
@ -34,7 +114,7 @@ index c7721a6..66d4ced 100644
|
|||||||
&btrfs_lzo_compress,
|
&btrfs_lzo_compress,
|
||||||
+ &btrfs_zstd_compress,
|
+ &btrfs_zstd_compress,
|
||||||
};
|
};
|
||||||
|
|
||||||
void __init btrfs_init_compress(void)
|
void __init btrfs_init_compress(void)
|
||||||
diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h
|
diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h
|
||||||
index 39ec43a..d99fc21 100644
|
index 39ec43a..d99fc21 100644
|
||||||
@ -50,20 +130,20 @@ index 39ec43a..d99fc21 100644
|
|||||||
+ BTRFS_COMPRESS_TYPES = 3,
|
+ BTRFS_COMPRESS_TYPES = 3,
|
||||||
+ BTRFS_COMPRESS_LAST = 4,
|
+ BTRFS_COMPRESS_LAST = 4,
|
||||||
};
|
};
|
||||||
|
|
||||||
struct btrfs_compress_op {
|
struct btrfs_compress_op {
|
||||||
@@ -92,5 +93,6 @@ struct btrfs_compress_op {
|
@@ -92,5 +93,6 @@ struct btrfs_compress_op {
|
||||||
|
|
||||||
extern const struct btrfs_compress_op btrfs_zlib_compress;
|
extern const struct btrfs_compress_op btrfs_zlib_compress;
|
||||||
extern const struct btrfs_compress_op btrfs_lzo_compress;
|
extern const struct btrfs_compress_op btrfs_lzo_compress;
|
||||||
+extern const struct btrfs_compress_op btrfs_zstd_compress;
|
+extern const struct btrfs_compress_op btrfs_zstd_compress;
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
|
diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
|
||||||
index 29b7fc2..878b23b9 100644
|
index 4f8f75d..61dd3dd 100644
|
||||||
--- a/fs/btrfs/ctree.h
|
--- a/fs/btrfs/ctree.h
|
||||||
+++ b/fs/btrfs/ctree.h
|
+++ b/fs/btrfs/ctree.h
|
||||||
@@ -270,6 +270,7 @@ struct btrfs_super_block {
|
@@ -271,6 +271,7 @@ struct btrfs_super_block {
|
||||||
BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS | \
|
BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS | \
|
||||||
BTRFS_FEATURE_INCOMPAT_BIG_METADATA | \
|
BTRFS_FEATURE_INCOMPAT_BIG_METADATA | \
|
||||||
BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO | \
|
BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO | \
|
||||||
@ -72,24 +152,24 @@ index 29b7fc2..878b23b9 100644
|
|||||||
BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF | \
|
BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF | \
|
||||||
BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA | \
|
BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA | \
|
||||||
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
|
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
|
||||||
index 08b74da..0c43e4e 100644
|
index 5f678dc..49c0e91 100644
|
||||||
--- a/fs/btrfs/disk-io.c
|
--- a/fs/btrfs/disk-io.c
|
||||||
+++ b/fs/btrfs/disk-io.c
|
+++ b/fs/btrfs/disk-io.c
|
||||||
@@ -2853,6 +2853,8 @@ int open_ctree(struct super_block *sb,
|
@@ -2831,6 +2831,8 @@ int open_ctree(struct super_block *sb,
|
||||||
features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF;
|
features |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF;
|
||||||
if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
|
if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
|
||||||
features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO;
|
features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO;
|
||||||
+ else if (tree_root->fs_info->compress_type == BTRFS_COMPRESS_ZSTD)
|
+ else if (fs_info->compress_type == BTRFS_COMPRESS_ZSTD)
|
||||||
+ features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD;
|
+ features |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD;
|
||||||
|
|
||||||
if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA)
|
if (features & BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA)
|
||||||
btrfs_info(fs_info, "has skinny extents");
|
btrfs_info(fs_info, "has skinny extents");
|
||||||
diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
|
diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
|
||||||
index dabfc7a..d8ea727 100644
|
index e176375..f732cfd 100644
|
||||||
--- a/fs/btrfs/ioctl.c
|
--- a/fs/btrfs/ioctl.c
|
||||||
+++ b/fs/btrfs/ioctl.c
|
+++ b/fs/btrfs/ioctl.c
|
||||||
@@ -327,8 +327,10 @@ static int btrfs_ioctl_setflags(struct file *file, void __user *arg)
|
@@ -327,8 +327,10 @@ static int btrfs_ioctl_setflags(struct file *file, void __user *arg)
|
||||||
|
|
||||||
if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
|
if (fs_info->compress_type == BTRFS_COMPRESS_LZO)
|
||||||
comp = "lzo";
|
comp = "lzo";
|
||||||
- else
|
- else
|
||||||
@ -101,13 +181,13 @@ index dabfc7a..d8ea727 100644
|
|||||||
comp, strlen(comp), 0);
|
comp, strlen(comp), 0);
|
||||||
if (ret)
|
if (ret)
|
||||||
@@ -1463,6 +1465,8 @@ int btrfs_defrag_file(struct inode *inode, struct file *file,
|
@@ -1463,6 +1465,8 @@ int btrfs_defrag_file(struct inode *inode, struct file *file,
|
||||||
|
|
||||||
if (range->compress_type == BTRFS_COMPRESS_LZO) {
|
if (range->compress_type == BTRFS_COMPRESS_LZO) {
|
||||||
btrfs_set_fs_incompat(fs_info, COMPRESS_LZO);
|
btrfs_set_fs_incompat(fs_info, COMPRESS_LZO);
|
||||||
+ } else if (range->compress_type == BTRFS_COMPRESS_ZSTD) {
|
+ } else if (range->compress_type == BTRFS_COMPRESS_ZSTD) {
|
||||||
+ btrfs_set_fs_incompat(fs_info, COMPRESS_ZSTD);
|
+ btrfs_set_fs_incompat(fs_info, COMPRESS_ZSTD);
|
||||||
}
|
}
|
||||||
|
|
||||||
ret = defrag_count;
|
ret = defrag_count;
|
||||||
diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c
|
diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c
|
||||||
index d6cb155..162105f 100644
|
index d6cb155..162105f 100644
|
||||||
@ -119,7 +199,7 @@ index d6cb155..162105f 100644
|
|||||||
return 0;
|
return 0;
|
||||||
+ else if (!strncmp("zstd", value, len))
|
+ else if (!strncmp("zstd", value, len))
|
||||||
+ return 0;
|
+ return 0;
|
||||||
|
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
}
|
}
|
||||||
@@ -405,6 +407,8 @@ static int prop_compression_apply(struct inode *inode,
|
@@ -405,6 +407,8 @@ static int prop_compression_apply(struct inode *inode,
|
||||||
@ -130,7 +210,7 @@ index d6cb155..162105f 100644
|
|||||||
+ type = BTRFS_COMPRESS_ZSTD;
|
+ type = BTRFS_COMPRESS_ZSTD;
|
||||||
else
|
else
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
|
|
||||||
@@ -422,6 +426,8 @@ static const char *prop_compression_extract(struct inode *inode)
|
@@ -422,6 +426,8 @@ static const char *prop_compression_extract(struct inode *inode)
|
||||||
return "zlib";
|
return "zlib";
|
||||||
case BTRFS_COMPRESS_LZO:
|
case BTRFS_COMPRESS_LZO:
|
||||||
@ -138,10 +218,10 @@ index d6cb155..162105f 100644
|
|||||||
+ case BTRFS_COMPRESS_ZSTD:
|
+ case BTRFS_COMPRESS_ZSTD:
|
||||||
+ return "zstd";
|
+ return "zstd";
|
||||||
}
|
}
|
||||||
|
|
||||||
return NULL;
|
return NULL;
|
||||||
diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
|
diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
|
||||||
index da687dc..b064456 100644
|
index 4f1cdd5..4f792d5 100644
|
||||||
--- a/fs/btrfs/super.c
|
--- a/fs/btrfs/super.c
|
||||||
+++ b/fs/btrfs/super.c
|
+++ b/fs/btrfs/super.c
|
||||||
@@ -513,6 +513,14 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char *options,
|
@@ -513,6 +513,14 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char *options,
|
||||||
@ -159,7 +239,7 @@ index da687dc..b064456 100644
|
|||||||
} else if (strncmp(args[0].from, "no", 2) == 0) {
|
} else if (strncmp(args[0].from, "no", 2) == 0) {
|
||||||
compress_type = "no";
|
compress_type = "no";
|
||||||
btrfs_clear_opt(info->mount_opt, COMPRESS);
|
btrfs_clear_opt(info->mount_opt, COMPRESS);
|
||||||
@@ -1230,8 +1238,10 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry)
|
@@ -1240,8 +1248,10 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry)
|
||||||
if (btrfs_test_opt(info, COMPRESS)) {
|
if (btrfs_test_opt(info, COMPRESS)) {
|
||||||
if (info->compress_type == BTRFS_COMPRESS_ZLIB)
|
if (info->compress_type == BTRFS_COMPRESS_ZLIB)
|
||||||
compress_type = "zlib";
|
compress_type = "zlib";
|
||||||
@ -631,7 +711,7 @@ index 0000000..838741b
|
|||||||
+ .decompress = zstd_decompress,
|
+ .decompress = zstd_decompress,
|
||||||
+};
|
+};
|
||||||
diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h
|
diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h
|
||||||
index db4c253..f26c34f 100644
|
index a456e53..992c150 100644
|
||||||
--- a/include/uapi/linux/btrfs.h
|
--- a/include/uapi/linux/btrfs.h
|
||||||
+++ b/include/uapi/linux/btrfs.h
|
+++ b/include/uapi/linux/btrfs.h
|
||||||
@@ -255,13 +255,7 @@ struct btrfs_ioctl_fs_info_args {
|
@@ -255,13 +255,7 @@ struct btrfs_ioctl_fs_info_args {
|
||||||
@ -646,6 +726,8 @@ index db4c253..f26c34f 100644
|
|||||||
- */
|
- */
|
||||||
-#define BTRFS_FEATURE_INCOMPAT_COMPRESS_LZOv2 (1ULL << 4)
|
-#define BTRFS_FEATURE_INCOMPAT_COMPRESS_LZOv2 (1ULL << 4)
|
||||||
+#define BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD (1ULL << 4)
|
+#define BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD (1ULL << 4)
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* older kernels tried to do bigger metadata blocks, but the
|
* older kernels tried to do bigger metadata blocks, but the
|
||||||
|
--
|
||||||
|
2.9.3
|
||||||
|
@ -1,11 +1,67 @@
|
|||||||
|
From 5ff6a64abaea7b7f11d37cb0fdf08642316a3a90 Mon Sep 17 00:00:00 2001
|
||||||
|
From: Nick Terrell <terrelln@fb.com>
|
||||||
|
Date: Mon, 12 Jun 2017 12:18:23 -0700
|
||||||
|
Subject: [PATCH 4/4] squashfs: Add zstd support
|
||||||
|
|
||||||
|
Add zstd compression and decompression support to SquashFS. zstd is a
|
||||||
|
great fit for SquashFS because it can compress at ratios approaching xz,
|
||||||
|
while decompressing twice as fast as zlib. For SquashFS in particular,
|
||||||
|
it can decompress as fast as lzo and lz4. It also has the flexibility
|
||||||
|
to turn down the compression ratio for faster compression times.
|
||||||
|
|
||||||
|
The compression benchmark is run on the file tree from the SquashFS archive
|
||||||
|
found in ubuntu-16.10-desktop-amd64.iso [1]. It uses `mksquashfs` with the
|
||||||
|
default block size (128 KB) and and various compression algorithms/levels.
|
||||||
|
xz and zstd are also benchmarked with 256 KB blocks. The decompression
|
||||||
|
benchmark times how long it takes to `tar` the file tree into `/dev/null`.
|
||||||
|
See the benchmark file in the upstream zstd source repository located under
|
||||||
|
`contrib/linux-kernel/squashfs-benchmark.sh` [2] for details.
|
||||||
|
|
||||||
|
I ran the benchmarks on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
|
||||||
|
The VM is running on a MacBook Pro with a 3.1 GHz Intel Core i7 processor,
|
||||||
|
16 GB of RAM, and a SSD.
|
||||||
|
|
||||||
|
| Method | Ratio | Compression MB/s | Decompression MB/s |
|
||||||
|
|----------------|-------|------------------|--------------------|
|
||||||
|
| gzip | 2.92 | 15 | 128 |
|
||||||
|
| lzo | 2.64 | 9.5 | 217 |
|
||||||
|
| lz4 | 2.12 | 94 | 218 |
|
||||||
|
| xz | 3.43 | 5.5 | 35 |
|
||||||
|
| xz 256 KB | 3.53 | 5.4 | 40 |
|
||||||
|
| zstd 1 | 2.71 | 96 | 210 |
|
||||||
|
| zstd 5 | 2.93 | 69 | 198 |
|
||||||
|
| zstd 10 | 3.01 | 41 | 225 |
|
||||||
|
| zstd 15 | 3.13 | 11.4 | 224 |
|
||||||
|
| zstd 16 256 KB | 3.24 | 8.1 | 210 |
|
||||||
|
|
||||||
|
This patch was written by Sean Purcell <me@seanp.xyz>, but I will be
|
||||||
|
taking over the submission process.
|
||||||
|
|
||||||
|
[1] http://releases.ubuntu.com/16.10/
|
||||||
|
[2] https://github.com/facebook/zstd/blob/dev/contrib/linux-kernel/squashfs-benchmark.sh
|
||||||
|
|
||||||
|
zstd source repository: https://github.com/facebook/zstd
|
||||||
|
|
||||||
|
Cc: Sean Purcell <me@seanp.xyz>
|
||||||
|
Signed-off-by: Nick Terrell <terrelln@fb.com>
|
||||||
|
---
|
||||||
|
fs/squashfs/Kconfig | 14 +++++
|
||||||
|
fs/squashfs/Makefile | 1 +
|
||||||
|
fs/squashfs/decompressor.c | 7 +++
|
||||||
|
fs/squashfs/decompressor.h | 4 ++
|
||||||
|
fs/squashfs/squashfs_fs.h | 1 +
|
||||||
|
fs/squashfs/zstd_wrapper.c | 150 +++++++++++++++++++++++++++++++++++++++++++++
|
||||||
|
6 files changed, 177 insertions(+)
|
||||||
|
create mode 100644 fs/squashfs/zstd_wrapper.c
|
||||||
|
|
||||||
diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig
|
diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig
|
||||||
index ffb093e..1adb334 100644
|
index ffb093e..1adb334 100644
|
||||||
--- a/fs/squashfs/Kconfig
|
--- a/fs/squashfs/Kconfig
|
||||||
+++ b/fs/squashfs/Kconfig
|
+++ b/fs/squashfs/Kconfig
|
||||||
@@ -165,6 +165,20 @@ config SQUASHFS_XZ
|
@@ -165,6 +165,20 @@ config SQUASHFS_XZ
|
||||||
|
|
||||||
If unsure, say N.
|
If unsure, say N.
|
||||||
|
|
||||||
+config SQUASHFS_ZSTD
|
+config SQUASHFS_ZSTD
|
||||||
+ bool "Include support for ZSTD compressed file systems"
|
+ bool "Include support for ZSTD compressed file systems"
|
||||||
+ depends on SQUASHFS
|
+ depends on SQUASHFS
|
||||||
@ -39,7 +95,7 @@ index d2bc136..8366398 100644
|
|||||||
@@ -65,6 +65,12 @@ static const struct squashfs_decompressor squashfs_zlib_comp_ops = {
|
@@ -65,6 +65,12 @@ static const struct squashfs_decompressor squashfs_zlib_comp_ops = {
|
||||||
};
|
};
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
+#ifndef CONFIG_SQUASHFS_ZSTD
|
+#ifndef CONFIG_SQUASHFS_ZSTD
|
||||||
+static const struct squashfs_decompressor squashfs_zstd_comp_ops = {
|
+static const struct squashfs_decompressor squashfs_zstd_comp_ops = {
|
||||||
+ NULL, NULL, NULL, NULL, ZSTD_COMPRESSION, "zstd", 0
|
+ NULL, NULL, NULL, NULL, ZSTD_COMPRESSION, "zstd", 0
|
||||||
@ -56,7 +112,7 @@ index d2bc136..8366398 100644
|
|||||||
+ &squashfs_zstd_comp_ops,
|
+ &squashfs_zstd_comp_ops,
|
||||||
&squashfs_unknown_comp_ops
|
&squashfs_unknown_comp_ops
|
||||||
};
|
};
|
||||||
|
|
||||||
diff --git a/fs/squashfs/decompressor.h b/fs/squashfs/decompressor.h
|
diff --git a/fs/squashfs/decompressor.h b/fs/squashfs/decompressor.h
|
||||||
index a25713c..0f5a8e4 100644
|
index a25713c..0f5a8e4 100644
|
||||||
--- a/fs/squashfs/decompressor.h
|
--- a/fs/squashfs/decompressor.h
|
||||||
@ -64,7 +120,7 @@ index a25713c..0f5a8e4 100644
|
|||||||
@@ -58,4 +58,8 @@ extern const struct squashfs_decompressor squashfs_lzo_comp_ops;
|
@@ -58,4 +58,8 @@ extern const struct squashfs_decompressor squashfs_lzo_comp_ops;
|
||||||
extern const struct squashfs_decompressor squashfs_zlib_comp_ops;
|
extern const struct squashfs_decompressor squashfs_zlib_comp_ops;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
+#ifdef CONFIG_SQUASHFS_ZSTD
|
+#ifdef CONFIG_SQUASHFS_ZSTD
|
||||||
+extern const struct squashfs_decompressor squashfs_zstd_comp_ops;
|
+extern const struct squashfs_decompressor squashfs_zstd_comp_ops;
|
||||||
+#endif
|
+#endif
|
||||||
@ -79,7 +135,7 @@ index 506f4ba..24d12fd 100644
|
|||||||
#define XZ_COMPRESSION 4
|
#define XZ_COMPRESSION 4
|
||||||
#define LZ4_COMPRESSION 5
|
#define LZ4_COMPRESSION 5
|
||||||
+#define ZSTD_COMPRESSION 6
|
+#define ZSTD_COMPRESSION 6
|
||||||
|
|
||||||
struct squashfs_super_block {
|
struct squashfs_super_block {
|
||||||
__le32 s_magic;
|
__le32 s_magic;
|
||||||
diff --git a/fs/squashfs/zstd_wrapper.c b/fs/squashfs/zstd_wrapper.c
|
diff --git a/fs/squashfs/zstd_wrapper.c b/fs/squashfs/zstd_wrapper.c
|
||||||
@ -238,3 +294,5 @@ index 0000000..8cb7c76
|
|||||||
+ .name = "zstd",
|
+ .name = "zstd",
|
||||||
+ .supported = 1
|
+ .supported = 1
|
||||||
+};
|
+};
|
||||||
|
--
|
||||||
|
2.9.3
|
||||||
|
99
contrib/linux-kernel/btrfs-extract-benchmark.sh
Executable file
99
contrib/linux-kernel/btrfs-extract-benchmark.sh
Executable file
@ -0,0 +1,99 @@
|
|||||||
|
# !/bin/sh
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# Benchmarks run on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
|
||||||
|
# The VM is running on a Macbook Pro with a 3.1 GHz Intel Core i7 processor and
|
||||||
|
# 16 GB of RAM and an SSD.
|
||||||
|
|
||||||
|
# silesia is a directory that can be downloaded from
|
||||||
|
# http://mattmahoney.net/dc/silesia.html
|
||||||
|
# ls -l silesia/
|
||||||
|
# total 203M
|
||||||
|
# -rwxr-xr-x 1 terrelln 9.8M Apr 12 2002 dickens
|
||||||
|
# -rwxr-xr-x 1 terrelln 49M May 31 2002 mozilla
|
||||||
|
# -rwxr-xr-x 1 terrelln 9.6M Mar 20 2003 mr
|
||||||
|
# -rwxr-xr-x 1 terrelln 32M Apr 2 2002 nci
|
||||||
|
# -rwxr-xr-x 1 terrelln 5.9M Jul 4 2002 ooffice
|
||||||
|
# -rwxr-xr-x 1 terrelln 9.7M Apr 11 2002 osdb
|
||||||
|
# -rwxr-xr-x 1 terrelln 6.4M Apr 2 2002 reymont
|
||||||
|
# -rwxr-xr-x 1 terrelln 21M Mar 25 2002 samba
|
||||||
|
# -rwxr-xr-x 1 terrelln 7.0M Mar 24 2002 sao
|
||||||
|
# -rwxr-xr-x 1 terrelln 40M Mar 25 2002 webster
|
||||||
|
# -rwxr-xr-x 1 terrelln 8.1M Apr 4 2002 x-ray
|
||||||
|
# -rwxr-xr-x 1 terrelln 5.1M Nov 30 2000 xml
|
||||||
|
|
||||||
|
# $HOME is on a ext4 filesystem
|
||||||
|
BENCHMARK_FILE="linux-4.11.6.tar"
|
||||||
|
BENCHMARK_DIR="$HOME/$BENCHMARK_FILE"
|
||||||
|
|
||||||
|
# Normalize the environment
|
||||||
|
sudo umount /mnt/btrfs 2> /dev/null > /dev/null || true
|
||||||
|
sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs
|
||||||
|
sudo rm -rf /mnt/btrfs/*
|
||||||
|
sync
|
||||||
|
sudo umount /mnt/btrfs
|
||||||
|
sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs
|
||||||
|
|
||||||
|
# Run the benchmark
|
||||||
|
echo "Copy"
|
||||||
|
time sh -c "sudo cp -r $BENCHMARK_DIR /mnt/btrfs/$BENCHMARK_FILE && sync"
|
||||||
|
|
||||||
|
echo "Approximate tarred compression ratio"
|
||||||
|
printf "%d / %d\n" \
|
||||||
|
$(df /mnt/btrfs --output=used -B 1 | tail -n 1) \
|
||||||
|
$(sudo du /mnt/btrfs -b -d 0 | tr '\t' '\n' | head -n 1);
|
||||||
|
|
||||||
|
# Unmount and remount to avoid any caching
|
||||||
|
sudo umount /mnt/btrfs
|
||||||
|
sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs
|
||||||
|
|
||||||
|
echo "Extract"
|
||||||
|
time sh -c "sudo tar -C /mnt/btrfs -xf /mnt/btrfs/$BENCHMARK_FILE && sync"
|
||||||
|
|
||||||
|
# Remove the tarball, leaving only the extracted data
|
||||||
|
sudo rm /mnt/btrfs/$BENCHMARK_FILE
|
||||||
|
# Unmount and remount to avoid any caching
|
||||||
|
sudo umount /mnt/btrfs
|
||||||
|
sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs
|
||||||
|
|
||||||
|
echo "Approximate extracted compression ratio"
|
||||||
|
printf "%d / %d\n" \
|
||||||
|
$(df /mnt/btrfs --output=used -B 1 | tail -n 1) \
|
||||||
|
$(sudo du /mnt/btrfs -b -d 0 | tr '\t' '\n' | head -n 1);
|
||||||
|
|
||||||
|
echo "Read"
|
||||||
|
time sudo tar -c /mnt/btrfs 2> /dev/null | wc -c > /dev/null
|
||||||
|
|
||||||
|
sudo rm -rf /mnt/btrfs/*
|
||||||
|
sudo umount /mnt/btrfs
|
||||||
|
|
||||||
|
# Run for each of -o compress-force={none, lzo, zlib, zstd} 5 times and take the
|
||||||
|
# min time and ratio.
|
||||||
|
|
||||||
|
# none
|
||||||
|
# copy: 0.981 s
|
||||||
|
# extract: 5.501 s
|
||||||
|
# read: 8.807 s
|
||||||
|
# tarball ratio: 0.97
|
||||||
|
# extracted ratio: 0.78
|
||||||
|
|
||||||
|
# lzo
|
||||||
|
# copy: 1.631 s
|
||||||
|
# extract: 8.458 s
|
||||||
|
# read: 8.585 s
|
||||||
|
# tarball ratio: 2.06
|
||||||
|
# extracted ratio: 1.38
|
||||||
|
|
||||||
|
# zlib
|
||||||
|
# copy: 7.750 s
|
||||||
|
# extract: 21.544 s
|
||||||
|
# read: 11.744 s
|
||||||
|
# tarball ratio : 3.40
|
||||||
|
# extracted ratio: 1.86
|
||||||
|
|
||||||
|
# zstd 1
|
||||||
|
# copy: 2.579 s
|
||||||
|
# extract: 11.479 s
|
||||||
|
# read: 9.389 s
|
||||||
|
# tarball ratio : 3.57
|
||||||
|
# extracted ratio: 1.85
|
Reference in New Issue
Block a user