From 32c658a2898e37f3e78ba64efff5fdc061389a36 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Thu, 27 Apr 2017 15:04:31 -0700 Subject: [PATCH 1/3] SquashFS linux kernel patch --- .../linux-kernel/fs/squashfs/zstd_wrapper.c | 146 +++++++++++ contrib/linux-kernel/squashfs.diff | 242 ++++++++++++++++++ 2 files changed, 388 insertions(+) create mode 100644 contrib/linux-kernel/fs/squashfs/zstd_wrapper.c create mode 100644 contrib/linux-kernel/squashfs.diff diff --git a/contrib/linux-kernel/fs/squashfs/zstd_wrapper.c b/contrib/linux-kernel/fs/squashfs/zstd_wrapper.c new file mode 100644 index 000000000..81f4b5a4b --- /dev/null +++ b/contrib/linux-kernel/fs/squashfs/zstd_wrapper.c @@ -0,0 +1,146 @@ +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2017 Facebook + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2, + * or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * zstd_wrapper.c + */ + +#include +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs.h" +#include "decompressor.h" +#include "page_actor.h" + +struct workspace { + void *mem; + size_t mem_size; +}; + +static void *zstd_init(struct squashfs_sb_info *msblk, void *buff) +{ + struct workspace *wksp = kmalloc(sizeof(*wksp), GFP_KERNEL); + if (wksp == NULL) + goto failed; + wksp->mem_size = ZSTD_DStreamWorkspaceBound(max_t(size_t, + msblk->block_size, SQUASHFS_METADATA_SIZE)); + wksp->mem = vmalloc(wksp->mem_size); + if (wksp->mem == NULL) + goto failed; + + return wksp; + +failed: + ERROR("Failed to allocate zstd workspace\n"); + kfree(wksp); + return ERR_PTR(-ENOMEM); +} + + +static void zstd_free(void *strm) +{ + struct workspace *wksp = strm; + + if (wksp) + vfree(wksp->mem); + kfree(wksp); +} + + +static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm, + struct buffer_head **bh, int b, int offset, int length, + struct squashfs_page_actor *output) +{ + struct workspace *wksp = strm; + ZSTD_DStream *stream; + size_t total_out = 0; + size_t zstd_err; + int k = 0; + ZSTD_inBuffer in_buf = { NULL, 0, 0 }; + ZSTD_outBuffer out_buf = { NULL, 0, 0 }; + + stream = ZSTD_initDStream(wksp->mem_size, wksp->mem, wksp->mem_size); + + if (!stream) { + ERROR("Failed to initialize zstd decompressor\n"); + goto out; + } + + out_buf.size = PAGE_SIZE; + out_buf.dst = squashfs_first_page(output); + + do { + if (in_buf.pos == in_buf.size && k < b) { + int avail = min(length, msblk->devblksize - offset); + length -= avail; + in_buf.src = bh[k]->b_data + offset; + in_buf.size = avail; + in_buf.pos = 0; + offset = 0; + } + + if (out_buf.pos == out_buf.size) { + out_buf.dst = squashfs_next_page(output); + out_buf.pos = 0; + if (out_buf.dst != NULL) { + out_buf.size = PAGE_SIZE; + } else { + out_buf.size = 0; + } + } + + total_out -= out_buf.pos; + zstd_err = ZSTD_decompressStream(stream, &out_buf, &in_buf); + total_out += out_buf.pos; /* add the additional data produced */ + + if (in_buf.pos == in_buf.size && k < b) + put_bh(bh[k++]); + } while (zstd_err != 0 && !ZSTD_isError(zstd_err)); + + if (ZSTD_isError(zstd_err)) { + ERROR("zstd decompression error: %d\n", + (int)ZSTD_getErrorCode(zstd_err)); + } + + squashfs_finish_page(output); + + if (k < b) + goto out; + + return total_out; + +out: + for (; k < b; k++) + put_bh(bh[k]); + + return -EIO; +} + +const struct squashfs_decompressor squashfs_zstd_comp_ops = { + .init = zstd_init, + .free = zstd_free, + .decompress = zstd_uncompress, + .id = ZSTD_COMPRESSION, + .name = "zstd", + .supported = 1 +}; diff --git a/contrib/linux-kernel/squashfs.diff b/contrib/linux-kernel/squashfs.diff new file mode 100644 index 000000000..e1e1235f0 --- /dev/null +++ b/contrib/linux-kernel/squashfs.diff @@ -0,0 +1,242 @@ +commit 7289653483a0579c3b63a06abf008210c8cc6c8b +Author: Sean Purcell +Date: Thu Apr 27 14:56:25 2017 -0700 + + Add zstd support to kernel squashfs + +diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig +index ffb093e..1adb334 100644 +--- a/fs/squashfs/Kconfig ++++ b/fs/squashfs/Kconfig +@@ -165,6 +165,20 @@ config SQUASHFS_XZ + + If unsure, say N. + ++config SQUASHFS_ZSTD ++ bool "Include support for ZSTD compressed file systems" ++ depends on SQUASHFS ++ select ZSTD_DECOMPRESS ++ help ++ Saying Y here includes support for reading Squashfs file systems ++ compressed with ZSTD compression. ZSTD gives better compression than ++ the default ZLIB compression, while using less CPU. ++ ++ ZSTD is not the standard compression used in Squashfs and so most ++ file systems will be readable without selecting this option. ++ ++ If unsure, say N. ++ + config SQUASHFS_4K_DEVBLK_SIZE + bool "Use 4K device block size?" + depends on SQUASHFS +diff --git a/fs/squashfs/Makefile b/fs/squashfs/Makefile +index 246a6f3..6655631 100644 +--- a/fs/squashfs/Makefile ++++ b/fs/squashfs/Makefile +@@ -15,3 +15,4 @@ squashfs-$(CONFIG_SQUASHFS_LZ4) += lz4_wrapper.o + squashfs-$(CONFIG_SQUASHFS_LZO) += lzo_wrapper.o + squashfs-$(CONFIG_SQUASHFS_XZ) += xz_wrapper.o + squashfs-$(CONFIG_SQUASHFS_ZLIB) += zlib_wrapper.o ++squashfs-$(CONFIG_SQUASHFS_ZSTD) += zstd_wrapper.o +diff --git a/fs/squashfs/decompressor.c b/fs/squashfs/decompressor.c +index d2bc136..8366398 100644 +--- a/fs/squashfs/decompressor.c ++++ b/fs/squashfs/decompressor.c +@@ -65,6 +65,12 @@ static const struct squashfs_decompressor squashfs_zlib_comp_ops = { + }; + #endif + ++#ifndef CONFIG_SQUASHFS_ZSTD ++static const struct squashfs_decompressor squashfs_zstd_comp_ops = { ++ NULL, NULL, NULL, NULL, ZSTD_COMPRESSION, "zstd", 0 ++}; ++#endif ++ + static const struct squashfs_decompressor squashfs_unknown_comp_ops = { + NULL, NULL, NULL, NULL, 0, "unknown", 0 + }; +@@ -75,6 +81,7 @@ static const struct squashfs_decompressor *decompressor[] = { + &squashfs_lzo_comp_ops, + &squashfs_xz_comp_ops, + &squashfs_lzma_unsupported_comp_ops, ++ &squashfs_zstd_comp_ops, + &squashfs_unknown_comp_ops + }; + +diff --git a/fs/squashfs/decompressor.h b/fs/squashfs/decompressor.h +index a25713c..0f5a8e4 100644 +--- a/fs/squashfs/decompressor.h ++++ b/fs/squashfs/decompressor.h +@@ -58,4 +58,8 @@ extern const struct squashfs_decompressor squashfs_lzo_comp_ops; + extern const struct squashfs_decompressor squashfs_zlib_comp_ops; + #endif + ++#ifdef CONFIG_SQUASHFS_ZSTD ++extern const struct squashfs_decompressor squashfs_zstd_comp_ops; ++#endif ++ + #endif +diff --git a/fs/squashfs/squashfs_fs.h b/fs/squashfs/squashfs_fs.h +index 506f4ba..24d12fd 100644 +--- a/fs/squashfs/squashfs_fs.h ++++ b/fs/squashfs/squashfs_fs.h +@@ -241,6 +241,7 @@ struct meta_index { + #define LZO_COMPRESSION 3 + #define XZ_COMPRESSION 4 + #define LZ4_COMPRESSION 5 ++#define ZSTD_COMPRESSION 6 + + struct squashfs_super_block { + __le32 s_magic; +diff --git a/fs/squashfs/zstd_wrapper.c b/fs/squashfs/zstd_wrapper.c +new file mode 100644 +index 0000000..81f4b5a +--- /dev/null ++++ b/fs/squashfs/zstd_wrapper.c +@@ -0,0 +1,146 @@ ++/* ++ * Squashfs - a compressed read only filesystem for Linux ++ * ++ * Copyright (c) 2017 Facebook ++ * ++ * This program is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License ++ * as published by the Free Software Foundation; either version 2, ++ * or (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, write to the Free Software ++ * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. ++ * ++ * zstd_wrapper.c ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "squashfs_fs.h" ++#include "squashfs_fs_sb.h" ++#include "squashfs.h" ++#include "decompressor.h" ++#include "page_actor.h" ++ ++struct workspace { ++ void *mem; ++ size_t mem_size; ++}; ++ ++static void *zstd_init(struct squashfs_sb_info *msblk, void *buff) ++{ ++ struct workspace *wksp = kmalloc(sizeof(*wksp), GFP_KERNEL); ++ if (wksp == NULL) ++ goto failed; ++ wksp->mem_size = ZSTD_DStreamWorkspaceBound(max_t(size_t, ++ msblk->block_size, SQUASHFS_METADATA_SIZE)); ++ wksp->mem = vmalloc(wksp->mem_size); ++ if (wksp->mem == NULL) ++ goto failed; ++ ++ return wksp; ++ ++failed: ++ ERROR("Failed to allocate zstd workspace\n"); ++ kfree(wksp); ++ return ERR_PTR(-ENOMEM); ++} ++ ++ ++static void zstd_free(void *strm) ++{ ++ struct workspace *wksp = strm; ++ ++ if (wksp) ++ vfree(wksp->mem); ++ kfree(wksp); ++} ++ ++ ++static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm, ++ struct buffer_head **bh, int b, int offset, int length, ++ struct squashfs_page_actor *output) ++{ ++ struct workspace *wksp = strm; ++ ZSTD_DStream *stream; ++ size_t total_out = 0; ++ size_t zstd_err; ++ int k = 0; ++ ZSTD_inBuffer in_buf = { NULL, 0, 0 }; ++ ZSTD_outBuffer out_buf = { NULL, 0, 0 }; ++ ++ stream = ZSTD_initDStream(wksp->mem_size, wksp->mem, wksp->mem_size); ++ ++ if (!stream) { ++ ERROR("Failed to initialize zstd decompressor\n"); ++ goto out; ++ } ++ ++ out_buf.size = PAGE_SIZE; ++ out_buf.dst = squashfs_first_page(output); ++ ++ do { ++ if (in_buf.pos == in_buf.size && k < b) { ++ int avail = min(length, msblk->devblksize - offset); ++ length -= avail; ++ in_buf.src = bh[k]->b_data + offset; ++ in_buf.size = avail; ++ in_buf.pos = 0; ++ offset = 0; ++ } ++ ++ if (out_buf.pos == out_buf.size) { ++ out_buf.dst = squashfs_next_page(output); ++ out_buf.pos = 0; ++ if (out_buf.dst != NULL) { ++ out_buf.size = PAGE_SIZE; ++ } else { ++ out_buf.size = 0; ++ } ++ } ++ ++ total_out -= out_buf.pos; ++ zstd_err = ZSTD_decompressStream(stream, &out_buf, &in_buf); ++ total_out += out_buf.pos; /* add the additional data produced */ ++ ++ if (in_buf.pos == in_buf.size && k < b) ++ put_bh(bh[k++]); ++ } while (zstd_err != 0 && !ZSTD_isError(zstd_err)); ++ ++ if (ZSTD_isError(zstd_err)) { ++ ERROR("zstd decompression error: %d\n", ++ (int)ZSTD_getErrorCode(zstd_err)); ++ } ++ ++ squashfs_finish_page(output); ++ ++ if (k < b) ++ goto out; ++ ++ return total_out; ++ ++out: ++ for (; k < b; k++) ++ put_bh(bh[k]); ++ ++ return -EIO; ++} ++ ++const struct squashfs_decompressor squashfs_zstd_comp_ops = { ++ .init = zstd_init, ++ .free = zstd_free, ++ .decompress = zstd_uncompress, ++ .id = ZSTD_COMPRESSION, ++ .name = "zstd", ++ .supported = 1 ++}; From eeeb201e700dfe592c352cf92077095e2a539b89 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Thu, 27 Apr 2017 16:54:05 -0700 Subject: [PATCH 2/3] Fix case where pages run out before end of stream --- .../linux-kernel/fs/squashfs/zstd_wrapper.c | 13 ++++++------ contrib/linux-kernel/squashfs.diff | 21 ++++++++++--------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/contrib/linux-kernel/fs/squashfs/zstd_wrapper.c b/contrib/linux-kernel/fs/squashfs/zstd_wrapper.c index 81f4b5a4b..af72c7b22 100644 --- a/contrib/linux-kernel/fs/squashfs/zstd_wrapper.c +++ b/contrib/linux-kernel/fs/squashfs/zstd_wrapper.c @@ -101,12 +101,13 @@ static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm, if (out_buf.pos == out_buf.size) { out_buf.dst = squashfs_next_page(output); - out_buf.pos = 0; - if (out_buf.dst != NULL) { - out_buf.size = PAGE_SIZE; - } else { - out_buf.size = 0; + if (out_buf.dst == NULL) { + /* shouldn't run out of pages before stream is + * done */ + goto out; } + out_buf.pos = 0; + out_buf.size = PAGE_SIZE; } total_out -= out_buf.pos; @@ -127,7 +128,7 @@ static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm, if (k < b) goto out; - return total_out; + return (int)total_out; out: for (; k < b; k++) diff --git a/contrib/linux-kernel/squashfs.diff b/contrib/linux-kernel/squashfs.diff index e1e1235f0..e3b9e8552 100644 --- a/contrib/linux-kernel/squashfs.diff +++ b/contrib/linux-kernel/squashfs.diff @@ -1,6 +1,6 @@ -commit 7289653483a0579c3b63a06abf008210c8cc6c8b +commit 47ba72b36506f91c3774c0bb1fa3c7a5dcfe3ea1 Author: Sean Purcell -Date: Thu Apr 27 14:56:25 2017 -0700 +Date: Thu Apr 27 16:50:53 2017 -0700 Add zstd support to kernel squashfs @@ -90,10 +90,10 @@ index 506f4ba..24d12fd 100644 __le32 s_magic; diff --git a/fs/squashfs/zstd_wrapper.c b/fs/squashfs/zstd_wrapper.c new file mode 100644 -index 0000000..81f4b5a +index 0000000..af72c7b --- /dev/null +++ b/fs/squashfs/zstd_wrapper.c -@@ -0,0 +1,146 @@ +@@ -0,0 +1,147 @@ +/* + * Squashfs - a compressed read only filesystem for Linux + * @@ -197,12 +197,13 @@ index 0000000..81f4b5a + + if (out_buf.pos == out_buf.size) { + out_buf.dst = squashfs_next_page(output); -+ out_buf.pos = 0; -+ if (out_buf.dst != NULL) { -+ out_buf.size = PAGE_SIZE; -+ } else { -+ out_buf.size = 0; ++ if (out_buf.dst == NULL) { ++ /* shouldn't run out of pages before stream is ++ * done */ ++ goto out; + } ++ out_buf.pos = 0; ++ out_buf.size = PAGE_SIZE; + } + + total_out -= out_buf.pos; @@ -223,7 +224,7 @@ index 0000000..81f4b5a + if (k < b) + goto out; + -+ return total_out; ++ return (int)total_out; + +out: + for (; k < b; k++) From c7e107197a80577057adfa3c2bd1cbc152f72285 Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Thu, 27 Apr 2017 17:19:20 -0700 Subject: [PATCH 3/3] Fix missing squashfs_finish_page's --- contrib/linux-kernel/fs/squashfs/zstd_wrapper.c | 6 ++++-- contrib/linux-kernel/squashfs.diff | 16 +++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/contrib/linux-kernel/fs/squashfs/zstd_wrapper.c b/contrib/linux-kernel/fs/squashfs/zstd_wrapper.c index af72c7b22..7cc93030b 100644 --- a/contrib/linux-kernel/fs/squashfs/zstd_wrapper.c +++ b/contrib/linux-kernel/fs/squashfs/zstd_wrapper.c @@ -104,6 +104,7 @@ static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm, if (out_buf.dst == NULL) { /* shouldn't run out of pages before stream is * done */ + squashfs_finish_page(output); goto out; } out_buf.pos = 0; @@ -118,13 +119,14 @@ static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm, put_bh(bh[k++]); } while (zstd_err != 0 && !ZSTD_isError(zstd_err)); + squashfs_finish_page(output); + if (ZSTD_isError(zstd_err)) { ERROR("zstd decompression error: %d\n", (int)ZSTD_getErrorCode(zstd_err)); + goto out; } - squashfs_finish_page(output); - if (k < b) goto out; diff --git a/contrib/linux-kernel/squashfs.diff b/contrib/linux-kernel/squashfs.diff index e3b9e8552..ddf7b3578 100644 --- a/contrib/linux-kernel/squashfs.diff +++ b/contrib/linux-kernel/squashfs.diff @@ -1,8 +1,8 @@ -commit 47ba72b36506f91c3774c0bb1fa3c7a5dcfe3ea1 +commit 16bb6b9fd684eadba41a36223d67805d7ea741e7 Author: Sean Purcell -Date: Thu Apr 27 16:50:53 2017 -0700 +Date: Thu Apr 27 17:17:58 2017 -0700 - Add zstd support to kernel squashfs + Add zstd support to squashfs diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig index ffb093e..1adb334 100644 @@ -90,10 +90,10 @@ index 506f4ba..24d12fd 100644 __le32 s_magic; diff --git a/fs/squashfs/zstd_wrapper.c b/fs/squashfs/zstd_wrapper.c new file mode 100644 -index 0000000..af72c7b +index 0000000..7cc9303 --- /dev/null +++ b/fs/squashfs/zstd_wrapper.c -@@ -0,0 +1,147 @@ +@@ -0,0 +1,149 @@ +/* + * Squashfs - a compressed read only filesystem for Linux + * @@ -200,6 +200,7 @@ index 0000000..af72c7b + if (out_buf.dst == NULL) { + /* shouldn't run out of pages before stream is + * done */ ++ squashfs_finish_page(output); + goto out; + } + out_buf.pos = 0; @@ -214,13 +215,14 @@ index 0000000..af72c7b + put_bh(bh[k++]); + } while (zstd_err != 0 && !ZSTD_isError(zstd_err)); + ++ squashfs_finish_page(output); ++ + if (ZSTD_isError(zstd_err)) { + ERROR("zstd decompression error: %d\n", + (int)ZSTD_getErrorCode(zstd_err)); ++ goto out; + } + -+ squashfs_finish_page(output); -+ + if (k < b) + goto out; +