@@ -4,6 +4,37 @@ use rayon::prelude::*;
44use std:: { convert:: TryFrom , fmt, io:: Read , io:: Write , path:: Path , str:: FromStr } ;
55use xz2:: { read:: XzDecoder , write:: XzEncoder } ;
66
7+ #[ derive( Default , Debug , Copy , Clone ) ]
8+ pub enum CompressionProfile {
9+ Fast ,
10+ #[ default]
11+ Balanced ,
12+ Best ,
13+ }
14+
15+ impl FromStr for CompressionProfile {
16+ type Err = Error ;
17+
18+ fn from_str ( input : & str ) -> Result < Self , Error > {
19+ Ok ( match input {
20+ "fast" => Self :: Fast ,
21+ "balanced" => Self :: Balanced ,
22+ "best" => Self :: Best ,
23+ other => anyhow:: bail!( "invalid compression profile: {other}" ) ,
24+ } )
25+ }
26+ }
27+
28+ impl fmt:: Display for CompressionProfile {
29+ fn fmt ( & self , f : & mut fmt:: Formatter < ' _ > ) -> fmt:: Result {
30+ match self {
31+ CompressionProfile :: Fast => f. write_str ( "fast" ) ,
32+ CompressionProfile :: Balanced => f. write_str ( "balanced" ) ,
33+ CompressionProfile :: Best => f. write_str ( "best" ) ,
34+ }
35+ }
36+ }
37+
738#[ derive( Debug , Copy , Clone ) ]
839pub enum CompressionFormat {
940 Gz ,
@@ -26,7 +57,11 @@ impl CompressionFormat {
2657 }
2758 }
2859
29- pub ( crate ) fn encode ( & self , path : impl AsRef < Path > ) -> Result < Box < dyn Encoder > , Error > {
60+ pub ( crate ) fn encode (
61+ & self ,
62+ path : impl AsRef < Path > ,
63+ profile : CompressionProfile ,
64+ ) -> Result < Box < dyn Encoder > , Error > {
3065 let mut os = path. as_ref ( ) . as_os_str ( ) . to_os_string ( ) ;
3166 os. push ( format ! ( ".{}" , self . extension( ) ) ) ;
3267 let path = Path :: new ( & os) ;
@@ -37,49 +72,64 @@ impl CompressionFormat {
3772 let file = crate :: util:: create_new_file ( path) ?;
3873
3974 Ok ( match self {
40- CompressionFormat :: Gz => Box :: new ( GzEncoder :: new ( file, flate2:: Compression :: best ( ) ) ) ,
75+ CompressionFormat :: Gz => Box :: new ( GzEncoder :: new (
76+ file,
77+ match profile {
78+ CompressionProfile :: Fast => flate2:: Compression :: fast ( ) ,
79+ CompressionProfile :: Balanced => flate2:: Compression :: new ( 6 ) ,
80+ CompressionProfile :: Best => flate2:: Compression :: best ( ) ,
81+ } ,
82+ ) ) ,
4183 CompressionFormat :: Xz => {
42- let mut filters = xz2:: stream:: Filters :: new ( ) ;
43- // the preset is overridden by the other options so it doesn't matter
44- let mut lzma_ops = xz2:: stream:: LzmaOptions :: new_preset ( 9 ) . unwrap ( ) ;
45- // This sets the overall dictionary size, which is also how much memory (baseline)
46- // is needed for decompression.
47- lzma_ops. dict_size ( 64 * 1024 * 1024 ) ;
48- // Use the best match finder for compression ratio.
49- lzma_ops. match_finder ( xz2:: stream:: MatchFinder :: BinaryTree4 ) ;
50- lzma_ops. mode ( xz2:: stream:: Mode :: Normal ) ;
51- // Set nice len to the maximum for best compression ratio
52- lzma_ops. nice_len ( 273 ) ;
53- // Set depth to a reasonable value, 0 means auto, 1000 is somwhat high but gives
54- // good results.
55- lzma_ops. depth ( 1000 ) ;
56- // 2 is the default and does well for most files
57- lzma_ops. position_bits ( 2 ) ;
58- // 0 is the default and does well for most files
59- lzma_ops. literal_position_bits ( 0 ) ;
60- // 3 is the default and does well for most files
61- lzma_ops. literal_context_bits ( 3 ) ;
62-
63- filters. lzma2 ( & lzma_ops) ;
64-
65- let mut builder = xz2:: stream:: MtStreamBuilder :: new ( ) ;
66- builder. filters ( filters) ;
67-
68- // On 32-bit platforms limit ourselves to 3 threads, otherwise we exceed memory
69- // usage this process can take. In the future we'll likely only do super-fast
70- // compression in CI and move this heavyweight processing to promote-release (which
71- // is always 64-bit and can run on big-memory machines) but for now this lets us
72- // move forward.
73- if std:: mem:: size_of :: < usize > ( ) == 4 {
74- builder. threads ( 3 ) ;
75- } else {
76- builder. threads ( 6 ) ;
77- }
78-
79- let compressor = XzEncoder :: new_stream (
80- std:: io:: BufWriter :: new ( file) ,
81- builder. encoder ( ) . unwrap ( ) ,
82- ) ;
84+ let encoder = match profile {
85+ CompressionProfile :: Fast => {
86+ xz2:: stream:: MtStreamBuilder :: new ( ) . threads ( 6 ) . preset ( 1 ) . encoder ( ) . unwrap ( )
87+ }
88+ CompressionProfile :: Balanced => {
89+ xz2:: stream:: MtStreamBuilder :: new ( ) . threads ( 6 ) . preset ( 6 ) . encoder ( ) . unwrap ( )
90+ }
91+ CompressionProfile :: Best => {
92+ let mut filters = xz2:: stream:: Filters :: new ( ) ;
93+ // the preset is overridden by the other options so it doesn't matter
94+ let mut lzma_ops = xz2:: stream:: LzmaOptions :: new_preset ( 9 ) . unwrap ( ) ;
95+ // This sets the overall dictionary size, which is also how much memory (baseline)
96+ // is needed for decompression.
97+ lzma_ops. dict_size ( 64 * 1024 * 1024 ) ;
98+ // Use the best match finder for compression ratio.
99+ lzma_ops. match_finder ( xz2:: stream:: MatchFinder :: BinaryTree4 ) ;
100+ lzma_ops. mode ( xz2:: stream:: Mode :: Normal ) ;
101+ // Set nice len to the maximum for best compression ratio
102+ lzma_ops. nice_len ( 273 ) ;
103+ // Set depth to a reasonable value, 0 means auto, 1000 is somwhat high but gives
104+ // good results.
105+ lzma_ops. depth ( 1000 ) ;
106+ // 2 is the default and does well for most files
107+ lzma_ops. position_bits ( 2 ) ;
108+ // 0 is the default and does well for most files
109+ lzma_ops. literal_position_bits ( 0 ) ;
110+ // 3 is the default and does well for most files
111+ lzma_ops. literal_context_bits ( 3 ) ;
112+
113+ filters. lzma2 ( & lzma_ops) ;
114+
115+ let mut builder = xz2:: stream:: MtStreamBuilder :: new ( ) ;
116+ builder. filters ( filters) ;
117+
118+ // On 32-bit platforms limit ourselves to 3 threads, otherwise we exceed memory
119+ // usage this process can take. In the future we'll likely only do super-fast
120+ // compression in CI and move this heavyweight processing to promote-release (which
121+ // is always 64-bit and can run on big-memory machines) but for now this lets us
122+ // move forward.
123+ if std:: mem:: size_of :: < usize > ( ) == 4 {
124+ builder. threads ( 3 ) ;
125+ } else {
126+ builder. threads ( 6 ) ;
127+ }
128+ builder. encoder ( ) . unwrap ( )
129+ }
130+ } ;
131+
132+ let compressor = XzEncoder :: new_stream ( std:: io:: BufWriter :: new ( file) , encoder) ;
83133 Box :: new ( compressor)
84134 }
85135 } )
0 commit comments