SF-3159 Upgrade to SharpZipLib (#2967)

Author: josephmyers

src/SIL.XForge.Scripture/SIL.XForge.Scripture.csproj

@@ -28,7 +28,6 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="CsvHelper" Version="33.0.1" />
-    <PackageReference Include="DotNetZip" Version="1.16.0" />
     <PackageReference Include="Duende.AccessTokenManagement" Version="3.2.0" />
     <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.7" />
     <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
@@ -42,6 +41,7 @@
          InternetSettings.xml file. Also update server config scriptureforge.org_v2.yml. -->
     <PackageReference Include="ParatextData" Version="9.5.0.8" />
     <PackageReference Include="Serval.Client" Version="1.9.0" />
+    <PackageReference Include="SharpZipLib" Version="1.4.2" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
     <PackageReference Include="Swashbuckle.AspNetCore.Newtonsoft" Version="6.6.2" />
     <PackageReference Include="System.Text.Encoding.CodePages" Version="8.0.0" />

src/SIL.XForge.Scripture/Services/ParatextService.cs

@@ -274,7 +274,7 @@ SyncMetrics syncMetrics
                         + $"{paratextId}"
                 );
             }
-            EnsureProjectReposExists(userSecret, ptProject, source);
+            await EnsureProjectReposExistsAsync(userSecret, ptProject, source);
             if (ptProject is not ParatextResource)
             {
                 StartProgressReporting(progress);
@@ -2480,7 +2480,7 @@ private void InstallStyles()
     /// <summary>
     /// Ensure the target project repository exists on the local SF server, cloning if necessary.
     /// </summary>
-    private void EnsureProjectReposExists(
+    private async Task EnsureProjectReposExistsAsync(
         UserSecret userSecret,
         ParatextProject target,
         IInternetSharedRepositorySource repositorySource
@@ -2492,7 +2492,7 @@ IInternetSharedRepositorySource repositorySource
         if (target is ParatextResource resource)
         {
             // If the target is a resource, install it
-            InstallResource(username, resource, target.ParatextId, targetNeedsCloned);
+            await InstallResourceAsync(username, resource, target.ParatextId, targetNeedsCloned);
         }
         else if (targetNeedsCloned)
         {
@@ -2515,7 +2515,7 @@ IInternetSharedRepositorySource repositorySource
     /// <remarks>
     ///   <paramref name="targetParatextId" /> is required because the resource may be a source or target.
     /// </remarks>
-    private void InstallResource(
+    async private Task InstallResourceAsync(
         string username,
         ParatextResource resource,
         string targetParatextId,
@@ -2553,7 +2553,7 @@ bool needsToBeCloned
             {
                 string path = LocalProjectDir(targetParatextId);
                 _fileSystemService.CreateDirectory(path);
-                resource.InstallableResource.ExtractToDirectory(path);
+                await resource.InstallableResource.ExtractToDirectoryAsync(path);
                 MigrateResourceIfRequired(username, targetParatextId, overrideLanguageId);
             }
         }
@@ -2903,7 +2903,8 @@ CancellationToken token
             _exceptionHandler,
             _dblServerUri
         );
-        IReadOnlyDictionary<string, int> resourceRevisions = SFInstallableDblResource.GetInstalledResourceRevisions();
+        IReadOnlyDictionary<string, int> resourceRevisions =
+            await SFInstallableDblResource.GetInstalledResourceRevisionsAsync();
         return resources
             .OrderBy(r => r.FullName)
             .Select(r =>

src/SIL.XForge.Scripture/Services/SFInstallableDblResource.cs

@@ -3,7 +3,8 @@
 using System.IO;
 using System.Linq;
 using System.Net;
-using Ionic.Zip;
+using System.Threading.Tasks;
+using ICSharpCode.SharpZipLib.Zip;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using Paratext.Data;
@@ -376,7 +377,7 @@ public static IEnumerable<SFInstallableDblResource> GetInstallableDblResources(
     /// <remarks>
     /// After the resource is extracted, it can be a source or target.
     /// </remarks>
-    public void ExtractToDirectory(string path)
+    async public Task ExtractToDirectoryAsync(string path)
     {
         // Check parameters
         if (string.IsNullOrWhiteSpace(path))
@@ -398,11 +399,34 @@ public void ExtractToDirectory(string path)
             this.DBLEntryUid,
             ProjectFileManager.resourceFileExtension
         );
-        if (RobustFile.Exists(resourceFile))
+        if (_fileSystemService.FileExists(resourceFile))
         {
-            using var zipFile = ZipFile.Read(resourceFile);
-            zipFile.Password = this._passwordProvider?.GetPassword();
-            zipFile.ExtractAll(path, ExtractExistingFileAction.DoNotOverwrite);
+            await using Stream stream = _fileSystemService.OpenFile(resourceFile, FileMode.Open);
+            using ZipFile zipFile = new ZipFile(stream);
+            zipFile.Password = _passwordProvider?.GetPassword();
+            await ExtractAllAsync(zipFile, path);
+        }
+    }
+
+    private async Task ExtractAllAsync(ZipFile zip, string path)
+    {
+        foreach (ZipEntry entry in zip)
+        {
+            if (!entry.IsFile)
+                continue; // Skip directories
+
+            string entryPath = Path.Combine(path, entry.Name);
+
+            if (_fileSystemService.FileExists(entryPath))
+                continue; // Don't overwrite
+
+            // Ensure directories in the ZIP entry are created
+            _fileSystemService.CreateDirectory(Path.GetDirectoryName(entryPath));
+
+            // Extract the file
+            await using Stream zipStream = zip.GetInputStream(entry);
+            await using Stream output = _fileSystemService.CreateFile(entryPath);
+            await zipStream.CopyToAsync(output);
         }
     }
 
@@ -475,7 +499,7 @@ public override bool Install()
     /// <returns>
     /// A dictionary where the resource id is the key, and the revision is the value.
     /// </returns>
-    internal static IReadOnlyDictionary<string, int> GetInstalledResourceRevisions()
+    internal static async Task<IReadOnlyDictionary<string, int>> GetInstalledResourceRevisionsAsync()
     {
         // Initialize variables
         Dictionary<string, int> resourceRevisions = [];
@@ -510,9 +534,8 @@ internal static IReadOnlyDictionary<string, int> GetInstalledResourceRevisions()
                 // See if this a zip file, and if it contains the correct ID
                 try
                 {
-                    // This only uses DotNetZip because ParatextData uses DotNetZip
-                    // You could use System.IO.Compression if you wanted to
-                    using var zipFile = ZipFile.Read(resourceFile);
+                    await using var stream = new FileStream(resourceFile, FileMode.Open);
+                    using var zipFile = new ZipFile(stream);
                     // Zip files use forward slashes, even on Windows
                     const string idSearchPath = DblFolderName + "/id/";
                     const string revisionSearchPath = DblFolderName + "/revision/";
@@ -524,19 +547,19 @@ internal static IReadOnlyDictionary<string, int> GetInstalledResourceRevisions()
                         if (
                             string.IsNullOrWhiteSpace(fileId)
                             && !entry.IsDirectory
-                            && entry.FileName.StartsWith(idSearchPath, StringComparison.OrdinalIgnoreCase)
+                            && entry.Name.StartsWith(idSearchPath, StringComparison.OrdinalIgnoreCase)
                         )
                         {
-                            fileId = entry.FileName.Split('/', StringSplitOptions.RemoveEmptyEntries).Last();
+                            fileId = entry.Name.Split('/', StringSplitOptions.RemoveEmptyEntries).Last();
                         }
                         else if (
                             revision == 0
                             && !entry.IsDirectory
-                            && entry.FileName.StartsWith(revisionSearchPath, StringComparison.OrdinalIgnoreCase)
+                            && entry.Name.StartsWith(revisionSearchPath, StringComparison.OrdinalIgnoreCase)
                         )
                         {
                             string revisionFilename = entry
-                                .FileName.Split('/', StringSplitOptions.RemoveEmptyEntries)
+                                .Name.Split('/', StringSplitOptions.RemoveEmptyEntries)
                                 .Last();
                             if (!int.TryParse(revisionFilename, out revision))
                             {

src/SIL.XForge.Scripture/packages.lock.json

@@ -8,16 +8,6 @@
         "resolved": "33.0.1",
         "contentHash": "fev4lynklAU2A9GVMLtwarkwaanjSYB4wUqO2nOJX5hnzObORzUqVLe+bDYCUyIIRQM4o5Bsq3CcyJR89iMmEQ=="
       },
-      "DotNetZip": {
-        "type": "Direct",
-        "requested": "[1.16.0, )",
-        "resolved": "1.16.0",
-        "contentHash": "CS9sjjXF23FLIbwlLT5k/jXBYxVnLMn01lyAvN/hxFM+UDvSicjTPB9ZMA+Wp/QlvBPbD2pzkAXUp1O50gP/Lw==",
-        "dependencies": {
-          "System.Security.Permissions": "4.7.0",
-          "System.Text.Encoding.CodePages": "4.7.1"
-        }
-      },
       "Duende.AccessTokenManagement": {
         "type": "Direct",
         "requested": "[3.2.0, )",
@@ -164,7 +154,6 @@
         "resolved": "9.5.0.8",
         "contentHash": "TFRH8KBM77q5seEI6W8iOENk5XY5KXRIxJnllEVO6PT9D37nDHIqDKvq5Y3ilpsPZjhZj/F/lVciw1dOkZ9gzQ==",
         "dependencies": {
-          "DotNetZip": "1.16.0",
           "Icu4c.Win.Min": "59.1.7",
           "JetBrains.Annotations": "2021.2.0",
           "Microsoft.CSharp": "4.7.0",

test/SIL.XForge.Scripture.Tests/Services/ParatextServiceTests.cs

@@ -10,6 +10,7 @@
 using System.Threading.Tasks;
 using System.Xml;
 using System.Xml.Linq;
+using ICSharpCode.SharpZipLib.Zip;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
@@ -4370,6 +4371,10 @@ public async Task SendReceiveAsync_SourceResource_MigratesLdml()
             )
             .Returns(resourceScrText);
         env.MockFileSystemService.FileExists(Arg.Is<string>(p => p.EndsWith(".p8z"))).Returns(true);
+        await using var zipStream = await TestEnvironment.CreateZipStubAsync();
+        env.MockFileSystemService.OpenFile(Arg.Is<string>(p => p.EndsWith(".p8z")), FileMode.Open).Returns(zipStream);
+        await using var stream = new MemoryStream();
+        env.MockFileSystemService.CreateFile(Arg.Any<string>()).Returns(stream);
 
         // Set up the Resource ScrText when it is installed on disk
         using MockScrText scrText = env.GetScrText(associatedPtUser, resourceId);
@@ -4420,6 +4425,10 @@ public async Task SendReceiveAsync_SourceResource_DblLanguageDifferent()
             )
             .Returns(resourceScrText);
         env.MockFileSystemService.FileExists(Arg.Is<string>(p => p.EndsWith(".p8z"))).Returns(true);
+        await using var zipStream = await TestEnvironment.CreateZipStubAsync();
+        env.MockFileSystemService.OpenFile(Arg.Is<string>(p => p.EndsWith(".p8z")), FileMode.Open).Returns(zipStream);
+        await using var stream = new MemoryStream();
+        env.MockFileSystemService.CreateFile(Arg.Any<string>()).Returns(stream);
 
         // Set up the Resource ScrText when it is installed on disk
         using MockScrText scrText = env.GetScrText(associatedPtUser, resourceId);
@@ -4483,6 +4492,10 @@ public async Task SendReceiveAsync_SourceResource_DblLanguageMissing()
         bool resourceDownloaded = false;
         env.MockFileSystemService.When(f => f.CreateDirectory(Arg.Any<string>())).Do(_ => resourceDownloaded = true);
         env.MockFileSystemService.FileExists(Arg.Is<string>(p => p.EndsWith(".p8z"))).Returns(_ => resourceDownloaded);
+        await using var zipStream = await TestEnvironment.CreateZipStubAsync();
+        env.MockFileSystemService.OpenFile(Arg.Is<string>(p => p.EndsWith(".p8z")), FileMode.Open).Returns(zipStream);
+        await using var stream = new MemoryStream();
+        env.MockFileSystemService.CreateFile(Arg.Any<string>()).Returns(stream);
 
         // Set up the Resource ScrText when it is installed on disk
         using MockScrText scrText = env.GetScrText(associatedPtUser, resourceId);
@@ -7295,6 +7308,23 @@ string[] dataIds
         public static async Task<IDocument<NoteThread>> GetNoteThreadDocAsync(IConnection connection, string dataId) =>
             await connection.FetchAsync<NoteThread>("project01:" + dataId);
 
+        public static async Task<Stream> CreateZipStubAsync()
+        {
+            var outputMemStream = new MemoryStream();
+            await using (var zipStream = new ZipOutputStream(outputMemStream))
+            {
+                ZipEntry newEntry = new ZipEntry("test.txt");
+                await zipStream.PutNextEntryAsync(newEntry);
+                await zipStream.CloseEntryAsync(CancellationToken.None);
+
+                // Stop ZipStream.Dispose() from also closing the underlying stream.
+                zipStream.IsStreamOwner = false;
+            }
+
+            outputMemStream.Position = 0;
+            return outputMemStream;
+        }
+
         public string SetupProject(string baseId, ParatextUser associatedPtUser, bool hasEditPermission = true)
         {
             string ptProjectId = PTProjectIds[baseId].Id;