Skip to content

Commit 37e1340

Browse files
levitationlevitation
committed
Refactored code into separate files. Increased min thread number of completion threads. Improved error handling. Added code for Volume Shadow Copy support, but keep it disabled for time being.
1 parent 96db866 commit 37e1340

22 files changed

+3562
-2292
lines changed

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,5 @@
33
/obj
44
/packages
55
/*.bak
6+
/FolderSyncNet.csproj.user
7+
/UpgradeLog.htm

‎App.config

+18-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<?xml version="1.0" encoding="utf-8"?>
22
<configuration>
33
<startup>
4-
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.1" />
4+
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.8" />
55
</startup>
66
<runtime>
77
<!-- https://docs.microsoft.com/en-us/archive/blogs/jeremykuhne/net-4-6-2-and-long-paths-on-windows-10 -->
@@ -13,7 +13,23 @@
1313
</dependentAssembly>
1414
<dependentAssembly>
1515
<assemblyIdentity name="System.Runtime.CompilerServices.Unsafe" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
16-
<bindingRedirect oldVersion="0.0.0.0-4.0.4.1" newVersion="4.0.4.1" />
16+
<bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
17+
</dependentAssembly>
18+
<dependentAssembly>
19+
<assemblyIdentity name="System.Diagnostics.Tracing" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
20+
<bindingRedirect oldVersion="0.0.0.0-4.2.0.0" newVersion="4.2.0.0" />
21+
</dependentAssembly>
22+
<dependentAssembly>
23+
<assemblyIdentity name="Microsoft.Bcl.AsyncInterfaces" publicKeyToken="cc7b13ffcd2ddd51" culture="neutral" />
24+
<bindingRedirect oldVersion="0.0.0.0-7.0.0.0" newVersion="7.0.0.0" />
25+
</dependentAssembly>
26+
<dependentAssembly>
27+
<assemblyIdentity name="System.Threading.Tasks.Extensions" publicKeyToken="cc7b13ffcd2ddd51" culture="neutral" />
28+
<bindingRedirect oldVersion="0.0.0.0-4.2.0.1" newVersion="4.2.0.1" />
29+
</dependentAssembly>
30+
<dependentAssembly>
31+
<assemblyIdentity name="System.Collections.Immutable" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
32+
<bindingRedirect oldVersion="0.0.0.0-7.0.0.0" newVersion="7.0.0.0" />
1733
</dependentAssembly>
1834
</assemblyBinding>
1935
</runtime>

‎BinaryFileExtensions.cs

+142-66
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,10 @@
1010
using System;
1111
using System.Data.Linq;
1212
using System.IO;
13+
using System.Runtime.InteropServices;
1314
using System.Threading;
1415
using System.Threading.Tasks;
16+
using Alphaleonis.Win32.Vss;
1517

1618
namespace FolderSync
1719
{
@@ -25,7 +27,7 @@ public static bool BinaryEqual(Binary a, Binary b)
2527
return a.Equals(b);
2628
}
2729

28-
public static async Task<Tuple<byte[], long>> ReadAllBytesAsync(string path, CancellationToken cancellationToken = default(CancellationToken), long maxFileSize = 0, int retryCount = 0, int readBufferKB = 0, int bufferReadDelayMs = 0, int? timeout = null, bool suppressLongRunningOperationMessage = false)
30+
public static async Task<Tuple<byte[], long>> ReadAllBytesAsync(string path, bool allowVSS, CancellationToken cancellationToken = default(CancellationToken), long maxFileSize = 0, int retryCount = 0, int readBufferKB = 0, int bufferReadDelayMs = 0, int? timeout = null, bool suppressLongRunningOperationMessage = false)
2931
{
3032
if (path == null)
3133
throw new ArgumentNullException(nameof(path));
@@ -41,72 +43,32 @@ public static bool BinaryEqual(Binary a, Binary b)
4143

4244
try
4345
{
44-
using (var stream = new FileStream
45-
(
46-
path,
47-
FileMode.Open,
48-
FileAccess.Read,
49-
FileShare.ReadWrite,
50-
bufferSize: 1024 * 1024,
51-
useAsync: true
52-
))
46+
if (allowVSS)
5347
{
54-
long longLen = stream.Length; //NB! the length might change during the code execution, so need to save it into separate variable
55-
56-
maxFileSize = Math.Min(MaxByteArraySize, maxFileSize);
57-
if (maxFileSize > 0 && longLen > maxFileSize)
58-
{
59-
return new Tuple<byte[], long>(null, longLen);
48+
try
49+
{
50+
var result = await ReadAllBytesWithVSSAsync(path, cancellationToken, maxFileSize, readBufferKB, bufferReadDelayMs, timeout, suppressLongRunningOperationMessage);
51+
return result;
6052
}
61-
62-
63-
int len = (int)longLen;
64-
byte[] result = new byte[len];
65-
66-
#if false
67-
//await stream.ReadAsync(result, 0, (int)len, cancellationToken);
68-
await Extensions.FSOperation
69-
(
70-
async () => await stream.ReadAsync(result, 0, (int)len, cancellationToken),
71-
path,
72-
cancellationToken,
73-
timeout: timeout ?? Global.FileBufferReadTimeout,
74-
suppressLongRunningOperationMessage: suppressLongRunningOperationMessage
75-
);
76-
#else
77-
var readBufferLength = readBufferKB * 1024;
78-
if (readBufferLength <= 0/* || bufferReadDelayMs <= 0*/) //NB! disable write buffer length limit if delay is 0
79-
readBufferLength = len;
80-
81-
for (int readOffset = 0; readOffset < len; readOffset += readBufferLength)
53+
catch (VssException)
8254
{
83-
if (readOffset > 0 && bufferReadDelayMs > 0)
84-
{
85-
#if !NOASYNC
86-
await Task.Delay(bufferReadDelayMs, cancellationToken);
87-
#else
88-
cancellationToken.WaitHandle.WaitOne(bufferReadDelayMs);
89-
#endif
90-
}
91-
92-
//await stream.WriteAsync(contents, i, writeBufferLength, cancellationToken);
93-
await Extensions.FSOperation
94-
(
95-
async () => await stream.ReadAsync(result, readOffset, Math.Min(readBufferLength, len - readOffset), cancellationToken),
96-
path,
97-
cancellationToken,
98-
timeout: timeout ?? Global.FileBufferReadTimeout,
99-
suppressLongRunningOperationMessage: suppressLongRunningOperationMessage
100-
);
101-
} //for (int i = 0; i < contents.Length; i += writeBufferLength)
102-
#endif
103-
104-
return new Tuple<byte[], long>(result, len);
55+
var result = await ReadAllBytesNoVSSAsync(path, cancellationToken, maxFileSize, readBufferKB, bufferReadDelayMs, timeout, suppressLongRunningOperationMessage);
56+
return result;
57+
}
58+
}
59+
else
60+
{
61+
var result = await ReadAllBytesNoVSSAsync(path, cancellationToken, maxFileSize, readBufferKB, bufferReadDelayMs, timeout, suppressLongRunningOperationMessage);
62+
return result;
10563
}
10664
}
10765
catch (Exception ex) when (
108-
ex is IOException
66+
/*ex is IOException
67+
|| ex is TimeoutException
10968
|| ex is UnauthorizedAccessException //can happen when a folder was just created //TODO: abandon retries after a certain number of attempts in this case
69+
|| */ex.GetInnermostException() is IOException
70+
|| ex.GetInnermostException() is TimeoutException
71+
|| ex.GetInnermostException() is UnauthorizedAccessException
11072
)
11173
{
11274
//retry after delay
@@ -126,7 +88,109 @@ ex is IOException
12688

12789
} //public static async Task<Tuple<byte[], long>> ReadAllBytesAsync()
12890

129-
public static async Task WriteAllBytesAsync(string path, byte[] contents, bool createTempFileFirst, CancellationToken cancellationToken = default(CancellationToken), int writeBufferKB = 0, int bufferWriteDelayMs = 0, int? timeout = null, bool suppressLongRunningOperationMessage = false)
91+
public static async Task<Tuple<byte[], long>> ReadAllBytesWithVSSAsync(string path, CancellationToken cancellationToken, long maxFileSize, int readBufferKB, int bufferReadDelayMs, int? timeout, bool suppressLongRunningOperationMessage)
92+
{
93+
if (path.StartsWith(@"\\?\")) //roland: VSS does not like \\?\ paths
94+
path = path.Substring(4);
95+
96+
using (var vss = new VssBackup())
97+
{
98+
vss.Setup(Path.GetPathRoot(path));
99+
100+
#if true
101+
using (var stream = vss.GetStream(path))
102+
#else
103+
using (var stream = new FileStream
104+
(
105+
vss.GetSnapshotPath(path),
106+
FileMode.Open,
107+
FileAccess.Read,
108+
FileShare.Read, //Write,
109+
bufferSize: 1024 * 1024,
110+
//useAsync: true
111+
options: FileOptions.Asynchronous | FileOptions.SequentialScan
112+
))
113+
#endif
114+
{
115+
var result = await ReadAllBytesFromStreamAsync(stream, path, cancellationToken, maxFileSize, readBufferKB, bufferReadDelayMs, timeout, suppressLongRunningOperationMessage);
116+
return result;
117+
}
118+
}
119+
}
120+
121+
public static async Task<Tuple<byte[], long>> ReadAllBytesNoVSSAsync(string path, CancellationToken cancellationToken, long maxFileSize, int readBufferKB, int bufferReadDelayMs, int? timeout, bool suppressLongRunningOperationMessage)
122+
{
123+
using (var stream = new FileStream
124+
(
125+
path,
126+
FileMode.Open,
127+
FileAccess.Read,
128+
FileShare.ReadWrite,
129+
bufferSize: 1024 * 1024,
130+
//useAsync: true
131+
options: FileOptions.Asynchronous | FileOptions.SequentialScan
132+
))
133+
{
134+
var result = await ReadAllBytesFromStreamAsync(stream, path, cancellationToken, maxFileSize, readBufferKB, bufferReadDelayMs, timeout, suppressLongRunningOperationMessage);
135+
return result;
136+
}
137+
}
138+
139+
public static async Task<Tuple<byte[], long>> ReadAllBytesFromStreamAsync(Stream stream, string path, CancellationToken cancellationToken, long maxFileSize, int readBufferKB, int bufferReadDelayMs, int? timeout, bool suppressLongRunningOperationMessage)
140+
{
141+
long longLen = stream.Length; //NB! the length might change during the code execution, so need to save it into separate variable
142+
143+
maxFileSize = Math.Min(MaxByteArraySize, maxFileSize);
144+
if (maxFileSize > 0 && longLen > maxFileSize)
145+
{
146+
return new Tuple<byte[], long>(null, longLen);
147+
}
148+
149+
150+
int len = (int)longLen;
151+
byte[] result = new byte[len];
152+
153+
#if false
154+
//await stream.ReadAsync(result, 0, (int)len, cancellationToken);
155+
await Extensions.FSOperation
156+
(
157+
async () => await stream.ReadAsync(result, 0, (int)len, cancellationToken),
158+
path,
159+
cancellationToken,
160+
timeout: timeout ?? Global.FileBufferReadTimeout,
161+
suppressLongRunningOperationMessage: suppressLongRunningOperationMessage
162+
);
163+
#else
164+
var readBufferLength = readBufferKB * 1024;
165+
if (readBufferLength <= 0/* || bufferReadDelayMs <= 0*/) //NB! disable read buffer length limit if delay is 0
166+
readBufferLength = len;
167+
168+
for (int readOffset = 0; readOffset < len; readOffset += readBufferLength)
169+
{
170+
if (readOffset > 0 && bufferReadDelayMs > 0)
171+
{
172+
#if !NOASYNC
173+
await Task.Delay(bufferReadDelayMs, cancellationToken);
174+
#else
175+
cancellationToken.WaitHandle.WaitOne(bufferReadDelayMs);
176+
#endif
177+
}
178+
179+
await Extensions.FSOperation
180+
(
181+
async () => await stream.ReadAsync(result, readOffset, Math.Min(readBufferLength, len - readOffset), cancellationToken),
182+
path,
183+
cancellationToken,
184+
timeout: timeout ?? Global.FileBufferReadTimeout,
185+
suppressLongRunningOperationMessage: suppressLongRunningOperationMessage
186+
);
187+
} //for (int i = 0; i < contents.Length; i += writeBufferLength)
188+
#endif
189+
190+
return new Tuple<byte[], long>(result, len);
191+
}
192+
193+
public static async Task WriteAllBytesAsync(string path, byte[] contents, bool createTempFileFirst, CancellationToken cancellationToken = default(CancellationToken), int retryCount = 0, int writeBufferKB = 0, int bufferWriteDelayMs = 0, int? timeout = null, bool suppressLongRunningOperationMessage = false)
130194
{
131195
if (path == null)
132196
throw new ArgumentNullException(nameof(path));
@@ -137,7 +201,8 @@ ex is IOException
137201
if (createTempFileFirst)
138202
tempPath += ".tmp";
139203

140-
while (true)
204+
retryCount = Math.Max(0, retryCount);
205+
for (int tryIndex = -1; tryIndex < retryCount; tryIndex++)
141206
{
142207
cancellationToken.ThrowIfCancellationRequested();
143208

@@ -149,8 +214,9 @@ ex is IOException
149214
FileMode.OpenOrCreate,
150215
FileAccess.Write,
151216
FileShare.Read,
152-
bufferSize: 1024 * 1024,
153-
useAsync: true
217+
bufferSize: 1024 * 1024,
218+
//useAsync: true
219+
options: FileOptions.Asynchronous | FileOptions.SequentialScan
154220
))
155221
{
156222
var writeBufferLength = writeBufferKB * 1024;
@@ -228,7 +294,14 @@ await Extensions.FSOperation
228294

229295
return; //exit while loop
230296
}
231-
catch (IOException)
297+
catch (Exception ex) when (
298+
/*ex is IOException
299+
|| ex is TimeoutException
300+
|| ex is UnauthorizedAccessException //can happen when a folder was just created //TODO: abandon retries after a certain number of attempts in this case
301+
|| */ex.GetInnermostException() is IOException
302+
|| ex.GetInnermostException() is TimeoutException
303+
|| ex.GetInnermostException() is UnauthorizedAccessException
304+
)
232305
{
233306
//retry after delay
234307

@@ -238,7 +311,10 @@ await Extensions.FSOperation
238311
cancellationToken.WaitHandle.WaitOne(1000);
239312
#endif
240313
}
241-
}
314+
} //for (int tryIndex = -1; tryIndex < retryCount; tryIndex++)
315+
316+
throw new ExternalException();
317+
242318
} //public static async Task WriteAllBytesAsync()
243319

244320
}

‎CachedFileInfo.cs

+78
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
//
2+
// Copyright (c) Roland Pihlakas 2019 - 2022
3+
// [email protected]
4+
//
5+
// Roland Pihlakas licenses this file to you under the GNU Lesser General Public License, ver 2.1.
6+
// See the LICENSE file for more information.
7+
//
8+
9+
using System;
10+
using System.Collections.Generic;
11+
using System.Diagnostics;
12+
using System.IO;
13+
using System.Linq;
14+
using System.Text;
15+
using System.Threading.Tasks;
16+
17+
namespace FolderSync
18+
{
19+
[Serializable]
20+
class CachedFileInfo
21+
{
22+
public long? Length { [DebuggerStepThrough]get; [DebuggerStepThrough]set; }
23+
public bool? Exists { [DebuggerStepThrough]get; [DebuggerStepThrough]set; }
24+
25+
public DateTime CreationTimeUtc { [DebuggerStepThrough]get; [DebuggerStepThrough]set; }
26+
public DateTime LastWriteTimeUtc { [DebuggerStepThrough]get; [DebuggerStepThrough]set; }
27+
28+
public string FullName { [DebuggerStepThrough]get; [DebuggerStepThrough]set; }
29+
30+
public FileAttributes Attributes { [DebuggerStepThrough]get; [DebuggerStepThrough]set; }
31+
32+
public CachedFileInfo(string fullName, long length, DateTime lastWriteTimeUtc)
33+
{
34+
Exists = true;
35+
Length = length;
36+
37+
CreationTimeUtc = lastWriteTimeUtc;
38+
LastWriteTimeUtc = lastWriteTimeUtc;
39+
FullName = fullName;
40+
Attributes = FileAttributes.Normal;
41+
}
42+
43+
public CachedFileInfo(CachedFileInfo fileInfo, bool useNonFullPath)
44+
{
45+
Exists = fileInfo.Exists;
46+
Length = fileInfo.Length;
47+
48+
CreationTimeUtc = fileInfo.CreationTimeUtc;
49+
LastWriteTimeUtc = fileInfo.LastWriteTimeUtc;
50+
FullName = useNonFullPath ? ConsoleWatch.GetNonFullName(fileInfo.FullName) : fileInfo.FullName;
51+
Attributes = fileInfo.Attributes;
52+
}
53+
54+
public CachedFileInfo(FileInfo fileInfo)
55+
{
56+
Exists = fileInfo.Exists;
57+
Length = Exists == true ? (long?)fileInfo.Length : null; //need to check for exists else exception occurs during reading length
58+
59+
CreationTimeUtc = fileInfo.CreationTimeUtc;
60+
LastWriteTimeUtc = fileInfo.LastWriteTimeUtc;
61+
FullName = fileInfo.FullName;
62+
Attributes = fileInfo.Attributes;
63+
}
64+
65+
public CachedFileInfo(FileSystemInfo fileSystemInfo)
66+
{
67+
var fileInfo = fileSystemInfo as FileInfo;
68+
69+
Exists = fileInfo?.Exists;
70+
Length = Exists == true ? fileInfo?.Length : null;
71+
72+
CreationTimeUtc = fileSystemInfo.CreationTimeUtc;
73+
LastWriteTimeUtc = fileSystemInfo.LastWriteTimeUtc;
74+
FullName = fileSystemInfo.FullName;
75+
Attributes = fileSystemInfo.Attributes;
76+
}
77+
} //private class CachedFileInfo : ISerializable
78+
}

0 commit comments

Comments
 (0)