Skip to content

Commit 0ad69ce

Browse files
committed
fix package split
1 parent 55445fd commit 0ad69ce

13 files changed

+50
-69
lines changed

Directory.Build.props

+1-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545

4646
<OutputPath Condition="'$(OutputPath)'==''">$(BaseOutputPath)$(PlatformConfig)\$(MSBuildProjectName)\</OutputPath>
4747

48-
<PackageAssetsPath>$(ObjDir)/$(Configuration)/packages/</PackageAssetsPath>
48+
<PackagePreparationPath>$(ObjDir)/$(Configuration)/packprep/</PackagePreparationPath>
4949

5050
<PackageOutputPath Condition="'$(PackageOutputPath)'==''">$(BinDir)packages/$(Configuration)/</PackageOutputPath>
5151

Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ all:
44
docs: doc-update yaml
55

66
doc-update:
7-
mdoc update -i ./bin/obj/packages/TorchSharp/lib/netstandard2.0/TorchSharp.xml -o ecmadocs/en ./bin/obj/packages/TorchSharp/lib/netstandard2.0/TorchSharp.dll
7+
mdoc update -i ./bin/obj/packprep/TorchSharp/lib/netstandard2.0/TorchSharp.xml -o ecmadocs/en ./bin/obj/packprep/TorchSharp/lib/netstandard2.0/TorchSharp.dll
88

99
yaml:
1010
-rm ecmadocs/en/ns-.xml

README.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -121,8 +121,8 @@ full range of native bits. Instead they are built using Azure Pipelines.
121121

122122
An ephemeral feed of packages from CI is available
123123

124-
* View link: https://dev.azure.com/migueldeicaza/TorchSharp/_packaging?_a=feed&feed=packages%40Local
125-
* Nuget link: https://pkgs.dev.azure.com/migueldeicaza/TorchSharp/_packaging/packages%40Local/nuget/v3/index.json.
124+
* View link: https://donsyme.visualstudio.com/TorchSharp/_packaging?_a=feed&feed=packages2
125+
* Nuget feed: https://donsyme.pkgs.visualstudio.com/TorchSharp/_packaging/packages2/nuget/v3/index.json
126126

127127

128128

azure-pipelines.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ jobs:
149149
targetFolder: $(Build.ArtifactStagingDirectory)
150150

151151
# - script: |
152-
# cp $(Build.SourcesDirectory)\bin\obj\packages\$(BuildConfig)\*.nupkg $BUILD_ARTIFACTSTAGINGDIRECTORY
152+
# cp $(Build.SourcesDirectory)\bin\obj\packprep\$(BuildConfig)\*.nupkg $BUILD_ARTIFACTSTAGINGDIRECTORY
153153
# # rm $BUILD_ARTIFACTSTAGINGDIRECTORY/*.symbols.nupkg
154154
# displayName: Copy Nugets in staging directory
155155

build/sign.proj

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
<PropertyGroup>
1515
<!-- The OutDir and IntermediateOutputPath properties are required by MicroBuild. MicroBuild only
1616
signs files that are under these paths. -->
17-
<OutDir Condition="'$(OutDir)' == '' AND '$(SignNugetPackages)' != 'true'">$(PackageAssetsPath)</OutDir>
17+
<OutDir Condition="'$(OutDir)' == '' AND '$(SignNugetPackages)' != 'true'">$(PackagePreparationPath)</OutDir>
1818
<OutDir Condition="'$(OutDir)' == '' AND '$(SignNugetPackages)' == 'true'">$(PackageOutputPath)</OutDir>
1919
<IntermediateOutputPath Condition="'$(IntermediateOutputPath)' == ''">$(IntermediateOutputRootPath)</IntermediateOutputPath>
2020

build/vsts-ci.yml

+7-7
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ phases:
2929
- task: PublishBuildArtifacts@1
3030
displayName: Publish Linux package assets
3131
inputs:
32-
pathToPublish: $(Build.SourcesDirectory)/bin/obj/packages
32+
pathToPublish: $(Build.SourcesDirectory)/bin/obj/packprep
3333
artifactName: PackageAssets
3434
artifactType: container
3535

@@ -55,7 +55,7 @@ phases:
5555
- task: PublishBuildArtifacts@1
5656
displayName: Publish macOS package assets
5757
inputs:
58-
pathToPublish: $(Build.SourcesDirectory)/bin/obj/packages
58+
pathToPublish: $(Build.SourcesDirectory)/bin/obj/packprep
5959
artifactName: PackageAssets
6060
artifactType: container
6161

@@ -103,7 +103,7 @@ phases:
103103
- task: PublishBuildArtifacts@1
104104
displayName: Publish Windows_x64 package assets
105105
inputs:
106-
pathToPublish: $(Build.SourcesDirectory)/bin/obj/packages
106+
pathToPublish: $(Build.SourcesDirectory)/bin/obj/packprep
107107
artifactName: PackageAssets
108108
artifactType: container
109109

@@ -152,20 +152,20 @@ phases:
152152
displayName: Download package assets
153153
inputs:
154154
artifactName: PackageAssets
155-
downloadPath: $(Build.SourcesDirectory)/bin/obj/packages
155+
downloadPath: $(Build.SourcesDirectory)/bin/obj/packprep
156156

157157
# Workaround https://github.com/Microsoft/vsts-tasks/issues/6739
158158
- task: CopyFiles@2
159159
displayName: Copy package assets to correct folder
160160
inputs:
161-
sourceFolder: $(Build.SourcesDirectory)/bin/obj/packages/PackageAssets
162-
targetFolder: $(Build.SourcesDirectory)/bin/obj/packages
161+
sourceFolder: $(Build.SourcesDirectory)/bin/obj/packprep/PackageAssets
162+
targetFolder: $(Build.SourcesDirectory)/bin/obj/packprep
163163

164164
- script: ./build.cmd pack -c $(BuildConfig)
165165
displayName: Create Packages
166166

167167
- script: |
168-
cp $(Build.SourcesDirectory)/bin/obj/packages*.nupkg $BUILD_ARTIFACTSTAGINGDIRECTORY
168+
cp $(Build.SourcesDirectory)/bin/obj/packprep*.nupkg $BUILD_ARTIFACTSTAGINGDIRECTORY
169169
rm $BUILD_ARTIFACTSTAGINGDIRECTORY/*.symbols.nupkg
170170
displayName: Copy Nugets in staging directory
171171

pkg/Directory.Build.props

+8-8
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,11 @@
3333

3434

3535
<ItemGroup>
36-
<Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\lib\**\*" Pack="true" PackagePath="lib" />
37-
<Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\analyzers\**\*" Pack="true" PackagePath="analyzers" />
38-
<Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\**\*" Pack="true" PackagePath="runtimes" />
39-
<Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\fragments\**\*" Pack="true" PackagePath="fragments" />
40-
<Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\tools\**\*" Pack="true" PackagePath="tools" />
36+
<Content Include="$(PackagePreparationPath)$(PackageIdFolderName)\lib\**\*" Pack="true" PackagePath="lib" />
37+
<Content Include="$(PackagePreparationPath)$(PackageIdFolderName)\analyzers\**\*" Pack="true" PackagePath="analyzers" />
38+
<Content Include="$(PackagePreparationPath)$(PackageIdFolderName)\runtimes\**\*" Pack="true" PackagePath="runtimes" />
39+
<Content Include="$(PackagePreparationPath)$(PackageIdFolderName)\fragments\**\*" Pack="true" PackagePath="fragments" />
40+
<Content Include="$(PackagePreparationPath)$(PackageIdFolderName)\tools\**\*" Pack="true" PackagePath="tools" />
4141
</ItemGroup>
4242

4343
<ItemGroup Condition="'$(IncludeNotices)' != 'false'">
@@ -46,9 +46,9 @@
4646
</ItemGroup>
4747

4848
<ItemGroup Condition="'$(IsSymbolsPackage)' != 'true'">
49-
<Content Remove="$(PackageAssetsPath)$(PackageIdFolderName)\**\*.pdb" />
50-
<Content Remove="$(PackageAssetsPath)$(PackageIdFolderName)\**\*.dwarf" />
51-
<Content Remove="$(PackageAssetsPath)$(PackageIdFolderName)\**\*.dbg" />
49+
<Content Remove="$(PackagePreparationPath)$(PackageIdFolderName)\**\*.pdb" />
50+
<Content Remove="$(PackagePreparationPath)$(PackageIdFolderName)\**\*.dwarf" />
51+
<Content Remove="$(PackagePreparationPath)$(PackageIdFolderName)\**\*.dbg" />
5252
</ItemGroup>
5353

5454
<!-- Work around https://github.com/NuGet/Home/issues/6091 -->

pkg/common/CommonPackage.targets

+8-25
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
22

33

4-
<!-- some massive files are deliverd in fragments across multiple nuget packages -->
4+
<!-- Some massive files > 1GB are delivered in fragments across multiple nuget packages. -->
5+
<!-- Here we hack them back together on first build using the package install -->
56
<UsingTask
67
TaskName="FileDefragger"
78
TaskFactory="RoslynCodeTaskFactory"
@@ -35,41 +36,23 @@
3536
3637
if (File.Exists(fragmentFile1) && File.Exists(fragmentFile2) && File.Exists(fragmentFile3))
3738
{
38-
System.Console.WriteLine("Fragment files in parallel nuget packages for {0}", primaryFile);
39-
4039
Console.WriteLine("Found fragment file at {0}", fragmentFile1);
4140
Console.WriteLine("Found fragment file at {0}", fragmentFile2);
4241
Console.WriteLine("Found fragment file at {0}", fragmentFile3);
4342
var primaryBytes = File.ReadAllBytes(primaryFile);
4443
var fragmentBytes1 = File.ReadAllBytes(fragmentFile1);
4544
var fragmentBytes2 = File.ReadAllBytes(fragmentFile2);
4645
var fragmentBytes3 = File.ReadAllBytes(fragmentFile3);
46+
var outputBytes = new byte[primaryBytes.Length + fragmentBytes1.Length + fragmentBytes2.Length + fragmentBytes3.Length];
4747
48-
if (primaryBytes.Length != fragmentBytes1.Length)
49-
throw (new Exception(String.Format("Mismatched file sizes for multi-package deliver of single massive binary file. Primary file {0} found, but the corresponding fragment at {1} has a different size. You may have inconsistent nuget package references.", primaryFile, fragmentFile1)));
50-
51-
if (primaryBytes.Length != fragmentBytes2.Length)
52-
throw (new Exception(String.Format("Mismatched file sizes for multi-package deliver of single massive binary file. Primary file {0} found, but the corresponding fragment at {1} has a different size. You may have inconsistent nuget package references.", primaryFile, fragmentFile2)));
53-
54-
if (primaryBytes.Length != fragmentBytes3.Length)
55-
throw (new Exception(String.Format("Mismatched file sizes for multi-package deliver of single massive binary file. Primary file {0} found, but the corresponding fragment at {1} has a different size. You may have inconsistent nuget package references.", primaryFile, fragmentFile3)));
56-
57-
for (int i = 0; i < primaryBytes.Length; i++)
58-
{
59-
if ((primaryBytes[i] > 0) && (fragmentBytes1[i] > 0) && (primaryBytes[i] != fragmentBytes1[i]))
60-
throw (new Exception(String.Format("Inconsistent file contents for multi-package deliver of single massive binary file. Primary file {0} and corresponding fragment {1} had inconsistent contents. You may have inconsistent nuget package references.", primaryFile, fragmentFile1)));
61-
if ((primaryBytes[i] > 0) && (fragmentBytes2[i] > 0) && (primaryBytes[i] != fragmentBytes2[i]))
62-
throw (new Exception(String.Format("Inconsistent file contents for multi-package deliver of single massive binary file. Primary file {0} and corresponding fragment {1} had inconsistent contents. You may have inconsistent nuget package references.", primaryFile, fragmentFile2)));
63-
if ((primaryBytes[i] > 0) && (fragmentBytes3[i] > 0) && (primaryBytes[i] != fragmentBytes3[i]))
64-
throw (new Exception(String.Format("Inconsistent file contents for multi-package deliver of single massive binary file. Primary file {0} and corresponding fragment {1} had inconsistent contents. You may have inconsistent nuget package references.", primaryFile, fragmentFile3)));
65-
primaryBytes[i] |= fragmentBytes1[i];
66-
primaryBytes[i] |= fragmentBytes2[i];
67-
primaryBytes[i] |= fragmentBytes3[i];
68-
}
48+
Array.Copy(primaryBytes, 0, outputBytes, 0, primaryBytes.Length);
49+
Array.Copy(fragmentBytes1, 0, outputBytes, primaryBytes.Length, fragmentBytes1.Length);
50+
Array.Copy(fragmentBytes2, 0, outputBytes, primaryBytes.Length + fragmentBytes1.Length, fragmentBytes2.Length);
51+
Array.Copy(fragmentBytes3, 0, outputBytes, primaryBytes.Length + fragmentBytes1.Length + fragmentBytes2.Length, fragmentBytes3.Length);
6952
7053
var tmpFile = Path.GetTempFileName();
7154
Console.WriteLine("Writing restored primary file at {0}", tmpFile);
72-
File.WriteAllBytes(tmpFile, primaryBytes);
55+
File.WriteAllBytes(tmpFile, outputBytes);
7356
Console.WriteLine("Deleting {0}", primaryFile);
7457
File.Delete(primaryFile);
7558
Console.WriteLine("Moving {0} --> {1}", tmpFile, primaryFile);

pkg/pack.proj

+6-6
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,13 @@
33
<Target Name="Pack">
44
<Message Text="Restoring packaging projects..." Importance="high" />
55

6-
<Warning Text="Packages will be incomplete and unusable on linux platforms. To get a complete package you need the LibTorchSharp.so binaries for other platforms and copy them into '$(PackageAssetsPath)' to make complete packages. This is automated by Azure Pipelines."
7-
Condition="!Exists('$(PackageAssetsPath)\libtorch-cpu\runtimes\linux-x64\native\libLibTorchSharp.so')
8-
OR !Exists('$(PackageAssetsPath)\libtorch-cuda-$(CudaVersionDot)\runtimes\linux-x64\native\libLibTorchSharp.so')
6+
<Warning Text="Packages will be incomplete and unusable on linux platforms. To get a complete package you need the LibTorchSharp.so binaries for other platforms and copy them into '$(PackagePreparationPath)' to make complete packages. This is automated by Azure Pipelines."
7+
Condition="!Exists('$(PackagePreparationPath)\libtorch-cpu\runtimes\linux-x64\native\libLibTorchSharp.so')
8+
OR !Exists('$(PackagePreparationPath)\libtorch-cuda-$(CudaVersionDot)\runtimes\linux-x64\native\libLibTorchSharp.so')
99
" />
10-
<Warning Text="Packages will be incomplete and unusable on win-x64 platform. To get a complete package you need the LibTorchSharp.dll binaries for other platforms and copy them into '$(PackageAssetsPath)' to make complete packages. This is automated by Azure Pipelines."
11-
Condition="!Exists('$(PackageAssetsPath)\libtorch-cpu\runtimes\win-x64\native\LibTorchSharp.dll')
12-
OR !Exists('$(PackageAssetsPath)\libtorch-cuda-$(CudaVersionDot)\runtimes\win-x64\native\LibTorchSharp.dll')
10+
<Warning Text="Packages will be incomplete and unusable on win-x64 platform. To get a complete package you need the LibTorchSharp.dll binaries for other platforms and copy them into '$(PackagePreparationPath)' to make complete packages. This is automated by Azure Pipelines."
11+
Condition="!Exists('$(PackagePreparationPath)\libtorch-cpu\runtimes\win-x64\native\LibTorchSharp.dll')
12+
OR !Exists('$(PackagePreparationPath)\libtorch-cuda-$(CudaVersionDot)\runtimes\win-x64\native\LibTorchSharp.dll')
1313
" />
1414

1515

src/Directory.Build.targets

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
</ItemGroup>
2323

2424
<Copy SourceFiles="@(PackageAsset)"
25-
DestinationFolder="$(PackageAssetsPath)%(PackageAsset.RelativePath)" />
25+
DestinationFolder="$(PackagePreparationPath)%(PackageAsset.RelativePath)" />
2626

2727
</Target>
2828

src/Native/build.proj

+1-1
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@
7575
</ItemGroup>
7676

7777
<Copy SourceFiles="@(NativePackageAsset)"
78-
DestinationFolder="$(PackageAssetsPath)%(NativePackageAsset.RelativePath)" />
78+
DestinationFolder="$(PackagePreparationPath)%(NativePackageAsset.RelativePath)" />
7979

8080
</Target>
8181

src/Redist/libtorch-cpu/libtorch-cpu.proj

+3-3
Original file line numberDiff line numberDiff line change
@@ -231,12 +231,12 @@
231231

232232
<!-- copy to packaging location -->
233233
<FilesFromArchive Include="@(_fileFromArchive->'%(ExtractDirectory)\%(Identity)')"
234-
TargetPath="$(PackageAssetsPath)$(MSBuildProjectName)\%(PackagePath)" />
234+
TargetPath="$(PackagePreparationPath)$(MSBuildProjectName)\%(PackagePath)" />
235235
<!-- include LICENSE that was downloaded from GitHub -->
236236
<FilesFromArchive Include="$(IntermediateOutputPath)\LICENSE-LIBTORCH"
237-
TargetPath="$(PackageAssetsPath)$(MSBuildProjectName)\LICENSE-LIBTORCH.txt" />
237+
TargetPath="$(PackagePreparationPath)$(MSBuildProjectName)\LICENSE-LIBTORCH.txt" />
238238
<FilesFromArchive Include="$(IntermediateOutputPath)\LICENSE-MKL-DNN"
239-
TargetPath="$(PackageAssetsPath)$(MSBuildProjectName)\LICENSE-MKL-DNN.txt" />
239+
TargetPath="$(PackagePreparationPath)$(MSBuildProjectName)\LICENSE-MKL-DNN.txt" />
240240

241241
<!-- copy to NativeAssets location, only for current RID, so that they may be used by tests -->
242242
<FilesFromArchive Condition="'$(PackageRID)' == '%(_fileFromArchive.Runtime)'"

src/Redist/libtorch-cuda-10.2/libtorch-cuda-10.2.proj

+10-12
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
<DownloadedLicenseFolder>$(IntermediateOutputPath)</DownloadedLicenseFolder>
2525
<DownloadedLicenseFileName>LICENSE-LIBTORCH</DownloadedLicenseFileName>
2626
<DownloadedLicenseFile>$(DownloadedLicenseFolder)$(DownloadedLicenseFileName)</DownloadedLicenseFile>
27-
<MainPackageFolder>$(PackageAssetsPath)$(MSBuildProjectName)-$(Runtime)</MainPackageFolder>
27+
<MainPackageFolder>$(PackagePreparationPath)$(MSBuildProjectName)-$(Runtime)</MainPackageFolder>
2828
<CopyToPackageSemaphore>$(MainPackageFolder)\.copied</CopyToPackageSemaphore>
2929

3030
</PropertyGroup>
@@ -74,9 +74,9 @@
7474
<!-- This vast file needs a special magic, we split it into multiple chunks in different packages. -->
7575
<!-- The chunks actually all have the same size but have different sections zero'd out. We "or" the bits back together on package restore. -->
7676
<File Include= "libtorch\lib\libtorch_cuda.so" PackageSuffix="part2-primary" FileFraggerIndex="0" FileFraggerStart="0" FileFraggerSize="300000000" />
77-
<File Include= "libtorch\lib\libtorch_cuda.so" PackageSuffix="part2-fragment1" FileFraggerIndex="1" FileFraggerStart="300000001" FileFraggerSize="300000000" />
78-
<File Include= "libtorch\lib\libtorch_cuda.so" PackageSuffix="part2-fragment2" FileFraggerIndex="2" FileFraggerStart="600000001" FileFraggerSize="300000000" />
79-
<File Include= "libtorch\lib\libtorch_cuda.so" PackageSuffix="part2-fragment3" FileFraggerIndex="3" FileFraggerStart="600000001" FileFraggerSize="2000000000" />
77+
<File Include= "libtorch\lib\libtorch_cuda.so" PackageSuffix="part2-fragment1" FileFraggerIndex="1" FileFraggerStart="300000000" FileFraggerSize="300000000" />
78+
<File Include= "libtorch\lib\libtorch_cuda.so" PackageSuffix="part2-fragment2" FileFraggerIndex="2" FileFraggerStart="600000000" FileFraggerSize="300000000" />
79+
<File Include= "libtorch\lib\libtorch_cuda.so" PackageSuffix="part2-fragment3" FileFraggerIndex="3" FileFraggerStart="900000000" FileFraggerSize="2000000000" />
8080

8181
<File Include= "libtorch\lib\libtorch_global_deps.so" PackageSuffix="part1" />
8282
<File Include= "libtorch\lib\libtorch_python.so" PackageSuffix="part1" />
@@ -145,6 +145,7 @@
145145

146146
</Target>
147147

148+
<!-- Hack vast files into separate pieces so we can place them in separate nuget packages. -->
148149
<UsingTask
149150
TaskName="FileFragger"
150151
TaskFactory="RoslynCodeTaskFactory"
@@ -162,18 +163,15 @@
162163
<Code Type="Fragment" Language="cs">
163164
<![CDATA[
164165
string name = Path.GetFileName(SourceFile) + ((Start == 0) ? "" : ".fragment" + Index);
165-
byte[] bytes = File.ReadAllBytes(SourceFile);
166+
byte[] srcBytes = File.ReadAllBytes(SourceFile);
166167
if (Start != 0)
167168
DestinationFolder = DestinationFolder.Replace("runtimes","fragments");
168169
Directory.CreateDirectory(DestinationFolder);
169170
string outfile = Path.Combine(DestinationFolder, name);
170-
int lengthToKeep = System.Math.Max(System.Math.Min(Size, bytes.Length - Start), 0);
171-
// clear everything but the part we're keeping, the defragger will bitwise-or the file together
172-
// the zero-part will compress well
173-
if (Start > 0) Array.Clear(bytes, 0, Start - 1);
174-
int lengthAfter = bytes.Length - (Start + lengthToKeep);
175-
if (lengthAfter > 0) Array.Clear(bytes, Start+lengthToKeep, lengthAfter);
176-
File.WriteAllBytes(outfile, bytes);
171+
int lengthToKeep = System.Math.Max(System.Math.Min(Size, srcBytes.Length - Start), 0);
172+
var outBytes = new byte[lengthToKeep];
173+
Array.Copy(srcBytes, Start, outBytes, 0, lengthToKeep);
174+
File.WriteAllBytes(outfile, outBytes);
177175
]]>
178176
</Code>
179177
</Task>

0 commit comments

Comments
 (0)