Skip to content

Commit 55445fd

Browse files
committed
finalise package splitting
1 parent 8dd0e7c commit 55445fd

File tree

21 files changed

+102
-211
lines changed

21 files changed

+102
-211
lines changed

pkg/Directory.Build.props

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
<Import Project="..\Directory.Build.props" />
33

44
<PropertyGroup>
5-
<Authors>PyTorch and TorchSharp contributors</Authors>
5+
<Authors>PyTorch contributors;TorchSharp contributors</Authors>
66
<PackageLicenseFile>THIRD-PARTY-NOTICES.TXT</PackageLicenseFile>
77
<PackageProjectUrl>https://github.com/xamarin/TorchSharp</PackageProjectUrl>
88
<!-- space separated -->
@@ -36,6 +36,7 @@
3636
<Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\lib\**\*" Pack="true" PackagePath="lib" />
3737
<Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\analyzers\**\*" Pack="true" PackagePath="analyzers" />
3838
<Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\runtimes\**\*" Pack="true" PackagePath="runtimes" />
39+
<Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\fragments\**\*" Pack="true" PackagePath="fragments" />
3940
<Content Include="$(PackageAssetsPath)$(PackageIdFolderName)\tools\**\*" Pack="true" PackagePath="tools" />
4041
</ItemGroup>
4142

pkg/TorchSharp/TorchSharp.nupkgproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
</ItemGroup>
1111

1212
<ItemGroup>
13-
<Content Include="..\common\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
13+
<Content Include="..\common\CommonPackage.props" Pack="true" PackagePath="buildTransitive\netstandard2.0\$(MSBuildProjectName).props" />
1414
</ItemGroup>
1515

1616
</Project>

pkg/common/CommonPackage.props

+4-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
22

33
<ItemGroup>
4+
<FraggerPrimaryPackage Include="$(MSBuildThisFileDirectory)..\..\runtimes\"
5+
Condition="'$(MSBuildProjectName.IndexOf(`-primary`)' != '-1'"/>
46

5-
<Content Include="$(MSBuildThisFileDirectory)\..\..\runtimes\win-x64\native\*.dll"
7+
<!-- <Content Include="$(MSBuildThisFileDirectory)\..\..\runtimes\win-x64\native\*.dll"
68
Condition="'$(PlatformTarget)' == 'x64' AND '$(OS)' == 'Windows_NT'">
79
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
810
<Visible>false</Visible>
@@ -22,7 +24,7 @@
2224
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
2325
<Visible>false</Visible>
2426
<Link>%(Filename)%(Extension)</Link>
25-
</Content>
27+
</Content> -->
2628

2729
</ItemGroup>
2830

pkg/common/CommonPackage.targets

+64-61
Original file line numberDiff line numberDiff line change
@@ -7,78 +7,81 @@
77
TaskFactory="RoslynCodeTaskFactory"
88
AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.Core.dll" >
99
<ParameterGroup>
10-
<FraggerDirectory ParameterType="System.String" Required="true" />
10+
<FraggerPrimaryPackage ParameterType="System.String" Required="true" />
1111
</ParameterGroup>
1212
<Task>
1313
<Using Namespace="System"/>
1414
<Using Namespace="System.IO"/>
1515
<Code Type="Fragment" Language="cs">
1616
<![CDATA[
17-
if (Directory.Exists(FraggerDirectory))
17+
if (Directory.Exists(FraggerPrimaryPackage))
1818
{
19-
System.Console.WriteLine("Searching for primary files in {0}", FraggerDirectory);
20-
foreach (var p in Directory.EnumerateFiles(FraggerDirectory, "*.primary", SearchOption.AllDirectories))
19+
//System.Console.WriteLine("Searching for primary files in {0}", FraggerPrimaryPackage);
20+
foreach (var p in Directory.EnumerateFiles(FraggerPrimaryPackage, "*", SearchOption.AllDirectories))
2121
{
2222
2323
var primaryFile = Path.GetFullPath(p);
24-
Console.WriteLine("Found primary file at {0}", primaryFile);
25-
var restoredFile = primaryFile.Replace(".primary", "");
26-
27-
// Find the fragments in the parallel nuget package
28-
var fragmentFile1 = primaryFile.Replace("-primary", "-fragment1").Replace(".primary", ".fragment1");
29-
var fragmentFile2 = primaryFile.Replace("-primary", "-fragment2").Replace(".primary", ".fragment2");
30-
var fragmentFile3 = primaryFile.Replace("-primary", "-fragment3").Replace(".primary", ".fragment3");
31-
32-
if (!File.Exists(fragmentFile1))
33-
throw (new Exception(String.Format("Couldn't find parallel fragment file for multi-package deliver of single massive binary file. Primary file {0} found, expected corresponding fragment at {1}. You may be missing a nuget package reference.", primaryFile, fragmentFile1)));
34-
35-
if (!File.Exists(fragmentFile2))
36-
throw (new Exception(String.Format("Couldn't find parallel fragment file for multi-package deliver of single massive binary file. Primary file {0} found, expected corresponding fragment at {1}. You may be missing a nuget package reference.", primaryFile, fragmentFile2)));
37-
38-
if (!File.Exists(fragmentFile3))
39-
throw (new Exception(String.Format("Couldn't find parallel fragment file for multi-package deliver of single massive binary file. Primary file {0} found, expected corresponding fragment at {1}. You may be missing a nuget package reference.", primaryFile, fragmentFile3)));
40-
41-
Console.WriteLine("Found fragment file at {0}", fragmentFile1);
42-
Console.WriteLine("Found fragment file at {0}", fragmentFile2);
43-
Console.WriteLine("Found fragment file at {0}", fragmentFile3);
44-
var primaryBytes = File.ReadAllBytes(primaryFile);
45-
var fragmentBytes1 = File.ReadAllBytes(fragmentFile1);
46-
var fragmentBytes2 = File.ReadAllBytes(fragmentFile2);
47-
var fragmentBytes3 = File.ReadAllBytes(fragmentFile3);
48-
49-
if (primaryBytes.Length != fragmentBytes1.Length)
50-
throw (new Exception(String.Format("Mismatched file sizes for multi-package deliver of single massive binary file. Primary file {0} found, but the corresponding fragment at {1} has a different size. You may have inconsistent nuget package references.", primaryFile, fragmentFile1)));
51-
52-
if (primaryBytes.Length != fragmentBytes2.Length)
53-
throw (new Exception(String.Format("Mismatched file sizes for multi-package deliver of single massive binary file. Primary file {0} found, but the corresponding fragment at {1} has a different size. You may have inconsistent nuget package references.", primaryFile, fragmentFile2)));
54-
55-
if (primaryBytes.Length != fragmentBytes3.Length)
56-
throw (new Exception(String.Format("Mismatched file sizes for multi-package deliver of single massive binary file. Primary file {0} found, but the corresponding fragment at {1} has a different size. You may have inconsistent nuget package references.", primaryFile, fragmentFile3)));
57-
58-
for (int i = 0; i < primaryBytes.Length; i++)
24+
//Console.WriteLine("Found primary file at {0}", primaryFile);
25+
26+
// See if there are fragments in the parallel nuget packages. If the primary is
27+
// some-package-primary\runtimes\....\a.so
28+
// then the expected fragments are
29+
// some-package-fragment1\fragments\....\a.so
30+
// some-package-fragment2\fragments\....\a.so
31+
// some-package-fragment3\fragments\....\a.so
32+
var fragmentFile1 = primaryFile.Replace("-primary", "-fragment1").Replace("runtimes", "fragments") + ".fragment1";
33+
var fragmentFile2 = primaryFile.Replace("-primary", "-fragment2").Replace("runtimes", "fragments") + ".fragment2";
34+
var fragmentFile3 = primaryFile.Replace("-primary", "-fragment3").Replace("runtimes", "fragments") + ".fragment3";
35+
36+
if (File.Exists(fragmentFile1) && File.Exists(fragmentFile2) && File.Exists(fragmentFile3))
5937
{
60-
if ((primaryBytes[i] > 0) && (fragmentBytes1[i] > 0) && (primaryBytes[i] != fragmentBytes1[i]))
61-
throw (new Exception(String.Format("Inconsistent file contents for multi-package deliver of single massive binary file. Primary file {0} and corresponding fragment {1} had inconsistent contents. You may have inconsistent nuget package references.", primaryFile, fragmentFile1)));
62-
if ((primaryBytes[i] > 0) && (fragmentBytes2[i] > 0) && (primaryBytes[i] != fragmentBytes2[i]))
63-
throw (new Exception(String.Format("Inconsistent file contents for multi-package deliver of single massive binary file. Primary file {0} and corresponding fragment {1} had inconsistent contents. You may have inconsistent nuget package references.", primaryFile, fragmentFile2)));
64-
if ((primaryBytes[i] > 0) && (fragmentBytes3[i] > 0) && (primaryBytes[i] != fragmentBytes3[i]))
65-
throw (new Exception(String.Format("Inconsistent file contents for multi-package deliver of single massive binary file. Primary file {0} and corresponding fragment {1} had inconsistent contents. You may have inconsistent nuget package references.", primaryFile, fragmentFile3)));
66-
primaryBytes[i] |= fragmentBytes1[i];
67-
primaryBytes[i] |= fragmentBytes2[i];
68-
primaryBytes[i] |= fragmentBytes3[i];
38+
System.Console.WriteLine("Fragment files in parallel nuget packages for {0}", primaryFile);
39+
40+
Console.WriteLine("Found fragment file at {0}", fragmentFile1);
41+
Console.WriteLine("Found fragment file at {0}", fragmentFile2);
42+
Console.WriteLine("Found fragment file at {0}", fragmentFile3);
43+
var primaryBytes = File.ReadAllBytes(primaryFile);
44+
var fragmentBytes1 = File.ReadAllBytes(fragmentFile1);
45+
var fragmentBytes2 = File.ReadAllBytes(fragmentFile2);
46+
var fragmentBytes3 = File.ReadAllBytes(fragmentFile3);
47+
48+
if (primaryBytes.Length != fragmentBytes1.Length)
49+
throw (new Exception(String.Format("Mismatched file sizes for multi-package deliver of single massive binary file. Primary file {0} found, but the corresponding fragment at {1} has a different size. You may have inconsistent nuget package references.", primaryFile, fragmentFile1)));
50+
51+
if (primaryBytes.Length != fragmentBytes2.Length)
52+
throw (new Exception(String.Format("Mismatched file sizes for multi-package deliver of single massive binary file. Primary file {0} found, but the corresponding fragment at {1} has a different size. You may have inconsistent nuget package references.", primaryFile, fragmentFile2)));
53+
54+
if (primaryBytes.Length != fragmentBytes3.Length)
55+
throw (new Exception(String.Format("Mismatched file sizes for multi-package deliver of single massive binary file. Primary file {0} found, but the corresponding fragment at {1} has a different size. You may have inconsistent nuget package references.", primaryFile, fragmentFile3)));
56+
57+
for (int i = 0; i < primaryBytes.Length; i++)
58+
{
59+
if ((primaryBytes[i] > 0) && (fragmentBytes1[i] > 0) && (primaryBytes[i] != fragmentBytes1[i]))
60+
throw (new Exception(String.Format("Inconsistent file contents for multi-package deliver of single massive binary file. Primary file {0} and corresponding fragment {1} had inconsistent contents. You may have inconsistent nuget package references.", primaryFile, fragmentFile1)));
61+
if ((primaryBytes[i] > 0) && (fragmentBytes2[i] > 0) && (primaryBytes[i] != fragmentBytes2[i]))
62+
throw (new Exception(String.Format("Inconsistent file contents for multi-package deliver of single massive binary file. Primary file {0} and corresponding fragment {1} had inconsistent contents. You may have inconsistent nuget package references.", primaryFile, fragmentFile2)));
63+
if ((primaryBytes[i] > 0) && (fragmentBytes3[i] > 0) && (primaryBytes[i] != fragmentBytes3[i]))
64+
throw (new Exception(String.Format("Inconsistent file contents for multi-package deliver of single massive binary file. Primary file {0} and corresponding fragment {1} had inconsistent contents. You may have inconsistent nuget package references.", primaryFile, fragmentFile3)));
65+
primaryBytes[i] |= fragmentBytes1[i];
66+
primaryBytes[i] |= fragmentBytes2[i];
67+
primaryBytes[i] |= fragmentBytes3[i];
68+
}
69+
70+
var tmpFile = Path.GetTempFileName();
71+
Console.WriteLine("Writing restored primary file at {0}", tmpFile);
72+
File.WriteAllBytes(tmpFile, primaryBytes);
73+
Console.WriteLine("Deleting {0}", primaryFile);
74+
File.Delete(primaryFile);
75+
Console.WriteLine("Moving {0} --> {1}", tmpFile, primaryFile);
76+
File.Move(tmpFile, primaryFile);
77+
78+
Console.WriteLine("Deleting {0}", fragmentFile1);
79+
File.Delete(fragmentFile1); // free up space and prevent us doing this again
80+
Console.WriteLine("Deleting {0}", fragmentFile2);
81+
File.Delete(fragmentFile2); // free up space and prevent us doing this again
82+
Console.WriteLine("Deleting {0}", fragmentFile3);
83+
File.Delete(fragmentFile3); // free up space and prevent us doing this again
6984
}
70-
71-
Console.WriteLine("Writing restored file at {0}", restoredFile);
72-
File.WriteAllBytes(restoredFile, primaryBytes);
73-
74-
Console.WriteLine("Deleting {0}", primaryFile);
75-
File.Delete(primaryFile); // free up space and prevent us doing this again
76-
Console.WriteLine("Deleting {0}", fragmentFile1);
77-
File.Delete(fragmentFile1); // free up space and prevent us doing this again
78-
Console.WriteLine("Deleting {0}", fragmentFile2);
79-
File.Delete(fragmentFile2); // free up space and prevent us doing this again
80-
Console.WriteLine("Deleting {0}", fragmentFile3);
81-
File.Delete(fragmentFile3); // free up space and prevent us doing this again
8285
}
8386
}
8487
]]>
@@ -93,7 +96,7 @@
9396
</PropertyGroup>
9497

9598
<Target Name="FileDefragger">
96-
<FileDefragger FraggerDirectory="$(MSBuildThisFileDirectory)\..\..\runtimes\" />
99+
<FileDefragger FraggerPrimaryPackage="%(FraggerPrimaryPackage.Identity)" />
97100
</Target>
98101

99102

pkg/libtorch-cpu/libtorch-cpu.nupkgproj

+2-2
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@
88
</ItemGroup>
99

1010
<ItemGroup>
11-
<Content Include="..\common\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
12-
<Content Include="..\common\CommonPackage.targets" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).targets" />
11+
<Content Include="..\common\CommonPackage.props" Pack="true" PackagePath="buildTransitive\netstandard2.0\$(MSBuildProjectName).props" />
12+
<Content Include="..\common\CommonPackage.targets" Pack="true" PackagePath="buildTransitive\netstandard2.0\$(MSBuildProjectName).targets" />
1313
<Content Include="..\empty.txt" Pack="true" PackagePath="lib\netstandard2.0\_._" />
1414
</ItemGroup>
1515
</Project>

pkg/libtorch-cuda-10.2-linux-x64-part1/libtorch-cuda-10.2-linux-x64-part1.nupkgproj

+2-2
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77
</ItemGroup>
88

99
<ItemGroup>
10-
<Content Include="..\common\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
11-
<Content Include="..\common\CommonPackage.targets" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).targets" />
10+
<Content Include="..\common\CommonPackage.props" Pack="true" PackagePath="buildTransitive\netstandard2.0\$(MSBuildProjectName).props" />
11+
<Content Include="..\common\CommonPackage.targets" Pack="true" PackagePath="buildTransitive\netstandard2.0\$(MSBuildProjectName).targets" />
1212
<Content Include="..\empty.txt" Pack="true" PackagePath="lib\netstandard2.0\_._" />
1313
</ItemGroup>
1414
</Project>

pkg/libtorch-cuda-10.2-linux-x64-part2-fragment1/libtorch-cuda-10.2-linux-x64-part2-fragment1.nupkgproj

+2-2
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77
</ItemGroup>
88

99
<ItemGroup>
10-
<Content Include="..\common\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
11-
<Content Include="..\common\CommonPackage.targets" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).targets" />
10+
<Content Include="..\common\CommonPackage.props" Pack="true" PackagePath="buildTransitive\netstandard2.0\$(MSBuildProjectName).props" />
11+
<Content Include="..\common\CommonPackage.targets" Pack="true" PackagePath="buildTransitive\netstandard2.0\$(MSBuildProjectName).targets" />
1212
<Content Include="..\empty.txt" Pack="true" PackagePath="lib\netstandard2.0\_._" />
1313
</ItemGroup>
1414
</Project>

pkg/libtorch-cuda-10.2-linux-x64-part2-fragment2/libtorch-cuda-10.2-linux-x64-part2-fragment2.nupkgproj

+2-2
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77
</ItemGroup>
88

99
<ItemGroup>
10-
<Content Include="..\common\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
11-
<Content Include="..\common\CommonPackage.targets" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).targets" />
10+
<Content Include="..\common\CommonPackage.props" Pack="true" PackagePath="buildTransitive\netstandard2.0\$(MSBuildProjectName).props" />
11+
<Content Include="..\common\CommonPackage.targets" Pack="true" PackagePath="buildTransitive\netstandard2.0\$(MSBuildProjectName).targets" />
1212
<Content Include="..\empty.txt" Pack="true" PackagePath="lib\netstandard2.0\_._" />
1313
</ItemGroup>
1414
</Project>

pkg/libtorch-cuda-10.2-linux-x64-part2-fragment3/libtorch-cuda-10.2-linux-x64-part2-fragment3.nupkgproj

+2-2
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77
</ItemGroup>
88

99
<ItemGroup>
10-
<Content Include="..\common\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
11-
<Content Include="..\common\CommonPackage.targets" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).targets" />
10+
<Content Include="..\common\CommonPackage.props" Pack="true" PackagePath="buildTransitive\netstandard2.0\$(MSBuildProjectName).props" />
11+
<Content Include="..\common\CommonPackage.targets" Pack="true" PackagePath="buildTransitive\netstandard2.0\$(MSBuildProjectName).targets" />
1212
<Content Include="..\empty.txt" Pack="true" PackagePath="lib\netstandard2.0\_._" />
1313
</ItemGroup>
1414
</Project>

pkg/libtorch-cuda-10.2-linux-x64-part2-primary/libtorch-cuda-10.2-linux-x64-part2-primary.nupkgproj

+2-2
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@
77
</ItemGroup>
88

99
<ItemGroup>
10-
<Content Include="..\common\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
11-
<Content Include="..\common\CommonPackage.targets" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).targets" />
10+
<Content Include="..\common\CommonPackage.props" Pack="true" PackagePath="buildTransitive\netstandard2.0\$(MSBuildProjectName).props" />
11+
<Content Include="..\common\CommonPackage.targets" Pack="true" PackagePath="buildTransitive\netstandard2.0\$(MSBuildProjectName).targets" />
1212
<Content Include="..\empty.txt" Pack="true" PackagePath="lib\netstandard2.0\_._" />
1313
</ItemGroup>
1414
</Project>

pkg/libtorch-cuda-10.2-linux-x64/libtorch-cuda-10.2-linux-x64.nupkgproj

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@
1616
</ItemGroup>
1717

1818
<ItemGroup>
19-
<Content Include="..\common\CommonPackage.props" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).props" />
20-
<Content Include="..\common\CommonPackage.targets" Pack="true" PackagePath="build\netstandard2.0\$(MSBuildProjectName).targets" />
19+
<Content Include="..\common\CommonPackage.props" Pack="true" PackagePath="buildTransitive\netstandard2.0\$(MSBuildProjectName).props" />
20+
<Content Include="..\common\CommonPackage.targets" Pack="true" PackagePath="buildTransitive\netstandard2.0\$(MSBuildProjectName).targets" />
2121
<Content Include="..\empty.txt" Pack="true" PackagePath="lib\netstandard2.0\_._" />
2222
</ItemGroup>
2323
</Project>

0 commit comments

Comments
 (0)