Skip to content

Commit b37490c

Browse files
jpobstjpobst
committed
[XABT] Move marshal method generation to a "linker step".
1 parent 0520997 commit b37490c

18 files changed

+734
-139
lines changed

‎src/Xamarin.Android.Build.Tasks/Linker/MonoDroid.Tuner/RewriteMarshalMethodsStep.cs

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
using System;
2+
using System.Collections.Concurrent;
3+
using System.Collections.Generic;
4+
using System.Linq;
5+
using Java.Interop.Tools.JavaCallableWrappers;
6+
using Microsoft.Android.Build.Tasks;
7+
using Microsoft.Build.Utilities;
8+
using Mono.Cecil;
9+
using Mono.Linker;
10+
using Mono.Linker.Steps;
11+
using Xamarin.Android.Tasks;
12+
using Xamarin.Android.Tools;
13+
14+
namespace MonoDroid.Tuner;
15+
16+
/// <summary>
17+
/// Scans an assembly for marshal methods and converts them to LLVM marshal methods.
18+
/// </summary>
19+
public class RewriteMarshalMethodsStep : BaseStep, IAssemblyModifierPipelineStep
20+
{
21+
public TaskLoggingHelper Log { get; set; }
22+
23+
bool? brokenExceptionTransitionsEnabled;
24+
25+
public RewriteMarshalMethodsStep (TaskLoggingHelper log)
26+
{
27+
Log = log;
28+
}
29+
30+
public void ProcessAssembly (AssemblyDefinition assembly, StepContext context)
31+
{
32+
if (!context.IsAndroidAssembly)
33+
return;
34+
35+
var action = Annotations.HasAction (assembly) ? Annotations.GetAction (assembly) : AssemblyAction.Skip;
36+
37+
if (action == AssemblyAction.Delete)
38+
return;
39+
40+
// We need to parse the environment files supplied by the user to see if they want to use broken exception transitions. This information is needed
41+
// in order to properly generate wrapper methods in the marshal methods assembly rewriter.
42+
// We don't care about those generated by us, since they won't contain the `XA_BROKEN_EXCEPTION_TRANSITIONS` variable we look for.
43+
if (!brokenExceptionTransitionsEnabled.HasValue) {
44+
var environmentParser = new EnvironmentFilesParser ();
45+
brokenExceptionTransitionsEnabled = environmentParser.AreBrokenExceptionTransitionsEnabled (context.Environments);
46+
}
47+
48+
var collection = MarshalMethodsCollection.FromAssembly (context.Architecture, assembly, Context.Resolver, Log);
49+
50+
var state = new NativeCodeGenState (context.Architecture, Context, Context.Resolver, [], [], collection);
51+
Run (state, context);
52+
53+
var stateObject = MarshalMethodCecilAdapter.CreateNativeCodeGenState (context.Architecture, state);
54+
var destinationMarshalMethodsXml = MarshalMethodsXmlFile.GetMarshalMethodsXmlFilePath (context.Destination.ItemSpec);
55+
56+
MarshalMethodsXmlFile.Export (destinationMarshalMethodsXml, context.Architecture, stateObject, Log);
57+
58+
// TODO: Only return true if we actually modified the assembly
59+
context.IsAssemblyModified = true;
60+
}
61+
62+
void Run (NativeCodeGenState state, StepContext context)
63+
{
64+
if (state.Classifier is null) {
65+
Log.LogError ("state.Classifier cannot be null if marshal methods are enabled");
66+
return;
67+
}
68+
69+
if (!context.EnableManagedMarshalMethodsLookup) {
70+
RewriteMethods (state, brokenExceptionTransitionsEnabled.GetValueOrDefault ());
71+
state.Classifier.AddSpecialCaseMethods ();
72+
} else {
73+
// We need to run `AddSpecialCaseMethods` before `RewriteMarshalMethods` so that we can see the special case
74+
// methods (such as TypeManager.n_Activate_mm) when generating the managed lookup tables.
75+
state.Classifier.AddSpecialCaseMethods ();
76+
state.ManagedMarshalMethodsLookupInfo = new ManagedMarshalMethodsLookupInfo (Log);
77+
RewriteMethods (state, brokenExceptionTransitionsEnabled.GetValueOrDefault ());
78+
}
79+
80+
Log.LogDebugMessage ($"[{state.TargetArch}] Number of generated marshal methods: {state.Classifier.MarshalMethods.Count}");
81+
if (state.Classifier.DynamicallyRegisteredMarshalMethods.Count > 0) {
82+
Log.LogWarning ($"[{state.TargetArch}] Number of methods in the project that will be registered dynamically: {state.Classifier.DynamicallyRegisteredMarshalMethods.Count}");
83+
}
84+
85+
var wrappedCount = state.Classifier.MarshalMethods.Sum (m => m.Value.Count (m2 => m2.NeedsBlittableWorkaround));
86+
87+
if (wrappedCount > 0) {
88+
// TODO: change to LogWarning once the generator can output code which requires no non-blittable wrappers
89+
Log.LogDebugMessage ($"[{state.TargetArch}] Number of methods in the project that need marshal method wrappers: {wrappedCount}");
90+
}
91+
}
92+
93+
void RewriteMethods (NativeCodeGenState state, bool brokenExceptionTransitionsEnabled)
94+
{
95+
if (state.Classifier == null) {
96+
return;
97+
}
98+
99+
var rewriter = new MarshalMethodsAssemblyRewriter (Log, state.TargetArch, state.Classifier, state.Resolver, state.ManagedMarshalMethodsLookupInfo);
100+
rewriter.Rewrite (brokenExceptionTransitionsEnabled);
101+
}
102+
}

‎src/Xamarin.Android.Build.Tasks/Tasks/AssemblyModifierPipeline.cs

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ public class AssemblyModifierPipeline : AndroidTask
2424
{
2525
public override string TaskPrefix => "AMP";
2626

27+
public string AndroidSdkPlatform { get; set; } = "";
28+
2729
public string ApplicationJavaClass { get; set; } = "";
2830

2931
public string CodeGenerationTarget { get; set; } = "";
@@ -37,8 +39,15 @@ public class AssemblyModifierPipeline : AndroidTask
3739

3840
public bool EnableMarshalMethods { get; set; }
3941

42+
public bool EnableManagedMarshalMethodsLookup { get; set; }
43+
44+
public ITaskItem [] Environments { get; set; } = [];
45+
4046
public bool ErrorOnCustomJavaObject { get; set; }
4147

48+
// If we're using ILLink, this process modifies the linked assemblies in place
49+
protected virtual bool ModifiesAssembliesInPlace => true;
50+
4251
public string? PackageNamingPolicy { get; set; }
4352

4453
/// <summary>
@@ -64,7 +73,7 @@ public class AssemblyModifierPipeline : AndroidTask
6473
[Required]
6574
public string TargetName { get; set; } = "";
6675

67-
protected JavaPeerStyle codeGenerationTarget;
76+
JavaPeerStyle codeGenerationTarget;
6877

6978
public override bool RunTask ()
7079
{
@@ -76,6 +85,8 @@ public override bool RunTask ()
7685

7786
var readerParameters = new ReaderParameters {
7887
ReadSymbols = ReadSymbols,
88+
ReadWrite = ModifiesAssembliesInPlace || EnableMarshalMethods,
89+
InMemory = ModifiesAssembliesInPlace || EnableMarshalMethods,
7990
};
8091

8192
Dictionary<AndroidTargetArch, Dictionary<string, ITaskItem>> perArchAssemblies = MonoAndroidHelper.GetPerArchAssemblies (ResolvedAssemblies, Array.Empty<string> (), validate: false);
@@ -118,7 +129,7 @@ public override bool RunTask ()
118129

119130
Directory.CreateDirectory (Path.GetDirectoryName (destination.ItemSpec));
120131

121-
RunPipeline (pipeline!, source, destination);
132+
RunPipeline (pipeline!, source, destination, perArchAssemblies [sourceArch].Values.ToArray ());
122133
}
123134

124135
pipeline?.Dispose ();
@@ -138,6 +149,13 @@ protected virtual void BuildPipeline (AssemblyPipeline pipeline, MSBuildLinkCont
138149
findJavaObjectsStep.Initialize (context);
139150
pipeline.Steps.Add (findJavaObjectsStep);
140151

152+
// RewriteMarshalMethodsStep
153+
if (EnableMarshalMethods && !Debug) {
154+
var rewriteMarshalMethodsStep = new RewriteMarshalMethodsStep (Log);
155+
rewriteMarshalMethodsStep.Initialize (context);
156+
pipeline.Steps.Add (rewriteMarshalMethodsStep);
157+
}
158+
141159
// SaveChangedAssemblyStep
142160
var writerParameters = new WriterParameters {
143161
DeterministicMvid = Deterministic,
@@ -156,13 +174,15 @@ protected virtual void BuildPipeline (AssemblyPipeline pipeline, MSBuildLinkCont
156174
pipeline.Steps.Add (findTypeMapObjectsStep);
157175
}
158176

159-
void RunPipeline (AssemblyPipeline pipeline, ITaskItem source, ITaskItem destination)
177+
void RunPipeline (AssemblyPipeline pipeline, ITaskItem source, ITaskItem destination, ITaskItem [] archAssemblies)
160178
{
161179
var assembly = pipeline.Resolver.GetAssembly (source.ItemSpec);
162180

163-
var context = new StepContext (source, destination) {
181+
var context = new StepContext (source, destination, AndroidSdkPlatform, Environments, archAssemblies) {
182+
Architecture = MonoAndroidHelper.GetRequiredValidArchitecture (source),
164183
CodeGenerationTarget = codeGenerationTarget,
165184
EnableMarshalMethods = EnableMarshalMethods,
185+
EnableManagedMarshalMethodsLookup = EnableManagedMarshalMethodsLookup,
166186
IsAndroidAssembly = MonoAndroidHelper.IsAndroidAssembly (source),
167187
IsDebug = Debug,
168188
IsFrameworkAssembly = MonoAndroidHelper.IsFrameworkAssembly (source),

‎src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs

Lines changed: 109 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,7 @@ void Run (bool useMarshalMethods)
155155

156156
// Now that "never" never happened, we can proceed knowing that at least the assembly sets are the same for each architecture
157157
var nativeCodeGenStates = new ConcurrentDictionary<AndroidTargetArch, NativeCodeGenState> ();
158+
var nativeCodeGenStateObjects = new ConcurrentDictionary<AndroidTargetArch, NativeCodeGenStateObject> ();
158159
NativeCodeGenState? templateCodeGenState = null;
159160

160161
var firstArch = allAssembliesPerArch.First ().Key;
@@ -169,7 +170,7 @@ void Run (bool useMarshalMethods)
169170
// Pick the "first" one as the one to generate Java code for
170171
var generateJavaCode = arch == firstArch;
171172

172-
(bool success, NativeCodeGenState? state) = GenerateJavaSourcesAndMaybeClassifyMarshalMethods (arch, archAssemblies, MaybeGetArchAssemblies (userAssembliesPerArch, arch), useMarshalMethods, generateJavaCode);
173+
(bool success, NativeCodeGenState? state, NativeCodeGenStateObject? stateObject) = GenerateJavaSourcesAndMaybeClassifyMarshalMethods (arch, archAssemblies, MaybeGetArchAssemblies (userAssembliesPerArch, arch), useMarshalMethods, generateJavaCode);
173174

174175
if (!success) {
175176
generateSucceeded = false;
@@ -181,6 +182,7 @@ void Run (bool useMarshalMethods)
181182
}
182183

183184
nativeCodeGenStates.TryAdd (arch, state);
185+
nativeCodeGenStateObjects.TryAdd (arch, stateObject);
184186
});
185187

186188
// If we hit an error generating the Java code, we should bail out now
@@ -198,6 +200,21 @@ void Run (bool useMarshalMethods)
198200
// Save NativeCodeGenState for later tasks
199201
Log.LogDebugMessage ($"Saving {nameof (NativeCodeGenState)} to {nameof (NativeCodeGenStateRegisterTaskKey)}");
200202
BuildEngine4.RegisterTaskObjectAssemblyLocal (MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (NativeCodeGenStateRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory), nativeCodeGenStates, RegisteredTaskObjectLifetime.Build);
203+
204+
// If we still need the NativeCodeGenState in the <GenerateNativeMarshalMethodSources> task because we're using marshal methods,
205+
// we're going to transfer it to a new object that doesn't require holding open Cecil AssemblyDefinitions.
206+
if (useMarshalMethods) {
207+
//var nativeCodeGenStateObject = MarshalMethodCecilAdapter.GetNativeCodeGenStateCollection (Log, nativeCodeGenStates);
208+
var nativeCodeGenStateCollection = new NativeCodeGenStateCollection ();
209+
210+
foreach (var kvp in nativeCodeGenStateObjects) {
211+
nativeCodeGenStateCollection.States.Add (kvp.Key, kvp.Value);
212+
Log.LogDebugMessage ($"Added NativeCodeGenStateObject for arch: {kvp.Key}, containing {kvp.Value.MarshalMethods.Count} marshal methods");
213+
}
214+
215+
Log.LogDebugMessage ($"Saving {nameof (NativeCodeGenStateObject)} to {nameof (GenerateJavaStubs.NativeCodeGenStateObjectRegisterTaskKey)}");
216+
BuildEngine4.RegisterTaskObjectAssemblyLocal (MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateObjectRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory), nativeCodeGenStateCollection, RegisteredTaskObjectLifetime.Build);
217+
}
201218
}
202219

203220
internal static Dictionary<string, ITaskItem> MaybeGetArchAssemblies (Dictionary<AndroidTargetArch, Dictionary<string, ITaskItem>> dict, AndroidTargetArch arch)
@@ -209,7 +226,7 @@ internal static Dictionary<string, ITaskItem> MaybeGetArchAssemblies (Dictionary
209226
return archDict;
210227
}
211228

212-
(bool success, NativeCodeGenState? stubsState) GenerateJavaSourcesAndMaybeClassifyMarshalMethods (AndroidTargetArch arch, Dictionary<string, ITaskItem> assemblies, Dictionary<string, ITaskItem> userAssemblies, bool useMarshalMethods, bool generateJavaCode)
229+
(bool success, NativeCodeGenState? stubsState, NativeCodeGenStateObject? stateObject) GenerateJavaSourcesAndMaybeClassifyMarshalMethods (AndroidTargetArch arch, Dictionary<string, ITaskItem> assemblies, Dictionary<string, ITaskItem> userAssemblies, bool useMarshalMethods, bool generateJavaCode)
213230
{
214231
XAAssemblyResolver resolver = MakeResolver (useMarshalMethods, arch, assemblies);
215232
var tdCache = new TypeDefinitionCache ();
@@ -227,15 +244,62 @@ internal static Dictionary<string, ITaskItem> MaybeGetArchAssemblies (Dictionary
227244
}
228245

229246
if (!success) {
230-
return (false, null);
247+
return (false, null, null);
231248
}
232249

233250
MarshalMethodsCollection? marshalMethodsCollection = null;
251+
NativeCodeGenStateObject? stateObject = null;
252+
253+
if (useMarshalMethods) {
254+
stateObject = new NativeCodeGenStateObject ();
255+
256+
foreach (var assembly in assemblies.Values) {
257+
var marshalMethodXmlFile = MarshalMethodsXmlFile.GetMarshalMethodsXmlFilePath (assembly.ItemSpec);
258+
259+
if (!File.Exists (marshalMethodXmlFile))
260+
continue;
261+
262+
var xml = MarshalMethodsXmlFile.Import (marshalMethodXmlFile);
263+
264+
if (xml is null || xml?.Value.MarshalMethods.Count == 0) {
265+
Log.LogDebugMessage ($"'{marshalMethodXmlFile}' is empty, skipping.");
266+
continue;
267+
}
268+
269+
foreach (var kvp in xml.Value.Value.MarshalMethods) {
270+
if (!stateObject.MarshalMethods.TryGetValue (kvp.Key, out var methods)) {
271+
methods = new List<MarshalMethodEntryObject> ();
272+
stateObject.MarshalMethods.Add (kvp.Key, methods);
273+
}
274+
275+
foreach (var method in kvp.Value) {
276+
// We don't need to add the special case method multiple times
277+
if (methods.Count > 0 && method.IsSpecial)
278+
continue;
279+
methods.Add (method);
280+
}
281+
}
282+
}
283+
284+
//marshalMethodsCollection = MarshalMethodsCollection.FromAssemblies (arch, assemblies.Values.ToList (), resolver, Log);
285+
//marshalMethodsCollection.AddSpecialCaseMethods ();
286+
287+
marshalMethodsCollection = new EmptyMarshalMethodsCollection ();
288+
}
289+
290+
var state = new NativeCodeGenState (arch, tdCache, resolver, allJavaTypes, javaTypesForJCW, marshalMethodsCollection);
234291

235-
if (useMarshalMethods)
236-
marshalMethodsCollection = MarshalMethodsCollection.FromAssemblies (arch, assemblies.Values.ToList (), resolver, Log);
292+
//if (useMarshalMethods) {
293+
// var info = new ManagedMarshalMethodsLookupInfo (Log);
237294

238-
return (true, new NativeCodeGenState (arch, tdCache, resolver, allJavaTypes, javaTypesForJCW, marshalMethodsCollection));
295+
// foreach (var kvp in marshalMethodsCollection.MarshalMethods)
296+
// foreach (var method in kvp.Value)
297+
// info.AddNativeCallbackWrapper (method.NativeCallback);
298+
299+
// state.ManagedMarshalMethodsLookupInfo = info;
300+
//}
301+
302+
return (true, state, stateObject);
239303
}
240304

241305
(List<TypeDefinition> allJavaTypes, List<TypeDefinition> javaTypesForJCW) ScanForJavaTypes (XAAssemblyResolver res, TypeDefinitionCache cache, Dictionary<string, ITaskItem> assemblies, Dictionary<string, ITaskItem> userAssemblies, bool useMarshalMethods)
@@ -343,4 +407,43 @@ void CompareLists (string name, List<string> list1, List<string> list2)
343407
Log.LogDebugMessage ($"No differences");
344408
}
345409
}
410+
411+
class EmptyMarshalMethodsCollection : MarshalMethodsCollection
412+
{
413+
public override HashSet<AssemblyDefinition> AssembliesWithMarshalMethods => throw new NotSupportedException ();
414+
415+
/// <summary>
416+
/// Marshal methods that have already been rewritten as LLVM marshal methods.
417+
/// </summary>
418+
public override IDictionary<string, IList<ConvertedMarshalMethodEntry>> ConvertedMarshalMethods => throw new NotSupportedException ();
419+
420+
/// <summary>
421+
/// Marshal methods that cannot be rewritten and must be registered dynamically.
422+
/// </summary>
423+
public override List<DynamicallyRegisteredMarshalMethodEntry> DynamicallyRegisteredMarshalMethods => throw new NotSupportedException ();
424+
425+
/// <summary>
426+
/// Marshal methods that can be rewritten as LLVM marshal methods.
427+
/// </summary>
428+
public override IDictionary<string, IList<MarshalMethodEntry>> MarshalMethods => throw new NotSupportedException ();
429+
430+
public EmptyMarshalMethodsCollection ()
431+
{
432+
}
433+
434+
public override void AddSpecialCaseMethods ()
435+
{
436+
throw new NotImplementedException ();
437+
}
438+
439+
public override bool ShouldBeDynamicallyRegistered (TypeDefinition topType, MethodDefinition registeredMethod, MethodDefinition implementedMethod, CustomAttribute registerAttribute)
440+
{
441+
throw new NotImplementedException ();
442+
}
443+
444+
public override bool TypeHasDynamicallyRegisteredMethods (TypeDefinition type)
445+
{
446+
throw new NotImplementedException ();
447+
}
448+
}
346449
}

‎src/Xamarin.Android.Build.Tasks/Tasks/GenerateMainAndroidManifest.cs

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ public class GenerateMainAndroidManifest : AndroidTask
2828
public string CodeGenerationTarget { get; set; } = "";
2929
public bool Debug { get; set; }
3030
public bool EmbedAssemblies { get; set; }
31-
public bool EnableMarshalMethods { get; set; }
3231
[Required]
3332
public string IntermediateOutputDirectory { get; set; } = "";
3433
public string []? ManifestPlaceholders { get; set; }
@@ -52,8 +51,6 @@ public class GenerateMainAndroidManifest : AndroidTask
5251
AndroidRuntime androidRuntime;
5352
JavaPeerStyle codeGenerationTarget;
5453

55-
bool UseMarshalMethods => !Debug && EnableMarshalMethods;
56-
5754
public override bool RunTask ()
5855
{
5956
// Retrieve the stored NativeCodeGenState (and remove it from the cache)
@@ -74,16 +71,6 @@ public override bool RunTask ()
7471
var additionalProviders = MergeManifest (templateCodeGenState, GenerateJavaStubs.MaybeGetArchAssemblies (userAssembliesPerArch, templateCodeGenState.TargetArch));
7572
GenerateAdditionalProviderSources (templateCodeGenState, additionalProviders);
7673

77-
78-
// If we still need the NativeCodeGenState in the <GenerateNativeMarshalMethodSources> task because we're using marshal methods,
79-
// we're going to transfer it to a new object that doesn't require holding open Cecil AssemblyDefinitions.
80-
if (UseMarshalMethods) {
81-
var nativeCodeGenStateObject = MarshalMethodCecilAdapter.GetNativeCodeGenStateCollection (Log, nativeCodeGenStates);
82-
83-
Log.LogDebugMessage ($"Saving {nameof (NativeCodeGenStateObject)} to {nameof (GenerateJavaStubs.NativeCodeGenStateObjectRegisterTaskKey)}");
84-
BuildEngine4.RegisterTaskObjectAssemblyLocal (MonoAndroidHelper.GetProjectBuildSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateObjectRegisterTaskKey, WorkingDirectory, IntermediateOutputDirectory), nativeCodeGenStateObject, RegisteredTaskObjectLifetime.Build);
85-
}
86-
8774
// Dispose the Cecil resolvers so the assemblies are closed.
8875
Log.LogDebugMessage ($"Disposing all {nameof (NativeCodeGenState)}.{nameof (NativeCodeGenState.Resolver)}");
8976

0 commit comments

Comments
 (0)