Skip to content

Commit 9a98abb

Browse files
cccclaicccclai
authored
Back out "Back out "[executorch][PR] [Core ML] Implement intermediate tensor logging""
Differential Revision: D61359607 Pull Request resolved: #4739
1 parent aead1d5 commit 9a98abb

33 files changed

+1281
-218
lines changed

‎backends/apple/coreml/CMakeLists.txt

+8
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,11 @@ if(NOT EXECUTORCH_ROOT)
1313
set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
1414
endif()
1515

16+
if(EXECUTORCH_BUILD_SDK)
17+
# protobuf requires frtti
18+
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -frtti" )
19+
endif()
20+
1621
option(COREML_BUILD_EXECUTOR_RUNNER "Build CoreML executor runner." OFF)
1722

1823
# inmemoryfs sources
@@ -59,6 +64,7 @@ set(SDK_SOURCES
5964
runtime/sdk/ETCoreMLModelAnalyzer.mm
6065
runtime/sdk/ETCoreMLModelStructurePath.mm
6166
runtime/sdk/ETCoreMLOperationProfilingInfo.mm
67+
runtime/sdk/ETCoreMLModelDebugInfo.mm
6268
runtime/sdk/ETCoreMLModelDebugger.mm
6369
runtime/sdk/ETCoreMLModelProfiler.mm
6470
runtime/sdk/ETCoreMLPair.mm
@@ -141,6 +147,8 @@ if(EXECUTORCH_BUILD_SDK)
141147
add_subdirectory(
142148
${CMAKE_CURRENT_SOURCE_DIR}/third-party/coremltools/deps/protobuf/cmake
143149
)
150+
151+
target_link_options_shared_lib(libprotobuf-lite)
144152
target_link_libraries(coremldelegate PRIVATE libprotobuf-lite)
145153
endif()
146154

‎backends/apple/coreml/runtime/delegate/ETCoreMLModelManager.mm

+28-44
Original file line numberDiff line numberDiff line change
@@ -5,36 +5,37 @@
55
//
66
// Please refer to the license found in the LICENSE file in the root directory of the source tree.
77

8-
#import <ETCoreMLAsset.h>
9-
#import <ETCoreMLAssetManager.h>
10-
#import <ETCoreMLDefaultModelExecutor.h>
11-
#import <ETCoreMLLogging.h>
12-
#import <ETCoreMLModel.h>
13-
#import <ETCoreMLModelCompiler.h>
14-
#import <ETCoreMLModelExecutor.h>
15-
#import <ETCoreMLModelLoader.h>
16-
#import <ETCoreMLModelManager.h>
17-
#import <ETCoreMLStrings.h>
18-
#import <MLModel_Prewarm.h>
19-
#import <MLMultiArray_Copy.h>
8+
#import "ETCoreMLAsset.h"
9+
#import "ETCoreMLAssetManager.h"
10+
#import "ETCoreMLDefaultModelExecutor.h"
11+
#import "ETCoreMLLogging.h"
12+
#import "ETCoreMLModel.h"
13+
#import "ETCoreMLModelCompiler.h"
14+
#import "ETCoreMLModelExecutor.h"
15+
#import "ETCoreMLModelLoader.h"
16+
#import "ETCoreMLModelManager.h"
17+
#import "ETCoreMLStrings.h"
18+
#import "MLModel_Prewarm.h"
19+
#import "MLMultiArray_Copy.h"
2020
#import <filesystem>
21-
#import <inmemory_filesystem_utils.hpp>
21+
#import "inmemory_filesystem_utils.hpp"
2222
#import <iostream>
2323
#import <memory>
24-
#import <model_metadata.h>
25-
#import <multiarray.h>
26-
#import <objc_array_util.h>
24+
#import "model_metadata.h"
25+
#import "multiarray.h"
26+
#import "objc_array_util.h"
2727
#import <optional>
2828
#import <os/lock.h>
29-
#import <serde_json.h>
29+
#import "serde_json.h"
3030
#import <string>
3131
#import <system_error>
3232
#import <vector>
3333

3434
#if ET_EVENT_TRACER_ENABLED
35-
#import <ETCoreMLModelAnalyzer.h>
36-
#import <ETCoreMLModelStructurePath.h>
37-
#import <objc_safe_cast.h>
35+
#import "ETCoreMLModelAnalyzer.h"
36+
#import "ETCoreMLModelDebugInfo.h"
37+
#import "ETCoreMLModelStructurePath.h"
38+
#import "objc_safe_cast.h"
3839
#endif
3940

4041
namespace {
@@ -317,31 +318,14 @@ void add_compute_unit(std::string& identifier, MLComputeUnits compute_units) {
317318
return [[ETCoreMLAsset alloc] initWithBackingAsset:std::move(backingAsset.value())];
318319
}
319320

320-
NSDictionary<ETCoreMLModelStructurePath *, NSString *> * _Nullable get_operation_path_to_symbol_name_map(const inmemoryfs::InMemoryFileSystem *inMemoryFS,
321-
NSError * __autoreleasing *error) {
321+
ETCoreMLModelDebugInfo * _Nullable get_model_debug_info(const inmemoryfs::InMemoryFileSystem *inMemoryFS,
322+
NSError * __autoreleasing *error) {
322323
NSData *file_data = get_file_data(inMemoryFS, ETCoreMLStrings.debugInfoFileRelativePath);
323324
if (!file_data) {
324325
return nil;
325326
}
326-
327-
id object = [NSJSONSerialization JSONObjectWithData:file_data options:(NSJSONReadingOptions)0 error:error];
328-
if (!object) {
329-
return nil;
330-
}
331-
332-
NSDictionary<NSString *, id> *json_dict = SAFE_CAST(object, NSDictionary);
333-
NSMutableDictionary<ETCoreMLModelStructurePath *, NSString *> *result = [NSMutableDictionary dictionaryWithCapacity:json_dict.count];
334-
NSDictionary<NSString *, NSArray<id> *> *debug_symbol_to_operation_path_map = SAFE_CAST(json_dict[ETCoreMLStrings.debugSymbolToOperationPathKeyName], NSDictionary);
335-
for (NSString *symbol_name in debug_symbol_to_operation_path_map) {
336-
NSArray<NSDictionary<NSString *, id> *> *components = SAFE_CAST(debug_symbol_to_operation_path_map[symbol_name], NSArray);
337-
if (components.count == 0) {
338-
continue;
339-
}
340-
ETCoreMLModelStructurePath *path = [[ETCoreMLModelStructurePath alloc] initWithComponents:components];
341-
result[path] = symbol_name;
342-
}
343-
344-
return result;
327+
328+
return [ETCoreMLModelDebugInfo modelDebugInfoFromData:file_data error:error];
345329
}
346330

347331
#endif
@@ -490,16 +474,16 @@ - (nullable NSURL *)compiledModelURLWithIdentifier:(NSString *)identifier
490474
}
491475

492476
NSError *localError = nil;
493-
NSDictionary<ETCoreMLModelStructurePath *, NSString *> *operation_path_to_symbol_name_map = get_operation_path_to_symbol_name_map(inMemoryFS,
494-
&localError);
477+
ETCoreMLModelDebugInfo *debug_info = get_model_debug_info(inMemoryFS, &localError);
495478
if (localError) {
496479
ETCoreMLLogError(localError, "Failed to parse debug info file");
497480
}
498481

482+
499483
return [[ETCoreMLModelAnalyzer alloc] initWithCompiledModelAsset:compiledModelAsset
500484
modelAsset:modelAsset
485+
modelDebugInfo:debug_info
501486
metadata:metadata
502-
operationPathToDebugSymbolMap:operation_path_to_symbol_name_map
503487
configuration:configuration
504488
assetManager:self.assetManager
505489
error:error];

‎backends/apple/coreml/runtime/delegate/ETCoreMLStrings.h

+2
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ NS_ASSUME_NONNULL_BEGIN
6666
@property (class, copy, readonly, nonatomic, nullable) NSString* debugInfoFileRelativePath;
6767
/// The debug symbol to operation path key name.
6868
@property (class, copy, readonly, nonatomic, nullable) NSString* debugSymbolToOperationPathKeyName;
69+
/// The debug symbol to handles key name.
70+
@property (class, copy, readonly, nonatomic, nullable) NSString* debugSymbolToHandlesKeyName;
6971

7072
@end
7173

‎backends/apple/coreml/runtime/delegate/ETCoreMLStrings.mm

+5
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,11 @@ + (NSString *)debugSymbolToOperationPathKeyName {
9595
return ETCoreMLDebugSymbolToOperationPathKeyName;
9696
}
9797

98+
+ (NSString *)debugSymbolToHandlesKeyName {
99+
static NSString * const ETCoreMLDebugSymbolToHandlesKeyName = @"debugSymbolToHandles";
100+
return ETCoreMLDebugSymbolToHandlesKeyName;
101+
}
102+
98103
+ (nullable NSString *)assetsDirectoryPath {
99104
static dispatch_once_t onceToken;
100105
static NSString *result = nil;

‎backends/apple/coreml/runtime/delegate/coreml_backend_delegate.mm

+2-1
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,8 @@ ModelLoggingOptions get_logging_options(BackendExecutionContext& context) {
124124
auto event_tracer = context.event_tracer();
125125
if (event_tracer) {
126126
options.log_profiling_info = true;
127-
options.log_intermediate_tensors = event_tracer->intermediate_outputs_logging_status();
127+
auto debug_level = event_tracer->event_tracer_debug_level();
128+
options.log_intermediate_tensors = (debug_level >= EventTracerDebugLogLevel::kIntermediateOutputs);
128129
}
129130

130131
return options;

‎backends/apple/coreml/runtime/delegate/model_event_logger.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@ class ModelEventLogger {
3434
///
3535
/// @param op_path_to_value_map A dictionary with the operation path as the key and the operation's value as the
3636
/// value.
37-
/// @param op_path_to_debug_symbol_name_map A dictionary with the operation path as the key and the symbol name as
38-
/// the value. The symbol name is the delegate handle.
37+
/// @param op_path_to_debug_symbol_name_map A dictionary with the operation path as the key and the debug symbol
38+
/// name as the value.
3939
virtual void log_intermediate_tensors(
4040
NSDictionary<ETCoreMLModelStructurePath*, MLMultiArray*>* op_path_to_value_map,
4141
NSDictionary<ETCoreMLModelStructurePath*, NSString*>* op_path_to_debug_symbol_name_map) const noexcept = 0;

‎backends/apple/coreml/runtime/sdk/ETCoreMLModelAnalyzer.h

+4-4
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,15 @@
77

88
#import <CoreML/CoreML.h>
99

10-
#import <ETCoreMLModelExecutor.h>
10+
#import "ETCoreMLModelExecutor.h"
1111

1212
namespace executorchcoreml {
1313
struct ModelMetadata;
1414
}
1515

1616
@class ETCoreMLAsset;
1717
@class ETCoreMLAssetManager;
18+
@class ETCoreMLModelDebugInfo;
1819
@class ETCoreMLModelStructurePath;
1920
@protocol ETCoreMLModelEventLogger;
2021

@@ -32,16 +33,15 @@ __attribute__((objc_subclassing_restricted))
3233
///
3334
/// @param compiledModelAsset The compiled model asset (mlmodelc).
3435
/// @param modelAsset The model asset (mlpackage).
36+
/// @param modelDebugInfo The model debug info.
3537
/// @param metadata The model metadata.
36-
/// @param operationPathToDebugSymbolMap The operation path to debug symbol map.
3738
/// @param configuration The model configuration.
3839
/// @param assetManager The asset manager used to manage storage of compiled models.
3940
/// @param error On failure, error is filled with the failure information.
4041
- (nullable instancetype)initWithCompiledModelAsset:(ETCoreMLAsset*)compiledModelAsset
4142
modelAsset:(nullable ETCoreMLAsset*)modelAsset
43+
modelDebugInfo:(nullable ETCoreMLModelDebugInfo*)modelDebugInfo
4244
metadata:(const executorchcoreml::ModelMetadata&)metadata
43-
operationPathToDebugSymbolMap:
44-
(nullable NSDictionary<ETCoreMLModelStructurePath*, NSString*>*)operationPathToDebugSymbolMap
4545
configuration:(MLModelConfiguration*)configuration
4646
assetManager:(ETCoreMLAssetManager*)assetManager
4747
error:(NSError* __autoreleasing*)error NS_DESIGNATED_INITIALIZER;

‎backends/apple/coreml/runtime/sdk/ETCoreMLModelAnalyzer.mm

+25-21
Original file line numberDiff line numberDiff line change
@@ -5,22 +5,24 @@
55
//
66
// Please refer to the license found in the LICENSE file in the root directory of the source tree.
77

8-
#import <ETCoreMLAsset.h>
9-
#import <ETCoreMLAssetManager.h>
10-
#import <ETCoreMLDefaultModelExecutor.h>
11-
#import <ETCoreMLLogging.h>
12-
#import <ETCoreMLModel.h>
13-
#import <ETCoreMLModelAnalyzer.h>
14-
#import <ETCoreMLModelLoader.h>
15-
#import <ETCoreMLModelStructurePath.h>
16-
#import <ETCoreMLModelDebugger.h>
17-
#import <ETCoreMLModelProfiler.h>
18-
#import <ETCoreMLStrings.h>
19-
#import <model_logging_options.h>
20-
#import <model_event_logger.h>
21-
#import <model_metadata.h>
22-
#import <model_package_info.h>
23-
#import <objc_safe_cast.h>
8+
#import "ETCoreMLModelAnalyzer.h"
9+
10+
#import "ETCoreMLAsset.h"
11+
#import "ETCoreMLAssetManager.h"
12+
#import "ETCoreMLDefaultModelExecutor.h"
13+
#import "ETCoreMLLogging.h"
14+
#import "ETCoreMLModel.h"
15+
#import "ETCoreMLModelLoader.h"
16+
#import "ETCoreMLModelStructurePath.h"
17+
#import "ETCoreMLModelDebugInfo.h"
18+
#import "ETCoreMLModelDebugger.h"
19+
#import "ETCoreMLModelProfiler.h"
20+
#import "ETCoreMLStrings.h"
21+
#import "model_logging_options.h"
22+
#import "model_event_logger.h"
23+
#import "model_metadata.h"
24+
#import "model_package_info.h"
25+
#import "objc_safe_cast.h"
2426

2527
namespace {
2628
using namespace executorchcoreml;
@@ -34,7 +36,7 @@ @interface ETCoreMLModelAnalyzer ()
3436
@property (strong, nonatomic, nullable) ETCoreMLModelProfiler *profiler;
3537
@property (strong, nonatomic, nullable) ETCoreMLModelDebugger *debugger;
3638
@property (strong, nonatomic, nullable) id<ETCoreMLModelExecutor> executor;
37-
@property (readonly, copy, nonatomic, nullable) NSDictionary<ETCoreMLModelStructurePath *, NSString *> *operationPathToDebugSymbolMap;
39+
@property (readonly, copy, nonatomic, nullable) ETCoreMLModelDebugInfo *modelDebugInfo;
3840
@property (readonly, strong, nonatomic) MLModelConfiguration *configuration;
3941

4042
@end
@@ -43,8 +45,8 @@ @implementation ETCoreMLModelAnalyzer
4345

4446
- (nullable instancetype)initWithCompiledModelAsset:(ETCoreMLAsset *)compiledModelAsset
4547
modelAsset:(nullable ETCoreMLAsset *)modelAsset
48+
modelDebugInfo:(nullable ETCoreMLModelDebugInfo *)modelDebugInfo
4649
metadata:(const executorchcoreml::ModelMetadata&)metadata
47-
operationPathToDebugSymbolMap:(nullable NSDictionary<ETCoreMLModelStructurePath *, NSString *> *)operationPathToDebugSymbolMap
4850
configuration:(MLModelConfiguration *)configuration
4951
assetManager:(ETCoreMLAssetManager *)assetManager
5052
error:(NSError * __autoreleasing *)error {
@@ -72,9 +74,9 @@ - (nullable instancetype)initWithCompiledModelAsset:(ETCoreMLAsset *)compiledMod
7274
if (self) {
7375
_model = model;
7476
_modelAsset = modelAsset;
77+
_modelDebugInfo = modelDebugInfo;
7578
_assetManager = assetManager;
7679
_configuration = configuration;
77-
_operationPathToDebugSymbolMap = operationPathToDebugSymbolMap;
7880
_executor = [[ETCoreMLDefaultModelExecutor alloc] initWithModel:model];
7981
}
8082

@@ -113,7 +115,7 @@ - (nullable instancetype)initWithCompiledModelAsset:(ETCoreMLAsset *)compiledMod
113115
return nil;
114116
}
115117

116-
eventLogger->log_profiling_infos(profilingInfos, self.operationPathToDebugSymbolMap);
118+
eventLogger->log_profiling_infos(profilingInfos, self.modelDebugInfo.pathToDebugSymbolMap);
117119
return modelOutputs;
118120
}
119121

@@ -131,6 +133,7 @@ - (nullable instancetype)initWithCompiledModelAsset:(ETCoreMLAsset *)compiledMod
131133

132134
if (!self.debugger) {
133135
self.debugger = [[ETCoreMLModelDebugger alloc] initWithModelAsset:self.modelAsset
136+
modelDebugInfo:self.modelDebugInfo
134137
outputNames:self.model.orderedOutputNames
135138
configuration:self.configuration
136139
assetManager:self.assetManager
@@ -143,6 +146,7 @@ - (nullable instancetype)initWithCompiledModelAsset:(ETCoreMLAsset *)compiledMod
143146

144147
NSArray<MLMultiArray *> *modelOutputs = nil;
145148
NSArray<ETCoreMLModelStructurePath *> *operationPaths = self.debugger.operationPaths;
149+
NSDictionary<ETCoreMLModelStructurePath *, NSString *> *operationPathToDebugSymbolMap = self.debugger.operationPathToDebugSymbolMap;
146150
NSInteger n = operationPaths.count/MAX_MODEL_OUTPUTS_COUNT + (operationPaths.count % MAX_MODEL_OUTPUTS_COUNT == 0 ? 0 : 1);
147151
for (NSInteger i = 0; i < n; i++) {
148152
@autoreleasepool {
@@ -157,7 +161,7 @@ - (nullable instancetype)initWithCompiledModelAsset:(ETCoreMLAsset *)compiledMod
157161
}
158162

159163
if (outputs.count > 0) {
160-
eventLogger->log_intermediate_tensors(outputs, self.operationPathToDebugSymbolMap);
164+
eventLogger->log_intermediate_tensors(outputs, operationPathToDebugSymbolMap);
161165
}
162166
}
163167
}

‎backends/apple/coreml/runtime/sdk/ETCoreMLModelDebugInfo.h

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
//
2+
// ETCoreMLModelDebugInfo.h
3+
//
4+
// Copyright © 2024 Apple Inc. All rights reserved.
5+
//
6+
// Please refer to the license found in the LICENSE file in the root directory of the source tree.
7+
8+
#import <Foundation/Foundation.h>
9+
10+
@class ETCoreMLModelStructurePath;
11+
12+
NS_ASSUME_NONNULL_BEGIN
13+
14+
/// A class representing the profiling info of an operation.
15+
__attribute__((objc_subclassing_restricted))
16+
@interface ETCoreMLModelDebugInfo : NSObject
17+
18+
- (instancetype)init NS_UNAVAILABLE;
19+
20+
+ (instancetype)new NS_UNAVAILABLE;
21+
22+
23+
/// Constructs an `ETCoreMLModelDebugInfo` instance.
24+
///
25+
/// @param pathToDebugSymbolMap Operation path to debug symbol map.
26+
/// @param pathToDebugHandlesMap Operation path to debug handles map.
27+
- (instancetype)initWithPathToDebugSymbolMap:(NSDictionary<ETCoreMLModelStructurePath*, NSString*>*)pathToDebugSymbolMap
28+
pathToDebugHandlesMap:
29+
(NSDictionary<ETCoreMLModelStructurePath*, NSArray<NSString*>*>*)pathToDebugHandlesMap
30+
NS_DESIGNATED_INITIALIZER;
31+
32+
/// Constructs an `ETCoreMLModelDebugInfo` instance.
33+
///
34+
/// @param data The json data.
35+
/// @param error On failure, error is filled with the failure information.
36+
+ (nullable instancetype)modelDebugInfoFromData:(NSData*)data error:(NSError* __autoreleasing*)error;
37+
38+
/// Operation path to debug symbol map.
39+
@property (readonly, strong, nonatomic) NSDictionary<ETCoreMLModelStructurePath*, NSString*>* pathToDebugSymbolMap;
40+
41+
/// Operation path to debug handles map.
42+
@property (readonly, strong, nonatomic)
43+
NSDictionary<ETCoreMLModelStructurePath*, NSArray<NSString*>*>* pathToDebugHandlesMap;
44+
45+
@end
46+
47+
NS_ASSUME_NONNULL_END

0 commit comments

Comments
 (0)