Skip to content

Commit 1d5de15

Browse files
authored
Libraries testing (dotnet#178)
* Add Libraries Testing framework package as inline The existing package Microsoft.DotNet.CoreFxTesting lived in Arcade because of no infrastructure being available to compile local tasks in the repository. As the runtime repository now offers that we can inline the testing framework. * Hardcode configuration for installer.tasks * Update ReportGenerator global tool version * Add vstest support * Update binary serialization blobs
1 parent 5e5a0d4 commit 1d5de15

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+1228
-125
lines changed

.config/dotnet-tools.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
]
1010
},
1111
"dotnet-reportgenerator-globaltool": {
12-
"version": "4.3.0",
12+
"version": "4.3.6",
1313
"commands": [
1414
"reportgenerator"
1515
]

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ cross/android-rootfs/
5050
#NUNIT
5151
*.VisualState.xml
5252
TestResult.xml
53+
testResults.xml
5354

5455
# Build Results of an ATL Project
5556
[Dd]ebugPS/

Directory.Build.props

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,18 +11,23 @@
1111
<Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" Condition="'$(SkipImportArcadeSdkFromRoot)' != 'true'" />
1212

1313
<!-- Common paths -->
14-
<PropertyGroup>
15-
<!-- Set these properties early enough for libraries as they import the Arcade SDK not early enough. -->
14+
15+
<!-- Set these properties early enough for libraries as they import the Arcade SDK not early enough. -->
16+
<PropertyGroup Condition="'$(SkipImportArcadeSdkFromRoot)' == 'true'">
1617
<RepoRoot>$(MSBuildThisFileDirectory)</RepoRoot>
1718
<RepositoryEngineeringDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'eng'))</RepositoryEngineeringDir>
19+
<ArtifactsDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts'))</ArtifactsDir>
20+
<ArtifactsBinDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'bin'))</ArtifactsBinDir>
21+
</PropertyGroup>
1822

23+
<PropertyGroup>
1924
<RepoToolsLocalDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'tools-local'))</RepoToolsLocalDir>
2025
<RepoTasksDir>$([MSBuild]::NormalizeDirectory('$(RepoToolsLocalDir)', 'tasks'))</RepoTasksDir>
2126

2227
<!-- Installer specific, required during restore. -->
23-
<InstallerTasksOutputPath>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'installer.tasks', '$(Configuration)'))</InstallerTasksOutputPath>
24-
<InstallerTasksAssemblyPath Condition="'$(MSBuildRuntimeType)' == 'Core'">$([MSBuild]::NormalizePath('$(InstallerTasksOutputPath)', 'netstandard2.0', 'installer.tasks.dll'))</InstallerTasksAssemblyPath>
25-
<InstallerTasksAssemblyPath Condition="'$(MSBuildRuntimeType)' != 'Core'">$([MSBuild]::NormalizePath('$(InstallerTasksOutputPath)', 'net46', 'installer.tasks.dll'))</InstallerTasksAssemblyPath>
28+
<InstallerTasksOutputPath>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'installer.tasks'))</InstallerTasksOutputPath>
29+
<InstallerTasksAssemblyPath Condition="'$(MSBuildRuntimeType)' == 'Core'">$([MSBuild]::NormalizePath('$(InstallerTasksOutputPath)', 'Debug', 'netstandard2.0', 'installer.tasks.dll'))</InstallerTasksAssemblyPath>
30+
<InstallerTasksAssemblyPath Condition="'$(MSBuildRuntimeType)' != 'Core'">$([MSBuild]::NormalizePath('$(InstallerTasksOutputPath)', 'Debug', 'net46', 'installer.tasks.dll'))</InstallerTasksAssemblyPath>
2631
<HostMachineInfoProps>$(ArtifactsObjDir)HostMachineInfo.props</HostMachineInfoProps>
2732

2833
<LibrariesProjectRoot>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'libraries'))</LibrariesProjectRoot>

NuGet.config

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
-->
1313
<!-- TEST_RESTORE_SOURCES_INSERTION_LINE -->
1414
<add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
15+
<add key="dotnet-tools" value="https://dnceng.pkgs.visualstudio.com/public/_packaging/dotnet-tools/nuget/v3/index.json" />
1516
<add key="dotnet-eng" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json" />
1617
<add key="dotnet5-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json" />
1718
<add key="dotnet-core" value="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" />

docs/libraries/building/code-coverage.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ The results for this one library will then show up in the aforementioned index.h
4545

4646
And then once the run completes:
4747

48-
$(TestPath)\report\index.htm
48+
$(OutDir)\report\index.htm
4949

5050
**Note:** If you only want to measure the coverage of your local changes (that haven't been pushed to git), run:
5151

docs/libraries/project-docs/developer-guide.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -423,7 +423,7 @@ cd src\System.Collections.Immutable\tests
423423
dotnet msbuild /t:BuildAndTest /p:Coverage=true
424424
```
425425
426-
If coverage succeeds, the individual report can be found at `$(TestPath)\report\index.htm`.
426+
If coverage succeeds, the individual report can be found at `$(OutDir)\report\index.htm`.
427427
428428
Code coverage reports from the continuous integration system are available from the links on the front page of the corefx repo.
429429

eng/Build.props

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@
7979
</ItemGroup>
8080

8181
<MSBuild Projects="@(RepoTaskProjects)"
82+
Properties="Configuration=Debug"
8283
Targets="Restore;Build"/>
8384

8485
<WriteLinesToFile File="$(RepoTasksOutputFile)"
@@ -89,7 +90,7 @@
8990
<Target Name="GetRepoTasksSrc">
9091
<PropertyGroup>
9192
<RepoTasksDir>$(RepoTasksDir)</RepoTasksDir>
92-
<RepoTasksOutputFile>$(ArtifactsObjDir)runtime.tasks\$(Configuration)\build-semaphore.txt</RepoTasksOutputFile>
93+
<RepoTasksOutputFile>$(ArtifactsObjDir)runtime.tasks\Debug\build-semaphore.txt</RepoTasksOutputFile>
9394
</PropertyGroup>
9495

9596
<ItemGroup>

eng/Version.Details.xml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,5 +130,9 @@
130130
<Uri>https://dev.azure.com/dnceng/internal/_git/dotnet-optimization</Uri>
131131
<Sha>d0bb63d2ec7060714e63ee4082fac48f2e57f3e2</Sha>
132132
</Dependency>
133+
<Dependency Name="Microsoft.NET.Test.Sdk" Version="16.4.0">
134+
<Uri>https://github.com/Microsoft/vstest</Uri>
135+
<Sha>ca987de449d17284f8c9806df36f42276f13962a</Sha>
136+
</Dependency>
133137
</ToolsetDependencies>
134138
</Dependencies>

eng/Versions.props

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -80,12 +80,12 @@
8080
<RefOnlyNugetPackagingVersion>4.9.4</RefOnlyNugetPackagingVersion>
8181
<!-- sni -->
8282
<RuntimeWinX64RuntimeNativeSystemDataSqlClientSniVersion>4.4.0</RuntimeWinX64RuntimeNativeSystemDataSqlClientSniVersion>
83-
<RuntimeNativeSystemDataSqlClientSniVersion>4.4.0</RuntimeNativeSystemDataSqlClientSniVersion>
8483
<!-- Testing -->
85-
<MicrosoftNETTestSdkVersion>16.3.0</MicrosoftNETTestSdkVersion>
84+
<MicrosoftNETTestSdkVersion>16.4.0</MicrosoftNETTestSdkVersion>
8685
<XUnitVersion>2.4.1</XUnitVersion>
8786
<TraceEventVersion>2.0.5</TraceEventVersion>
88-
<NewtonsoftJsonVersion>12.0.1</NewtonsoftJsonVersion>
87+
<NewtonsoftJsonVersion>12.0.3</NewtonsoftJsonVersion>
88+
<XUnitXmlTestLoggerVersion>2.1.26</XUnitXmlTestLoggerVersion>
8989
<!-- Test data -->
9090
<SystemIOCompressionTestDataVersion>1.0.16</SystemIOCompressionTestDataVersion>
9191
<SystemIOPackagingTestDataVersion>1.0.4</SystemIOPackagingTestDataVersion>

eng/pipelines/libraries/corefx-base.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ jobs:
8484
- _msbuildCommonParameters: /p:OfficialBuildId=$(Build.BuildNumber)
8585

8686
- ${{ if eq(job.submitToHelix, 'true') }}:
87-
- _archiveTestsParameter: /p:ArchiveTests=Tests
87+
- _archiveTestsParameter: /p:ArchiveTests=true
8888
- ${{ if eq(parameters.isOfficialBuild, 'true') }}:
8989
- group: DotNet-HelixApi-Access
9090

eng/pipelines/libraries/windows.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ stages:
134134
-allconfigurations
135135
-arch $(_architecture)
136136
/p:RuntimeOS=win10
137-
/p:ArchiveTests=Packages
137+
/p:ArchiveTests=true
138138
$(_msbuildCommonParameters)
139139
displayName: Build Packages and Tests
140140

eng/references.targets

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,9 @@
1717
<IncludeDefaultReferences Condition="'$(MSBuildProjectExtension)' == '.csproj'">true</IncludeDefaultReferences>
1818
<IncludeDefaultReferences Condition="'$(MSBuildProjectExtension)' == '.vbproj'">true</IncludeDefaultReferences>
1919
</PropertyGroup>
20+
2021
<Target Name="SetupDefaultReferences">
21-
<ItemGroup Condition="'$(IncludeDefaultReferences)' =='true'">
22+
<ItemGroup Condition="'$(IncludeDefaultReferences)' == 'true'">
2223
<!-- netstandard is a default reference whenever building for NETStandard or building an implementation assembly -->
2324
<DefaultReference Condition="('$(TargetsNetStandard)' == 'true' or '$(IsReferenceAssembly)' != 'true') and Exists('$(RefPath)netstandard.dll')" Include="netstandard" />
2425
</ItemGroup>
@@ -35,5 +36,32 @@
3536
</ItemGroup>
3637
</Target>
3738

39+
<Target Name="AddDefaultTestReferences" BeforeTargets="SetupDefaultReferences" Condition="'$(IsTestProject)' == 'true'">
40+
<ItemGroup>
41+
<DefaultReferenceExclusions Include="@(ReferenceFromRuntime)"/>
42+
43+
<!-- Reference everything in the targeting pack directory -->
44+
<DefaultReferenceDirs Include="$(RefPath)" />
45+
<DefaultReferenceItems Include="%(DefaultReferenceDirs.Identity)/*.dll" />
46+
47+
<DefaultReferenceExclusions>
48+
<RefDir>%(DefaultReferenceDirs.Identity)</RefDir>
49+
</DefaultReferenceExclusions>
50+
<_defaultReferenceExclusionsFullPath Include="%(DefaultReferenceExclusions.RefDir)%(DefaultReferenceExclusions.Identity).dll" />
51+
52+
<!-- Ensure conflict resolution can see these references. -->
53+
<Reference Include="%(DefaultReferenceItems.FullPath)" Private="false" />
54+
</ItemGroup>
55+
</Target>
56+
57+
<Target Name="RemoveConflictResolutionAssetsForTests"
58+
AfterTargets="_HandlePackageFileConflicts"
59+
DependsOnTargets="AddDefaultTestReferences"
60+
Condition="'@(_defaultReferenceExclusionsFullPath)' != ''">
61+
<ItemGroup>
62+
<Reference Remove="@(_defaultReferenceExclusionsFullPath)" />
63+
</ItemGroup>
64+
</Target>
65+
3866
<Target Name="AddReferencesDynamically" BeforeTargets="PrepareForBuild" />
3967
</Project>

eng/testing/RunnerTemplate.Unix.txt

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
#!/usr/bin/env bash
2+
3+
usage()
4+
{
5+
echo "Usage: RunTests.sh {-r|--runtime-path} <runtime-path> [{--rsp-file} <rsp-file>]"
6+
echo ""
7+
echo "Parameters:"
8+
echo "--runtime-path (Mandatory) Testhost containing the test runtime used during test execution (short: -r)"
9+
echo "--rsp-file RSP file to pass in additional arguments"
10+
echo "--help Print help and exit (short: -h)"
11+
}
12+
13+
EXECUTION_DIR=$(dirname "$0")
14+
RUNTIME_PATH=''
15+
RSP_FILE=''
16+
17+
while [[ $# > 0 ]]; do
18+
opt="$(echo "${1}" | awk '{print tolower($0)}')"
19+
case "$opt" in
20+
--help|-h)
21+
usage
22+
exit 0
23+
;;
24+
--runtime-path|-r)
25+
RUNTIME_PATH=$2
26+
shift
27+
;;
28+
--rsp-file)
29+
RSP_FILE=\@$2
30+
shift
31+
;;
32+
*)
33+
echo "Invalid argument: $1"
34+
usage
35+
exit -1
36+
;;
37+
esac
38+
39+
shift
40+
done
41+
42+
if [ "$RUNTIME_PATH" == "" ]; then
43+
echo "error: -r|--runtime-path argument is required."
44+
usage
45+
exit -1
46+
fi
47+
48+
# Don't use a globally installed SDK.
49+
export DOTNET_MULTILEVEL_LOOKUP=0
50+
51+
exitcode_list[0]="Exited Successfully"
52+
exitcode_list[130]="SIGINT Ctrl-C occurred. Likely tests timed out."
53+
exitcode_list[131]="SIGQUIT Ctrl-\ occurred. Core dumped."
54+
exitcode_list[132]="SIGILL Illegal Instruction. Core dumped. Likely codegen issue."
55+
exitcode_list[133]="SIGTRAP Breakpoint hit. Core dumped."
56+
exitcode_list[134]="SIGABRT Abort. Managed or native assert, or runtime check such as heap corruption, caused call to abort(). Core dumped."
57+
exitcode_list[135]="IGBUS Unaligned memory access. Core dumped."
58+
exitcode_list[136]="SIGFPE Bad floating point arguments. Core dumped."
59+
exitcode_list[137]="SIGKILL Killed eg by kill"
60+
exitcode_list[139]="SIGSEGV Illegal memory access. Deref invalid pointer, overrunning buffer, stack overflow etc. Core dumped."
61+
exitcode_list[143]="SIGTERM Terminated. Usually before SIGKILL."
62+
exitcode_list[159]="SIGSYS Bad System Call."
63+
64+
function print_info_from_core_file_using_lldb {
65+
local core_file_name=$1
66+
local executable_name=$2
67+
local plugin_path_name="$RUNTIME_PATH/shared/Microsoft.NETCore.App/9.9.9/libsosplugin.so"
68+
69+
# check for existence of lldb on the path
70+
hash lldb 2>/dev/null || { echo >&2 "lldb was not found. Unable to print core file."; return; }
71+
72+
# pe, clrstack, and dumpasync are defined in libsosplugin.so
73+
if [ ! -f $plugin_path_name ]; then
74+
echo $plugin_path_name cannot be found.
75+
return
76+
fi
77+
78+
echo ----- start =============== lldb Output =====================================================
79+
echo Printing managed exceptions, managed call stacks, and async state machines.
80+
lldb -O "settings set target.exec-search-paths $RUNTIME_PATH" -o "plugin load $plugin_path_name" -o "clrthreads -managedexception" -o "pe -nested" -o "clrstack -all -a -f" -o "dumpasync -fields -stacks -roots" -o "quit" --core $core_file_name $executable_name
81+
echo ----- end =============== lldb Output =======================================================
82+
}
83+
84+
function print_info_from_core_file_using_gdb {
85+
local core_file_name=$1
86+
local executable_name=$2
87+
88+
# Check for the existence of GDB on the path
89+
hash gdb 2>/dev/null || { echo >&2 "GDB was not found. Unable to print core file."; return; }
90+
91+
echo ----- start =============== GDB Output =====================================================
92+
# Open the dump in GDB and print the stack from each thread. We can add more
93+
# commands here if desired.
94+
echo printing native stack.
95+
gdb --batch -ex "thread apply all bt full" -ex "quit" $executable_name $core_file_name
96+
echo ----- end =============== GDB Output =======================================================
97+
}
98+
99+
function print_info_from_core_file {
100+
local core_file_name=$1
101+
local executable_name=$RUNTIME_PATH/$2
102+
103+
if ! [ -e $executable_name ]; then
104+
echo "Unable to find executable $executable_name"
105+
return
106+
elif ! [ -e $core_file_name ]; then
107+
echo "Unable to find core file $core_file_name"
108+
return
109+
fi
110+
echo "Printing info from core file $core_file_name"
111+
print_info_from_core_file_using_gdb $core_file_name $executable_name
112+
print_info_from_core_file_using_lldb $core_file_name $executable_name
113+
}
114+
115+
function copy_core_file_to_temp_location {
116+
local core_file_name=$1
117+
118+
local storage_location="/tmp/coredumps"
119+
120+
# Create the directory (this shouldn't fail even if it already exists).
121+
mkdir -p $storage_location
122+
123+
local new_location=$storage_location/core.$RANDOM
124+
125+
echo "Copying core file $core_file_name to $new_location in case you need it."
126+
cp $core_file_name $new_location
127+
}
128+
129+
# ========================= BEGIN Core File Setup ============================
130+
if [ "$(uname -s)" == "Darwin" ]; then
131+
# On OS X, we will enable core dump generation only if there are no core
132+
# files already in /cores/ at this point. This is being done to prevent
133+
# inadvertently flooding the CI machines with dumps.
134+
if [[ ! -d "/cores" || ! "$(ls -A /cores)" ]]; then
135+
ulimit -c unlimited
136+
fi
137+
elif [ "$(uname -s)" == "Linux" ]; then
138+
# On Linux, we'll enable core file generation unconditionally, and if a dump
139+
# is generated, we will print some useful information from it and delete the
140+
# dump immediately.
141+
142+
if [ -e /proc/self/coredump_filter ]; then
143+
# Include memory in private and shared file-backed mappings in the dump.
144+
# This ensures that we can see disassembly from our shared libraries when
145+
# inspecting the contents of the dump. See 'man core' for details.
146+
echo -n 0x3F > /proc/self/coredump_filter
147+
fi
148+
149+
ulimit -c unlimited
150+
fi
151+
# ========================= END Core File Setup ==============================
152+
153+
# ========================= BEGIN Test Execution =============================
154+
echo ----- start $(date) =============== To repro directly: =====================================================
155+
echo pushd $EXECUTION_DIR
156+
[[RunCommandsEcho]]
157+
echo popd
158+
echo ===========================================================================================================
159+
pushd $EXECUTION_DIR
160+
[[RunCommands]]
161+
test_exitcode=$?
162+
popd
163+
echo ----- end $(date) ----- exit code $test_exitcode ----------------------------------------------------------
164+
165+
if [ "${exitcode_list[$test_exitcode]}" != "" ]; then
166+
echo exit code $test_exitcode means ${exitcode_list[$test_exitcode]}
167+
fi
168+
# ========================= END Test Execution ===============================
169+
170+
# ======================= BEGIN Core File Inspection =========================
171+
pushd $EXECUTION_DIR >/dev/null
172+
if [[ "$(uname -s)" == "Linux" && $test_exitcode -ne 0 ]]; then
173+
echo Looking around for any Linux dump...
174+
# Depending on distro/configuration, the core files may either be named "core"
175+
# or "core.<PID>" by default. We read /proc/sys/kernel/core_uses_pid to
176+
# determine which it is.
177+
core_name_uses_pid=0
178+
if [ -e /proc/sys/kernel/core_uses_pid ] && [ "1" == $(cat /proc/sys/kernel/core_uses_pid) ]; then
179+
core_name_uses_pid=1
180+
fi
181+
182+
if [ $core_name_uses_pid == "1" ]; then
183+
# We don't know what the PID of the process was, so let's look at all core
184+
# files whose name matches core.NUMBER
185+
echo Looking for files matching core.* ...
186+
for f in core.*; do
187+
[[ $f =~ core.[0-9]+ ]] && print_info_from_core_file "$f" "dotnet" && copy_core_file_to_temp_location "$f" && rm "$f"
188+
done
189+
elif [ -f core ]; then
190+
echo found a dump named core in $EXECUTION_DIR !
191+
print_info_from_core_file "core" "dotnet"
192+
copy_core_file_to_temp_location "core"
193+
rm "core"
194+
else
195+
echo ... found no dump in $PWD
196+
fi
197+
fi
198+
popd >/dev/null
199+
# ======================== END Core File Inspection ==========================
200+
exit $test_exitcode

0 commit comments

Comments
 (0)