diff --git a/Documentation/docs-mobile/building-apps/build-items.md b/Documentation/docs-mobile/building-apps/build-items.md index 57cc85ce9af..3415958ff0d 100644 --- a/Documentation/docs-mobile/building-apps/build-items.md +++ b/Documentation/docs-mobile/building-apps/build-items.md @@ -33,7 +33,7 @@ The following MSBuild metadata are required: - `%(JavaArtifact)`: The group and artifact id of the Java library matching the specifed POM file in the form `{GroupId}:{ArtifactId}`. - `%(JavaVersion)`: The version of the Java library matching the specified POM file. - + See the [Java Dependency Resolution documentation](../features/maven/java-dependency-verification.md) for more details. @@ -279,9 +279,9 @@ installing app bundles. ## AndroidMavenLibrary -`` allows a Maven artifact to be specified which will -automatically be downloaded and added to a .NET for Android binding project. -This can be useful to simplify maintenance of .NET for Android bindings for artifacts +`` allows a Maven artifact to be specified which will +automatically be downloaded and added to a .NET for Android binding project. +This can be useful to simplify maintenance of .NET for Android bindings for artifacts hosted in Maven. ```xml @@ -336,6 +336,20 @@ used to specify the ABI that the library targets. Thus, if you add ``` +## AndroidNativeLibraryNoJniPreload + +Every native library included in this item group will be exempt from the +JNI library preload mechanism. By default, all such libraries will be loaded +by the runtime early during application startup in order to assure their +proper initialization. However, in some cases it might not be the desired +behavior and this item group allows exclusion of libraries from this process +on individual basis. + +Some framework libraries which must be loaded at application startup will not +be affected if included in this item group. + +See also [`$(AndroidIgnoreAllJniPreload)`](build-properties.md#androidignorealljnipreload) + ## AndroidPackagingOptionsExclude A set of file glob compatible items which will allow for items to be diff --git a/Documentation/docs-mobile/building-apps/build-properties.md b/Documentation/docs-mobile/building-apps/build-properties.md index 02a75139a1c..20d6e33bbe0 100644 --- a/Documentation/docs-mobile/building-apps/build-properties.md +++ b/Documentation/docs-mobile/building-apps/build-properties.md @@ -689,6 +689,20 @@ The most common values for this property are: > [`@(AndroidEnvironment)`](build-items.md#androidenvironment) > will take precedence. +## AndroidIgnoreAllJniPreload + +A boolean value which, if set to `true`, exempts all the native JNI libraries +from being preloaded at application startup. By default, all such libraries +will be loaded by the runtime early during application startup in order to +assure their proper initialization. However, in some cases it might not be the +desired behavior and this property allows to effectively disable this behavior. + +Some framework libraries which must be loaded at application startup will not +be affected by this property. + +See also [`@(AndroidNativeLibraryNoJniPreload)`](build-items.md#androidnativelibrarynojnipreload) +for a more fine-grained way to exempt libraries from the preload mechanism. + ## AndroidIncludeWrapSh A boolean value that indicates whether the Android wrapper script diff --git a/build-tools/xaprepare/xaprepare/Application/KnownProperties.cs b/build-tools/xaprepare/xaprepare/Application/KnownProperties.cs index 28834a88636..9e055ee2651 100644 --- a/build-tools/xaprepare/xaprepare/Application/KnownProperties.cs +++ b/build-tools/xaprepare/xaprepare/Application/KnownProperties.cs @@ -56,6 +56,7 @@ static class KnownProperties public const string PkgXamarin_LibZipSharp = "PkgXamarin_LibZipSharp"; public const string ProductVersion = "ProductVersion"; public const string RuntimeRedistDirName = "_RuntimeRedistDirName"; + public const string TestOutputDirectory = "TestOutputDirectory"; public const string XABuildToolsFolder = "XABuildToolsFolder"; public const string XABuildToolsVersion = "XABuildToolsVersion"; public const string XABuildToolsPackagePrefixMacOS = "XABuildToolsPackagePrefixMacOS"; diff --git a/build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in b/build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in index 393c43a017c..3fed29ec076 100644 --- a/build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in +++ b/build-tools/xaprepare/xaprepare/Application/Properties.Defaults.cs.in @@ -60,6 +60,7 @@ namespace Xamarin.Android.Prepare properties.Add (KnownProperties.PkgXamarin_LibZipSharp, StripQuotes (@"@PkgXamarin_LibZipSharp@")); properties.Add (KnownProperties.ProductVersion, StripQuotes ("@ProductVersion@")); properties.Add (KnownProperties.RuntimeRedistDirName, StripQuotes ("@_RuntimeRedistDirName@")); + properties.Add (KnownProperties.TestOutputDirectory, StripQuotes (@"@TestOutputDirectory@")); properties.Add (KnownProperties.XABuildToolsFolder, StripQuotes (@"@XABuildToolsFolder@")); properties.Add (KnownProperties.XABuildToolsVersion, StripQuotes ("@XABuildToolsVersion@")); properties.Add (KnownProperties.XABuildToolsPackagePrefixMacOS, StripQuotes ("@XABuildToolsPackagePrefixMacOS@")); diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_GenerateFiles.cs b/build-tools/xaprepare/xaprepare/Steps/Step_GenerateFiles.cs index 41f04b00258..3198bc73d29 100644 --- a/build-tools/xaprepare/xaprepare/Steps/Step_GenerateFiles.cs +++ b/build-tools/xaprepare/xaprepare/Steps/Step_GenerateFiles.cs @@ -127,6 +127,7 @@ GeneratedFile GetCmakePresetsCommon (Context context, string sourcesDir) { "@NDK_ARM64_V8A_NONMONO_API_NET@", BuildAndroidPlatforms.NdkMinimumNonMonoAPI }, { "@NDK_X86_64_NONMONO_API_NET@", BuildAndroidPlatforms.NdkMinimumNonMonoAPI }, { "@XA_BUILD_CONFIGURATION@", context.Configuration }, + { "@XA_TEST_OUTPUT_DIR@", Utilities.EscapePathSeparators (props.GetRequiredValue (KnownProperties.TestOutputDirectory)) }, }; return new GeneratedPlaceholdersFile ( diff --git a/build-tools/xaprepare/xaprepare/xaprepare.targets b/build-tools/xaprepare/xaprepare/xaprepare.targets index d5ad4c1e58a..5575897219a 100644 --- a/build-tools/xaprepare/xaprepare/xaprepare.targets +++ b/build-tools/xaprepare/xaprepare/xaprepare.targets @@ -93,6 +93,7 @@ + diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateNativeApplicationConfigSources.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateNativeApplicationConfigSources.cs index 3d9eda66001..81a4ef23e30 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateNativeApplicationConfigSources.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateNativeApplicationConfigSources.cs @@ -28,6 +28,8 @@ public class GenerateNativeApplicationConfigSources : AndroidTask public ITaskItem[] ResolvedAssemblies { get; set; } = []; public ITaskItem[]? NativeLibraries { get; set; } + public ITaskItem[]? NativeLibrariesNoJniPreload { get; set; } + public ITaskItem[]? NativeLibrariesAlwaysJniPreload { get; set; } public ITaskItem[]? MonoComponents { get; set; } @@ -253,6 +255,8 @@ public override bool RunTask () NumberOfAssembliesInApk = assemblyCount, BundledAssemblyNameWidth = assemblyNameWidth, NativeLibraries = uniqueNativeLibraries, + NativeLibrariesNoJniPreload = NativeLibrariesNoJniPreload, + NativeLibrarysAlwaysJniPreload = NativeLibrariesAlwaysJniPreload, AndroidRuntimeJNIEnvToken = android_runtime_jnienv_class_token, JNIEnvInitializeToken = jnienv_initialize_method_token, JNIEnvRegisterJniNativesToken = jnienv_registerjninatives_method_token, @@ -279,6 +283,8 @@ public override bool RunTask () BundledAssemblyNameWidth = assemblyNameWidth, MonoComponents = (MonoComponent)monoComponents, NativeLibraries = uniqueNativeLibraries, + NativeLibrariesNoJniPreload = NativeLibrariesNoJniPreload, + NativeLibrarysAlwaysJniPreload = NativeLibrariesAlwaysJniPreload, HaveAssemblyStore = UseAssemblyStore, AndroidRuntimeJNIEnvToken = android_runtime_jnienv_class_token, JNIEnvInitializeToken = jnienv_initialize_method_token, diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest3.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest3.cs new file mode 100644 index 00000000000..957b506b6ce --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest3.cs @@ -0,0 +1,286 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using NUnit.Framework; +using Xamarin.Android.Tasks; +using Xamarin.Android.Tools; +using Xamarin.ProjectTools; + +namespace Xamarin.Android.Build.Tests; + +[Parallelizable (ParallelScope.Children)] +public partial class BuildTest3 : BaseTest +{ + const int ExpectedJniPreloadIndexStride = 4; + const string JniPreloadSourceLibraryName = "libtest-jni-library.so"; + + [Test] + public void NativeLibraryJniPreload_NoDuplicates ([Values] AndroidRuntime runtime) + { + const string MyLibKeep1 = "libMyStuffKeep.so"; + const string MyLibKeep2 = "libMyStuffKeep.so"; + + List? allPreloads = NativeLibraryJniPreload_CommonInitAndGetPreloads ( + runtime, + (XamarinAndroidApplicationProject proj, AndroidTargetArch[] supportedArches) => { + NativeLibraryJniPreload_AddNativeLibraries (proj, supportedArches, MyLibKeep1, MyLibKeep2); + } + ); + if (allPreloads == null) { + return; + } + + NativeLibraryJniPreload_VerifyLibs (allPreloads, new List { MyLibKeep1 }); + } + + [Test] + public void NativeLibraryJniPreload_IncludeCustomLibraries ([Values] AndroidRuntime runtime) + { + const string MyLib = "libMyStuff.so"; + + List? allPreloads = NativeLibraryJniPreload_CommonInitAndGetPreloads ( + runtime, + (XamarinAndroidApplicationProject proj, AndroidTargetArch[] supportedArches) => { + NativeLibraryJniPreload_AddNativeLibraries (proj, supportedArches, MyLib); + } + ); + if (allPreloads == null) { + return; + } + + NativeLibraryJniPreload_VerifyLibs (allPreloads, new List { MyLib }); + } + + [Test] + public void NativeLibraryJniPreload_ExcludeSomeCustomLibraries ([Values] AndroidRuntime runtime) + { + const string MyLibKeep = "libMyStuffKeep.so"; + const string MyLibExempt = "libMyStuffExempt.so"; + + List? allPreloads = NativeLibraryJniPreload_CommonInitAndGetPreloads ( + runtime, + (XamarinAndroidApplicationProject proj, AndroidTargetArch[] supportedArches) => { + NativeLibraryJniPreload_AddNativeLibraries (proj, supportedArches, MyLibKeep, MyLibExempt); + proj.OtherBuildItems.Add ( + new AndroidItem.AndroidNativeLibraryNoJniPreload (MyLibExempt) + ); + } + ); + if (allPreloads == null) { + return; + } + + NativeLibraryJniPreload_VerifyLibs (allPreloads, new List { MyLibKeep }); + } + + [Test] + public void NativeLibraryJniPreload_ExcludeAllCustomLibraries ([Values] AndroidRuntime runtime) + { + const string MyLibExempt1 = "libMyStuffExempt1.so"; + const string MyLibExempt2 = "libMyStuffExempt2.so"; + + List? allPreloads = NativeLibraryJniPreload_CommonInitAndGetPreloads ( + runtime, + (XamarinAndroidApplicationProject proj, AndroidTargetArch[] supportedArches) => { + NativeLibraryJniPreload_AddNativeLibraries (proj, supportedArches, MyLibExempt1, MyLibExempt2); + proj.OtherBuildItems.Add ( + new AndroidItem.AndroidNativeLibraryNoJniPreload (MyLibExempt1) + ); + proj.OtherBuildItems.Add ( + new AndroidItem.AndroidNativeLibraryNoJniPreload (MyLibExempt2) + ); + } + ); + if (allPreloads == null) { + return; + } + + NativeLibraryJniPreload_VerifyDefaults (allPreloads); + } + + [Test] + public void NativeLibraryJniPreload_AddSomeCustomLibrariesAndIgnoreAll ([Values] AndroidRuntime runtime) + { + List? allPreloads = NativeLibraryJniPreload_CommonInitAndGetPreloads ( + runtime, + (XamarinAndroidApplicationProject proj, AndroidTargetArch[] supportedArches) => { + NativeLibraryJniPreload_AddNativeLibraries (proj, supportedArches, "libMyStuffOne.so", "libMyStuffTwo.so"); + proj.SetProperty ("AndroidIgnoreAllJniPreload", "true"); + } + ); + if (allPreloads == null) { + return; + } + + // With `$(AndroidIgnoreAllJniPreload)=true` we still must have the defaults in the generated code. + NativeLibraryJniPreload_VerifyDefaults (allPreloads); + } + + [Test] + public void NativeLibraryJniPreload_AddSomeCustomLibrariesAndIgnoreAllByName ([Values] AndroidRuntime runtime) + { + const string MyLibExemptOne = "libMyStuffExemptOne.so"; + const string MyLibExemptTwo = "libMyStuffExemptTwo.so"; + + List? allPreloads = NativeLibraryJniPreload_CommonInitAndGetPreloads ( + runtime, + (XamarinAndroidApplicationProject proj, AndroidTargetArch[] supportedArches) => { + NativeLibraryJniPreload_AddNativeLibraries (proj, supportedArches, MyLibExemptOne, MyLibExemptTwo); + proj.OtherBuildItems.Add ( + new AndroidItem.AndroidNativeLibraryNoJniPreload (MyLibExemptOne) + ); + proj.OtherBuildItems.Add ( + new AndroidItem.AndroidNativeLibraryNoJniPreload (MyLibExemptTwo) + ); + } + ); + if (allPreloads == null) { + return; + } + + // With all custom libraries ignored, we still must have the defaults in the generated code. + NativeLibraryJniPreload_VerifyDefaults (allPreloads); + } + + void NativeLibraryJniPreload_AddNativeLibraries (XamarinAndroidApplicationProject proj, AndroidTargetArch[] supportedArches, string libName, params string[]? moreLibNames) + { + var libNames = new List { + libName, + }; + if (moreLibNames != null && moreLibNames.Length > 0) { + libNames.AddRange (moreLibNames); + } + + foreach (AndroidTargetArch arch in supportedArches) { + string libPath = Path.Combine (XABuildPaths.TestAssemblyOutputDirectory, MonoAndroidHelper.ArchToRid (arch), JniPreloadSourceLibraryName); + Assert.IsTrue (File.Exists (libPath), $"Native library '{libPath}' does not exist."); + + foreach (string lib in libNames) { + string abi = MonoAndroidHelper.ArchToAbi (arch); + proj.OtherBuildItems.Add ( + new AndroidItem.AndroidNativeLibrary ($"native/{abi}/{lib}") { + BinaryContent = () => File.ReadAllBytes (libPath), + MetadataValues = $"Link={abi}/{lib}", + } + ); + } + } + } + + [Test] + public void NativeLibraryJniPreload_IgnoreAll_PreservesRequired ([Values] AndroidRuntime runtime) + { + List? allPreloads = NativeLibraryJniPreload_CommonInitAndGetPreloads ( + runtime, + (XamarinAndroidApplicationProject proj, AndroidTargetArch[] supportedArches) => { + proj.SetProperty ("AndroidIgnoreAllJniPreload", "true"); + } + ); + + // With `$(AndroidIgnoreAllJniPreload)=true` we still must have the defaults in the generated code. + NativeLibraryJniPreload_VerifyDefaults (allPreloads); + } + + [Test] + public void NativeLibraryJniPreload_DefaultsWork ([Values] AndroidRuntime runtime) + { + List? allPreloads = NativeLibraryJniPreload_CommonInitAndGetPreloads (runtime); + NativeLibraryJniPreload_VerifyDefaults (allPreloads); + } + + void NativeLibraryJniPreload_VerifyDefaults (List? allPreloads) + { + NativeLibraryJniPreload_VerifyLibs (allPreloads, additionalLibs: null); + } + + void NativeLibraryJniPreload_VerifyLibs (List? allPreloads, List? additionalLibs) + { + if (allPreloads == null) { + return; + } + + int numberOfLibs = 1; + if (additionalLibs != null) { + numberOfLibs += additionalLibs.Count; + } + + int ExpectedEntryCount = ExpectedJniPreloadIndexStride * numberOfLibs; + foreach (EnvironmentHelper.JniPreloads preloads in allPreloads) { + Assert.IsTrue (preloads.IndexStride == (uint)ExpectedJniPreloadIndexStride, $"JNI preloads index stride should be {ExpectedJniPreloadIndexStride}, was {preloads.IndexStride} instead. Source file: {preloads.SourceFile}"); + Assert.IsTrue (preloads.Entries.Count == ExpectedEntryCount, $"JNI preloads index entry count should be {ExpectedEntryCount}, was {preloads.Entries.Count} instead. Source file: {preloads.SourceFile}"); + + // DSO cache entries are sorted based on their **mutated name's** 64-bit xxHash, which + // won't change but builds may add/remove libraries and, thus, change the indexes after + // sorting. For that reason we don't verify the index values and use them just for reporting. + // + // Also, all the entries will point to the same library name. Name variations aren't + // stored directly in the DSO cache, just their hashes which are used for lookup at run time. + // + // We use a Dictionary<> here because there might be more libraries to preload in the future. + var expectedLibNames = new Dictionary (StringComparer.Ordinal) { + { "libSystem.Security.Cryptography.Native.Android.so", 0 }, + }; + + if (additionalLibs != null) { + foreach (string extraLib in additionalLibs) { + expectedLibNames.Add (extraLib, 0); + } + } + + for (int i = 0; i < preloads.Entries.Count; i++) { + EnvironmentHelper.JniPreloadsEntry entry = preloads.Entries[i]; + Assert.IsFalse (entry.LibraryName == "libmonodroid.so", $"JNI preloads entry at index {i} refers to the .NET for Android native runtime. It must never be preloaded. Source file: {preloads.SourceFile}"); + Assert.IsTrue (expectedLibNames.ContainsKey (entry.LibraryName), $"JNI preloads entry at index {i}, referring to library at DSO cache index {entry.Index} has unexpected name '{entry.LibraryName}'; Source file: {preloads.SourceFile}"); + expectedLibNames[entry.LibraryName]++; + } + + foreach (var kvp in expectedLibNames) { + Assert.IsTrue (kvp.Value == ExpectedJniPreloadIndexStride, $"JNI preloads entry '{kvp.Key}' should have {ExpectedJniPreloadIndexStride} instances, it had {kvp.Value} instead. Source file: {preloads.SourceFile}"); + } + } + } + + List? NativeLibraryJniPreload_CommonInitAndGetPreloads (AndroidRuntime runtime, Action? configureProject = null) + { + const bool isRelease = true; + if (IgnoreUnsupportedConfiguration (runtime, release: isRelease)) { + return null; + } + + if (runtime == AndroidRuntime.NativeAOT) { + Assert.Ignore ("NativeAOT doesn't use JNI preload"); + } + + AndroidTargetArch[] supportedArches = new [] { + AndroidTargetArch.Arm64, + AndroidTargetArch.X86_64, + }; + + var proj = new XamarinAndroidApplicationProject { + IsRelease = isRelease, + }; + proj.SetRuntime (runtime); + proj.SetRuntimeIdentifiers (supportedArches); + configureProject?.Invoke (proj, supportedArches); + + using var builder = CreateApkBuilder (); + Assert.IsTrue (builder.Build (proj), "Build should have succeeded."); + + string objDirPath = Path.Combine (Root, builder.ProjectDirectory, proj.IntermediateOutputPath); + List envFiles = EnvironmentHelper.GatherEnvironmentFiles ( + objDirPath, + String.Join (";", supportedArches.Select (arch => MonoAndroidHelper.ArchToAbi (arch))), + true + ); + + EnvironmentHelper.IApplicationConfig app_config = EnvironmentHelper.ReadApplicationConfig (envFiles, runtime); + uint numberOfDsoCacheEntries = runtime switch { + AndroidRuntime.MonoVM => ((EnvironmentHelper.ApplicationConfig_MonoVM)app_config).number_of_dso_cache_entries, + AndroidRuntime.CoreCLR => ((EnvironmentHelper.ApplicationConfig_CoreCLR)app_config).number_of_dso_cache_entries, + _ => throw new NotSupportedException ($"Unsupported runtime '{runtime}'") + }; + + return EnvironmentHelper.ReadJniPreloads (envFiles, numberOfDsoCacheEntries, runtime); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs index 17c9293cf80..927e4ccad70 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs @@ -85,9 +85,9 @@ public sealed class ApplicationConfig_MonoVM : IApplicationConfig public uint system_property_count; public uint number_of_assemblies_in_apk; public uint bundled_assembly_name_width; - public uint number_of_assembly_store_files; public uint number_of_dso_cache_entries; public uint number_of_aot_cache_entries; + public uint number_of_shared_libraries; public uint android_runtime_jnienv_class_token; public uint jnienv_initialize_method_token; public uint jnienv_registerjninatives_method_token; @@ -98,6 +98,37 @@ public sealed class ApplicationConfig_MonoVM : IApplicationConfig public bool managed_marshal_methods_lookup_enabled; } + // This is shared between MonoVM and CoreCLR hosts, not used by NativeAOT + public sealed class DSOCacheEntry64 + { + // Hardcoded, by design - we want to know if there are any changes in the + // native assembly layout. + public const uint NativeSize_CoreCLR = 32; + public const uint NativeSize_MonoVM = 40; + + public ulong hash; + public ulong real_name_hash; + public bool ignore; + public bool is_jni_library; + public string name; // real structure has an index here, we fetch the string to make it easier + public IntPtr handle; + } + + // This is a synthetic class, not reflecting what's in the generated LLVM IR/assembler source + public sealed class JniPreloadsEntry + { + public uint Index; + public string LibraryName; + } + + // This is a synthetic class, not reflecting what's in the generated LLVM IR/assembler source + public sealed class JniPreloads + { + public uint IndexStride; + public List Entries; + public string SourceFile; + } + const uint ApplicationConfigFieldCount_MonoVM = 27; const string ApplicationConfigSymbolName = "application_config"; @@ -107,6 +138,12 @@ public sealed class ApplicationConfig_MonoVM : IApplicationConfig const string AppEnvironmentVariablesNativeAOTSymbolName = "__naot_android_app_environment_variables"; const string AppEnvironmentVariableContentsNativeAOTSymbolName = "__naot_android_app_environment_variable_contents"; + const string DsoJniPreloadsIdxStrideSymbolName = "dso_jni_preloads_idx_stride"; + const string DsoJniPreloadsIdxCountSymbolName = "dso_jni_preloads_idx_count"; + const string DsoJniPreloadsIdxSymbolName = "dso_jni_preloads_idx"; + const string DsoCacheSymbolName = "dso_cache"; + const string DsoNamesDataSymbolName = "dso_names_data"; + static readonly object ndkInitLock = new object (); static readonly char[] readElfFieldSeparator = new [] { ' ', '\t' }; static readonly Regex assemblerLabelRegex = new Regex ("^[_.a-zA-Z0-9]+:", RegexOptions.Compiled); @@ -122,6 +159,11 @@ public sealed class ApplicationConfig_MonoVM : IApplicationConfig ".long", }; + static readonly HashSet expectedUInt64Types = new HashSet (StringComparer.Ordinal) { + ".xword", + ".quad", + }; + static readonly string[] requiredSharedLibrarySymbolsMonoVM = { AppEnvironmentVariablesSymbolName, "app_system_properties", @@ -416,7 +458,7 @@ static IApplicationConfig ReadApplicationConfig_MonoVM (EnvironmentFile envFile) case 2: // aot_lazy_load: bool / .byte AssertFieldType (envFile.Path, parser.SourceFilePath, ".byte", field [0], item.LineNumber); - ret.uses_mono_aot = ConvertFieldToBool ("aot_lazy_load", envFile.Path, parser.SourceFilePath, item.LineNumber, field [1]); + ret.aot_lazy_load = ConvertFieldToBool ("aot_lazy_load", envFile.Path, parser.SourceFilePath, item.LineNumber, field [1]); break; case 3: // uses_assembly_preload: bool / .byte @@ -484,19 +526,19 @@ static IApplicationConfig ReadApplicationConfig_MonoVM (EnvironmentFile envFile) ret.bundled_assembly_name_width = ConvertFieldToUInt32 ("bundled_assembly_name_width", envFile.Path, parser.SourceFilePath, item.LineNumber, field [1]); break; - case 16: // number_of_assembly_store_files: uint32_t / .word | .long + case 16: // number_of_dso_cache_entries: uint32_t / .word | .long Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile.Path}:{item.LineNumber}': {field [0]}"); - ret.number_of_assembly_store_files = ConvertFieldToUInt32 ("number_of_assembly_store_files", envFile.Path, parser.SourceFilePath, item.LineNumber, field [1]); + ret.number_of_dso_cache_entries = ConvertFieldToUInt32 ("number_of_dso_cache_entries", envFile.Path, parser.SourceFilePath, item.LineNumber, field [1]); break; - case 17: // number_of_dso_cache_entries: uint32_t / .word | .long + case 17: // number_of_aot_cache_entries: uint32_t / .word | .long Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile.Path}:{item.LineNumber}': {field [0]}"); - ret.number_of_dso_cache_entries = ConvertFieldToUInt32 ("number_of_dso_cache_entries", envFile.Path, parser.SourceFilePath, item.LineNumber, field [1]); + ret.number_of_aot_cache_entries = ConvertFieldToUInt32 ("number_of_aot_cache_entries", envFile.Path, parser.SourceFilePath, item.LineNumber, field [1]); break; - case 18: // number_of_aot_cache_entries: uint32_t / .word | .long + case 18: // number_of_shared_libraries: uint32_t / .word | long Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile.Path}:{item.LineNumber}': {field [0]}"); - ret.number_of_aot_cache_entries = ConvertFieldToUInt32 ("number_of_aot_cache_entries", envFile.Path, parser.SourceFilePath, item.LineNumber, field [1]); + ret.number_of_shared_libraries = ConvertFieldToUInt32 ("number_of_shared_libraries", envFile.Path, parser.SourceFilePath, item.LineNumber, field [1]); break; case 19: // android_runtime_jnienv_class_token: uint32_t / .word | .long @@ -596,21 +638,8 @@ static Dictionary ReadEnvironmentVariables_CoreCLR_NativeAOT (En Assert.IsTrue (appEnvvarsContentsSymbol.Size != 0, $"{envvarsContentsSymbolName} size as specified in the '.size' directive must not be 0"); Assert.IsTrue (appEnvvarsContentsSymbol.Contents.Count == 1, $"{envvarsContentsSymbolName} symbol must have a single value."); - NativeAssemblyParser.AssemblerSymbolItem contentsItem = appEnvvarsContentsSymbol.Contents[0]; - string[] field = GetField (envFile.Path, parser.SourceFilePath, contentsItem.Contents, contentsItem.LineNumber);; - Assert.IsTrue (field[0] == ".asciz", $"{envvarsContentsSymbolName} must be of '.asciz' type"); - - - var sb = new StringBuilder (); - // We need to get rid of the '"' delimiter llc outputs.. - sb.Append (field[1].Trim ('"')); - - // ...and llc outputs NUL as the octal '\000' sequence, we need an actual NUL... - sb.Replace ("\\000", "\0"); - - // ...and since it's an .asciz variable, the string doesn't contain explicit terminating NUL, but we need one - sb.Append ('\0'); - string contents = sb.ToString (); + string[] field; + string contents = ReadStringBlob (envFile, appEnvvarsContentsSymbol, parser); var indexes = new List<(uint nameIdx, uint valueIdx)> (); // Environment variables are pairs of indexes into the contents array @@ -623,11 +652,16 @@ static Dictionary ReadEnvironmentVariables_CoreCLR_NativeAOT (En // Contents array is a collection of strings terminated with the NUL character var ret = new Dictionary (StringComparer.Ordinal); + + const string ContentsAssertionTag = "Environment Variables"; foreach (var envvar in indexes) { Assert.IsTrue (envvar.nameIdx < appEnvvarsContentsSymbol.Size, $"Environment variable name index {envvar.nameIdx} is out of range of the contents array"); Assert.IsTrue (envvar.valueIdx < appEnvvarsContentsSymbol.Size, $"Environment variable value index {envvar.valueIdx} is out of range of the contents array"); - ret.Add (GetFromContents (envvar.nameIdx), GetFromContents (envvar.valueIdx)); + ret.Add ( + GetStringFromBlobContents (ContentsAssertionTag, contents, envvar.nameIdx), + GetStringFromBlobContents (ContentsAssertionTag, contents, envvar.valueIdx) + ); } return ret; @@ -640,23 +674,6 @@ uint GetIndex (NativeAssemblyParser.AssemblerSymbolItem item, string name) Assert.IsTrue (expectedUInt32Types.Contains (field[0]), $"Environment variable {name} index field has invalid type '${field[0]}'"); return UInt32.Parse (field[1], CultureInfo.InvariantCulture); } - - string GetFromContents (uint idx) - { - var sb = new StringBuilder (); - bool foundNull = false; - - for (int i = (int)idx; i < contents.Length; i++) { - if (contents[i] == '\0') { - foundNull = true; - break; - } - sb.Append (contents[i]); - } - - Assert.IsTrue (foundNull, $"Environment variable contents string starting at index {idx} is not NUL-terminated"); - return sb.ToString (); - } } static Dictionary ReadEnvironmentVariables_MonoVM (EnvironmentFile envFile) @@ -921,6 +938,270 @@ static void AssertSharedLibraryHasRequiredSymbols (string dsoPath, string readEl } } + public static List ReadJniPreloads (List envFilePaths, uint expectedDsoCacheEntryCount, AndroidRuntime runtime) + { + var ret = new List (); + + foreach (EnvironmentFile envFile in envFilePaths) { + JniPreloads preloads = runtime switch { + AndroidRuntime.CoreCLR => ReadJniPreloads_CoreCLR (envFile, expectedDsoCacheEntryCount), + AndroidRuntime.MonoVM => ReadJniPreloads_MonoVM (envFile, expectedDsoCacheEntryCount), + _ => throw new NotSupportedException ($"Unsupported runtime '{runtime}'") + }; + + ret.Add (preloads); + } + + return ret; + } + + delegate List ReadDsoCacheFn (NativeAssemblyParser parser, EnvironmentFile envFile, NativeAssemblyParser.AssemblerSymbol dsoCacheSym); + + static JniPreloads ReadJniPreloads_Common (EnvironmentFile envFile, uint expectedDsoCacheEntryCount, uint dsoCacheEntrySize, ReadDsoCacheFn dsoReader) + { + NativeAssemblyParser parser = CreateAssemblyParser (envFile); + + NativeAssemblyParser.AssemblerSymbol dsoCache = GetNonEmptyRequiredSymbol (parser, envFile, DsoCacheSymbolName); + uint calculatedDsoCacheEntryCount = (uint)(dsoCache.Size / dsoCacheEntrySize); + Assert.IsTrue (calculatedDsoCacheEntryCount == expectedDsoCacheEntryCount, $"Calculated DSO cache entry count should be {expectedDsoCacheEntryCount} but was {calculatedDsoCacheEntryCount} instead."); + + uint calculatedDsoCacheEntrySize = (uint)(dsoCacheEntrySize * expectedDsoCacheEntryCount); + Assert.IsTrue (calculatedDsoCacheEntrySize == dsoCache.Size, $"Calculated DSO cache size should be {dsoCache.Size} but was {calculatedDsoCacheEntrySize} instead."); + + List dsoCacheEntries = dsoReader (parser, envFile, dsoCache); + Assert.IsTrue ((uint)dsoCacheEntries.Count == expectedDsoCacheEntryCount, $"DSO cache read from the source should have {expectedDsoCacheEntryCount} entries, it had {dsoCacheEntries.Count} instead."); + + NativeAssemblyParser.AssemblerSymbol dsoJniPreloadsIdxStride = GetNonEmptyRequiredSymbol (parser, envFile, DsoJniPreloadsIdxStrideSymbolName); + uint preloadsStride = GetSymbolValueAsUInt32 (dsoJniPreloadsIdxStride); + Assert.IsTrue (preloadsStride > 0, $"Symbol {dsoJniPreloadsIdxStride.Name} must have value larger than 0."); + + NativeAssemblyParser.AssemblerSymbol dsoJniPreloadsIdxCount = GetNonEmptyRequiredSymbol (parser, envFile, DsoJniPreloadsIdxCountSymbolName); + ulong preloadsCount = GetSymbolValueAsUInt64 (dsoJniPreloadsIdxCount); + Assert.IsTrue (preloadsCount > 0, $"Symbol {dsoJniPreloadsIdxCount.Name} must have value larger than 0."); + + NativeAssemblyParser.AssemblerSymbol dsoJniPreloadsIdx = GetNonEmptyRequiredSymbol (parser, envFile, DsoJniPreloadsIdxSymbolName); + ulong calculatedPreloadsIdxSize = preloadsCount * 4; // single index field is a 32-bit integer + Assert.IsTrue (dsoJniPreloadsIdx.Size == calculatedPreloadsIdxSize, $"JNI preloads index should have size of {calculatedPreloadsIdxSize} instead of {dsoJniPreloadsIdx.Size}"); + + var preloadsIndex = new List (); + for (int i = 0; i < (int)preloadsCount; i++) { + (ulong lineNumber, string value) = ReadNextArrayIndex (envFile, parser, dsoJniPreloadsIdx, i, expectedUInt32Types); + uint index = ConvertFieldToUInt32 ("index", envFile.Path, parser.SourceFilePath, lineNumber, value); + + Assert.True (index < (uint)dsoCacheEntries.Count, $"JNI preload index {index} is larger than the number of items in the DSO cache array ({dsoCacheEntries.Count})"); + preloadsIndex.Add ( + new JniPreloadsEntry { + Index = index, + LibraryName = dsoCacheEntries[(int)index].name, + } + ); + } + Assert.IsTrue (preloadsCount == (uint)preloadsIndex.Count, $"JNI preload index count should be equal to {preloadsCount}, but was {preloadsIndex.Count} instead."); + + return new JniPreloads { + IndexStride = preloadsStride, + Entries = preloadsIndex, + SourceFile = envFile.Path, + }; + + uint GetSymbolValueAsUInt32 (NativeAssemblyParser.AssemblerSymbol symbol) + { + NativeAssemblyParser.AssemblerSymbolItem item = symbol.Contents[0]; + string[] field = GetField (envFile.Path, parser.SourceFilePath, item.Contents, item.LineNumber); + Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected 32-bit integer field type for symbol {symbol.Name} in '{envFile.Path}:{item.LineNumber}': {field [0]}"); + return ConvertFieldToUInt32 (DsoJniPreloadsIdxStrideSymbolName, envFile.Path, parser.SourceFilePath, item.LineNumber, field[1]); + } + + ulong GetSymbolValueAsUInt64 (NativeAssemblyParser.AssemblerSymbol symbol) + { + NativeAssemblyParser.AssemblerSymbolItem item = symbol.Contents[0]; + string[] field = GetField (envFile.Path, parser.SourceFilePath, item.Contents, item.LineNumber); + Assert.IsTrue (expectedUInt64Types.Contains (field [0]), $"Unexpected 64-bit integer field type for symbol {symbol.Name} in '{envFile.Path}:{item.LineNumber}': {field [0]}"); + return ConvertFieldToUInt64 (DsoJniPreloadsIdxStrideSymbolName, envFile.Path, parser.SourceFilePath, item.LineNumber, field[1]); + } + } + + static NativeAssemblyParser.AssemblerSymbol GetNonEmptyRequiredSymbol (NativeAssemblyParser parser, EnvironmentFile envFile, string symbolName) + { + var symbol = GetRequiredSymbol (symbolName, envFile, parser); + + Assert.IsTrue (symbol.Size != 0, $"{symbolName} size as specified in the '.size' directive must not be 0"); + return symbol; + } + + static JniPreloads ReadJniPreloads_MonoVM (EnvironmentFile envFile, uint expectedDsoCacheEntryCount) + { + return ReadJniPreloads_Common ( + envFile, + expectedDsoCacheEntryCount, + DSOCacheEntry64.NativeSize_MonoVM, + (NativeAssemblyParser parser, EnvironmentFile envFile, NativeAssemblyParser.AssemblerSymbol dsoCacheSym) => { + return ReadDsoCache64_MonoVM (envFile, parser, dsoCacheSym); + } + ); + } + + static List ReadDsoCache64_MonoVM (EnvironmentFile envFile, NativeAssemblyParser parser, NativeAssemblyParser.AssemblerSymbol dsoCache) + { + var ret = new List (); + + // This follows a VERY strict format, by design. If anything changes in the generated source this is supposed + // to break. + // + // The code is almost identical to that of CoreCLR, but it is kept completely separate on purpose - it makes the code simpler, since + // it doesn't have to account for the small differences between runtimes and it also provides for independence of the two runtime + // hosts. + const int itemsPerEntry = 7; // Includes padding entries + for (int i = 0; i < dsoCache.Contents.Count; i += itemsPerEntry) { + ulong lineNumber; + string value; + int index = i; + + // uint64_t hash + (lineNumber, value) = ReadNextArrayIndex (envFile, parser, dsoCache, index++, expectedUInt64Types); + ulong hash = ConvertFieldToUInt64 ("hash", envFile.Path, parser.SourceFilePath, lineNumber, value); + + // uint64_t real_name_hash + (lineNumber, value) = ReadNextArrayIndex (envFile, parser, dsoCache, index++, expectedUInt64Types); + ulong real_name_hash = ConvertFieldToUInt64 ("real_name_hash", envFile.Path, parser.SourceFilePath, lineNumber, value); + + // bool ignore + (lineNumber, value) = ReadNextArrayIndex (envFile, parser, dsoCache, index++, ".byte"); + bool ignore = ConvertFieldToBool ("ignore", envFile.Path, parser.SourceFilePath, lineNumber, value); + + // bool is_jni_library + (lineNumber, value) = ReadNextArrayIndex (envFile, parser, dsoCache, index++, ".byte"); + bool is_jni_library = ConvertFieldToBool ("is_jni_library", envFile.Path, parser.SourceFilePath, lineNumber, value); + + // padding, 6 bytes + (lineNumber, value) = ReadNextArrayIndex (envFile, parser, dsoCache, index, ".zero"); + uint padding1 = ConvertFieldToUInt32 ("padding1", envFile.Path, parser.SourceFilePath, lineNumber, value); + Assert.IsTrue (padding1 == 6, $"Padding field #1 at index {index} of symbol '{dsoCache.Name}' should have had a value of 6, instead it was set to {padding1}"); + index++; + + // .pointer_type SYMBOL_NAME + (lineNumber, value) = ReadNextArrayIndex (envFile, parser, dsoCache, index++, expectedUInt64Types); + NativeAssemblyParser.AssemblerSymbol dsoLibNameSymbol = GetRequiredSymbol (value, envFile, parser); + + // void* handle + (lineNumber, value) = ReadNextArrayIndex (envFile, parser, dsoCache, index, expectedUInt64Types); + ulong handle = ConvertFieldToUInt64 ("handle", envFile.Path, parser.SourceFilePath, lineNumber, value); + Assert.IsTrue (handle == 0, $"Handle field at index {index} of symbol '{dsoCache.Name}' should have had a value of 0, instead it was set to {handle}"); + + string name = GetStringContents (dsoLibNameSymbol, envFile, parser); + + ret.Add ( + new DSOCacheEntry64 { + hash = hash, + real_name_hash = real_name_hash, + ignore = ignore, + is_jni_library = is_jni_library, + name = name, + handle = IntPtr.Zero, + } + ); + } + + return ret; + } + + static JniPreloads ReadJniPreloads_CoreCLR (EnvironmentFile envFile, uint expectedDsoCacheEntryCount) + { + return ReadJniPreloads_Common ( + envFile, + expectedDsoCacheEntryCount, + DSOCacheEntry64.NativeSize_CoreCLR, + (NativeAssemblyParser parser, EnvironmentFile envFile, NativeAssemblyParser.AssemblerSymbol dsoCacheSym) => { + NativeAssemblyParser.AssemblerSymbol dsoNamesData = GetNonEmptyRequiredSymbol (parser, envFile, DsoNamesDataSymbolName); + Assert.IsTrue (dsoNamesData.Size > 0, "DSO names data must have size larger than zero"); + + string dsoNames = ReadStringBlob (envFile, dsoNamesData, parser); + Assert.IsTrue (dsoNames.Length > 0, "DSO names read from source mustn't be empty"); + + return ReadDsoCache64_CoreCLR (envFile, parser, dsoCacheSym, dsoNames); + } + ); + } + + static List ReadDsoCache64_CoreCLR (EnvironmentFile envFile, NativeAssemblyParser parser, NativeAssemblyParser.AssemblerSymbol dsoCache, string dsoNamesBlob) + { + var ret = new List (); + + // This follows a VERY strict format, by design. If anything changes in the generated source this is supposed + // to break. + const int itemsPerEntry = 7; // Includes padding entries + for (int i = 0; i < dsoCache.Contents.Count; i += itemsPerEntry) { + ulong lineNumber; + string value; + int index = i; + + // uint64_t hash + (lineNumber, value) = ReadNextArrayIndex (envFile, parser, dsoCache, index++, expectedUInt64Types); + ulong hash = ConvertFieldToUInt64 ("hash", envFile.Path, parser.SourceFilePath, lineNumber, value); + + // uint64_t real_name_hash + (lineNumber, value) = ReadNextArrayIndex (envFile, parser, dsoCache, index++, expectedUInt64Types); + ulong real_name_hash = ConvertFieldToUInt64 ("real_name_hash", envFile.Path, parser.SourceFilePath, lineNumber, value); + + // bool ignore + (lineNumber, value) = ReadNextArrayIndex (envFile, parser, dsoCache, index++, ".byte"); + bool ignore = ConvertFieldToBool ("ignore", envFile.Path, parser.SourceFilePath, lineNumber, value); + + // bool is_jni_library + (lineNumber, value) = ReadNextArrayIndex (envFile, parser, dsoCache, index++, ".byte"); + bool is_jni_library = ConvertFieldToBool ("is_jni_library", envFile.Path, parser.SourceFilePath, lineNumber, value); + + // padding, 2 bytes + (lineNumber, value) = ReadNextArrayIndex (envFile, parser, dsoCache, index, ".zero"); + uint padding1 = ConvertFieldToUInt32 ("padding1", envFile.Path, parser.SourceFilePath, lineNumber, value); + Assert.IsTrue (padding1 == 2, $"Padding field #1 at index {index} of symbol '{dsoCache.Name}' should have had a value of 2, instead it was set to {padding1}"); + index++; + + // uint32_t name_index + (lineNumber, value) = ReadNextArrayIndex (envFile, parser, dsoCache, index++, expectedUInt32Types); + uint name_index = ConvertFieldToUInt32 ("name_index", envFile.Path, parser.SourceFilePath, lineNumber, value); + + // void* handle + (lineNumber, value) = ReadNextArrayIndex (envFile, parser, dsoCache, index, expectedUInt64Types); + ulong handle = ConvertFieldToUInt64 ("handle", envFile.Path, parser.SourceFilePath, lineNumber, value); + Assert.IsTrue (handle == 0, $"Handle field at index {index} of symbol '{dsoCache.Name}' should have had a value of 0, instead it was set to {handle}"); + + string name = GetStringFromBlobContents ("DSO JNI preloads", dsoNamesBlob, name_index); + ret.Add ( + new DSOCacheEntry64 { + hash = hash, + real_name_hash = real_name_hash, + ignore = ignore, + is_jni_library = is_jni_library, + name = name, + handle = IntPtr.Zero, + } + ); + } + + return ret; + } + + static (ulong line, string value) ReadNextArrayIndex (EnvironmentFile envFile, NativeAssemblyParser parser, NativeAssemblyParser.AssemblerSymbol array, int index, string expectedType) + { + return ReadNextArrayIndex (envFile, parser, array, index, new HashSet (StringComparer.Ordinal) { expectedType }); + } + + static (ulong line, string value) ReadNextArrayIndex (EnvironmentFile envFile, NativeAssemblyParser parser, NativeAssemblyParser.AssemblerSymbol array, + int index, HashSet expectedTypes) + { + Assert.IsFalse (index >= array.Contents.Count, $"Index {index} exceeds the number of items in the {array.Name} array."); + NativeAssemblyParser.AssemblerSymbolItem item = array.Contents[index]; + + string[] field = GetField (envFile.Path, parser.SourceFilePath, item.Contents, item.LineNumber); + Assert.IsTrue (field.Length == 2, $"Item {index} of symbol {array.Name} at {envFile.Path}:{item.LineNumber} has an invalid value."); + + string expectedTypesList = String.Join (" | ", expectedTypes); + Assert.IsTrue (expectedTypes.Contains (field[0]), $"Item {index} of symbol {array.Name} at {parser.SourceFilePath}:{item.LineNumber} should be of type '{expectedTypesList}', but was '{field[0]}' instead."); + + return (item.LineNumber, field[1]); + } + static (List stdout, List stderr) RunCommand (string executablePath, string arguments = null) { var psi = new ProcessStartInfo { @@ -1007,7 +1288,17 @@ static uint ConvertFieldToUInt32 (string fieldName, string llvmAssemblerEnvFile, Assert.IsTrue (value.Length > 0, $"Field '{fieldName}' in {nativeAssemblerEnvFile}:{fileLine} is not a valid uint32_t value (not long enough). File generated from '{llvmAssemblerEnvFile}'"); uint fv; - Assert.IsTrue (TryParseInteger (value, out fv), $"Field '{fieldName}' in {nativeAssemblerEnvFile}:{fileLine} is not a valid uint32_t value (not a valid integer). File generated from '{llvmAssemblerEnvFile}'"); + Assert.IsTrue (TryParseInteger (value, out fv), $"Field '{fieldName}' in {nativeAssemblerEnvFile}:{fileLine} is not a valid uint32_t value ('{value}' is not a valid integer). File generated from '{llvmAssemblerEnvFile}'"); + + return fv; + } + + static ulong ConvertFieldToUInt64 (string fieldName, string llvmAssemblerEnvFile, string nativeAssemblerEnvFile, ulong fileLine, string value) + { + Assert.IsTrue (value.Length > 0, $"Field '{fieldName}' in {nativeAssemblerEnvFile}:{fileLine} is not a valid uint64_t value (not long enough). File generated from '{llvmAssemblerEnvFile}'"); + + ulong fv; + Assert.IsTrue (TryParseInteger (value, out fv), $"Field '{fieldName}' in {nativeAssemblerEnvFile}:{fileLine} is not a valid uint64_t value ('{value}' is not a valid integer). File generated from '{llvmAssemblerEnvFile}'"); return fv; } @@ -1017,18 +1308,40 @@ static byte ConvertFieldToByte (string fieldName, string llvmAssemblerEnvFile, s Assert.IsTrue (value.Length > 0, $"Field '{fieldName}' in {nativeAssemblerEnvFile}:{fileLine} is not a valid uint8_t value (not long enough). File generated from '{llvmAssemblerEnvFile}'"); byte fv; - Assert.IsTrue (TryParseInteger (value, out fv), $"Field '{fieldName}' in {nativeAssemblerEnvFile}:{fileLine} is not a valid uint8_t value (not a valid integer). File generated from '{llvmAssemblerEnvFile}'"); + Assert.IsTrue (TryParseInteger (value, out fv), $"Field '{fieldName}' in {nativeAssemblerEnvFile}:{fileLine} is not a valid uint8_t value ('{value}' is not a valid integer). File generated from '{llvmAssemblerEnvFile}'"); return fv; } + // Integers are parsed as signed, since llc will always output signed integers. static bool TryParseInteger (string value, out uint fv) { if (value.StartsWith ("0x", StringComparison.Ordinal)) { return UInt32.TryParse (value.Substring (2), NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out fv); } - return UInt32.TryParse (value, out fv); + fv = 0; + if (!Int32.TryParse (value, out int signedFV)) { + return false; + } + + fv = (uint)signedFV; + return true; + } + + static bool TryParseInteger (string value, out ulong fv) + { + if (value.StartsWith ("0x", StringComparison.Ordinal)) { + return UInt64.TryParse (value.Substring (2), NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out fv); + } + + fv = 0; + if (!Int64.TryParse (value, out long signedFV)) { + return false; + } + + fv = (ulong)signedFV; + return true; } static bool TryParseInteger (string value, out byte fv) @@ -1039,5 +1352,41 @@ static bool TryParseInteger (string value, out byte fv) return Byte.TryParse (value, out fv); } + + static string ReadStringBlob (EnvironmentFile envFile, NativeAssemblyParser.AssemblerSymbol contentsSymbol, NativeAssemblyParser parser) + { + NativeAssemblyParser.AssemblerSymbolItem contentsItem = contentsSymbol.Contents[0]; + string[] field = GetField (envFile.Path, parser.SourceFilePath, contentsItem.Contents, contentsItem.LineNumber); + Assert.IsTrue (field[0] == ".asciz", $"{contentsSymbol.Name} must be of '.asciz' type"); + + var sb = new StringBuilder (); + // We need to get rid of the '"' delimiter llc outputs.. + sb.Append (field[1].Trim ('"')); + + // ...and llc outputs NUL as the octal '\000' sequence, we need an actual NUL... + sb.Replace ("\\000", "\0"); + + // ...and since it's an .asciz variable, the string doesn't contain explicit terminating NUL, but we need one + sb.Append ('\0'); + + return sb.ToString (); + } + + static string GetStringFromBlobContents (string assertionTag, string contents, uint idx) + { + var sb = new StringBuilder (); + bool foundNull = false; + + for (int i = (int)idx; i < contents.Length; i++) { + if (contents[i] == '\0') { + foundNull = true; + break; + } + sb.Append (contents[i]); + } + + Assert.IsTrue (foundNull, $"[{assertionTag} string starting at index {idx} of a string blob is not NUL-terminated"); + return sb.ToString (); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/AndroidBuildActions.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/AndroidBuildActions.cs index ac943cb571d..7b8721e418e 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/AndroidBuildActions.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/AndroidBuildActions.cs @@ -27,102 +27,107 @@ public static class AndroidBuildActions /// Build action for Android resource files (layout, drawable, values, etc.). /// public const string AndroidResource = "AndroidResource"; - + /// /// Build action for Android asset files stored in the assets folder. /// public const string AndroidAsset = "AndroidAsset"; - + /// /// Build action for Android AAR library files. /// public const string AndroidAarLibrary = "AndroidAarLibrary"; - + /// /// Build action for Android environment variable files. /// public const string AndroidEnvironment = "AndroidEnvironment"; - + /// /// Build action for Android interface description language files. /// public const string AndroidInterfaceDescription = "AndroidInterfaceDescription"; - + /// /// Build action for Java source files to be compiled as part of the Android project. /// public const string AndroidJavaSource = "AndroidJavaSource"; - + /// /// Build action for Java library JAR files. /// public const string AndroidJavaLibrary = "AndroidJavaLibrary"; - + /// /// Build action for Android library project references. /// public const string AndroidLibrary = "AndroidLibrary"; - + /// /// Build action for Maven-based Android library dependencies. /// public const string AndroidMavenLibrary = "AndroidMavenLibrary"; - + /// /// Build action for Android Lint configuration files. /// public const string AndroidLintConfig = "AndroidLintConfig"; - + /// /// Build action for native library files (.so) for Android. /// public const string AndroidNativeLibrary = "AndroidNativeLibrary"; - + + /// + /// Represents a native JNI library which should not be preloaded at application startup. + /// + public const string AndroidNativeLibraryNoJniPreload = "AndroidNativeLibraryNoJniPreload"; + /// /// Internal build action for Android member remapping metadata. /// public const string _AndroidRemapMembers = "_AndroidRemapMembers"; - + /// /// Build action for ProGuard configuration files. /// public const string ProguardConfiguration = "ProguardConfiguration"; - + /// /// Build action for XML transform files. /// public const string TransformFile = "TransformFile"; - + /// /// Build action for input JAR files in Java binding projects. /// public const string InputJar = "InputJar"; - + /// /// Build action for reference JAR files in Java binding projects. /// public const string ReferenceJar = "ReferenceJar"; - + /// /// Build action for embedded JAR files in Java binding projects. /// public const string EmbeddedJar = "EmbeddedJar"; - + /// /// Build action for embedded native library files. /// public const string EmbeddedNativeLibrary = "EmbeddedNativeLibrary"; - + /// /// Build action for embedded reference JAR files in Java binding projects. /// public const string EmbeddedReferenceJar = "EmbeddedReferenceJar"; - + /// /// Build action for Android library project ZIP files. /// public const string LibraryProjectZip = "LibraryProjectZip"; - + /// /// Build action for Android library project properties files. /// diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/AndroidItem.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/AndroidItem.cs index 8e9e30e98af..d9a201ebd72 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/AndroidItem.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/AndroidItem.cs @@ -1,4 +1,4 @@ -using System; +using System; namespace Xamarin.ProjectTools { @@ -156,6 +156,16 @@ public AndroidNativeLibrary (Func include) { } } + public class AndroidNativeLibraryNoJniPreload : BuildItem { + public AndroidNativeLibraryNoJniPreload (string include) + : this (() => include) + { + } + public AndroidNativeLibraryNoJniPreload (Func include) + : base (AndroidBuildActions.AndroidNativeLibraryNoJniPreload, include) + { + } + } public class AndroidJavaSource : BuildItem { public AndroidJavaSource (string include) : this (() => include) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index 62b7a0361a7..53d0c4646a6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -202,6 +202,8 @@ sealed class DsoCacheState public MonoComponent MonoComponents { get; set; } public PackageNamingPolicy PackageNamingPolicy { get; set; } public List NativeLibraries { get; set; } = []; + public ICollection? NativeLibrariesNoJniPreload { get; set; } + public ICollection? NativeLibrarysAlwaysJniPreload { get; set; } public bool MarshalMethodsEnabled { get; set; } public bool ManagedMarshalMethodsLookupEnabled { get; set; } public bool IgnoreSplitConfigs { get; set; } @@ -450,13 +452,14 @@ DsoCacheState InitDSOCache () var aotDsoCache = new List> (); var nameMutations = new List (); int nameMutationsCount = -1; + ICollection ignorePreload = ApplicationConfigNativeAssemblyGeneratorCLR.MakeJniPreloadIgnoreCollection (Log, NativeLibrarysAlwaysJniPreload, NativeLibrariesNoJniPreload); for (int i = 0; i < dsos.Count; i++) { string name = dsos[i].name; bool isJniLibrary = ELFHelper.IsJniLibrary (Log, dsos[i].item.ItemSpec); bool ignore = dsos[i].ignore; - bool ignore_for_preload = !ApplicationConfigNativeAssemblyGeneratorCLR.DsoCacheJniPreloadIgnore.Contains (name); + bool ignore_for_preload = ApplicationConfigNativeAssemblyGeneratorCLR.ShouldIgnoreForJniPreload (Log, ignorePreload, dsos[i].item); nameMutations.Clear(); AddNameMutations (name); @@ -482,7 +485,7 @@ DsoCacheState InitDSOCache () // We must add all aliases to the preloads indices array so that all of them have their handle // set when the library is preloaded. - if (entry.is_jni_library && ignore_for_preload) { + if (entry.is_jni_library && !ignore_for_preload) { jniPreloads.Add (entry); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGeneratorCLR.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGeneratorCLR.cs index 76ccaf37a0d..2f62ac2a456 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGeneratorCLR.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGeneratorCLR.cs @@ -6,6 +6,7 @@ using System.IO; using Java.Interop.Tools.TypeNameMappings; +using Microsoft.Android.Build.Tasks; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Xamarin.Android.Tasks.LLVMIR; @@ -281,6 +282,8 @@ sealed class DsoCacheState public int JniRemappingReplacementMethodIndexEntryCount { get; set; } public PackageNamingPolicy PackageNamingPolicy { get; set; } public List NativeLibraries { get; set; } = []; + public ICollection? NativeLibrariesNoJniPreload { get; set; } + public ICollection? NativeLibrarysAlwaysJniPreload { get; set; } public bool MarshalMethodsEnabled { get; set; } public bool ManagedMarshalMethodsLookupEnabled { get; set; } public bool IgnoreSplitConfigs { get; set; } @@ -683,6 +686,7 @@ DsoCacheState InitDSOCache () var nameMutations = new List (); var dsoNamesBlob = new LlvmIrStringBlob (); int nameMutationsCount = -1; + ICollection ignorePreload = MakeJniPreloadIgnoreCollection (Log, NativeLibrarysAlwaysJniPreload, NativeLibrariesNoJniPreload); for (int i = 0; i < dsos.Count; i++) { string name = dsos[i].name; @@ -690,7 +694,7 @@ DsoCacheState InitDSOCache () bool isJniLibrary = ELFHelper.IsJniLibrary (Log, dsos[i].item.ItemSpec); bool ignore = dsos[i].ignore; - bool ignore_for_preload = !DsoCacheJniPreloadIgnore.Contains (name); + bool ignore_for_preload = ShouldIgnoreForJniPreload (Log, ignorePreload, dsos[i].item); nameMutations.Clear(); AddNameMutations (name); @@ -718,7 +722,7 @@ DsoCacheState InitDSOCache () // We must add all aliases to the preloads indices array so that all of them have their handle // set when the library is preloaded. - if (entry.is_jni_library && ignore_for_preload) { + if (entry.is_jni_library && !ignore_for_preload) { jniPreloads.Add (entry); } @@ -775,4 +779,72 @@ void MapStructures (LlvmIrModule module) runtimePropertyIndexEntryStructureInfo = module.MapStructure (); appEnvironmentVariableStructureInfo = module.MapStructure (); } + + internal static bool ShouldIgnoreForJniPreload (TaskLoggingHelper log, ICollection libsToIgnore, ITaskItem libItem) + { + if (libsToIgnore.Count == 0) { + return false; + } + + string? libFileName = GetFileName (log, libItem); + if (libFileName == null) { + return false; // We have no idea what it is, so let the caller handle the situation + } + + return libsToIgnore.Contains (libFileName); + } + + internal static ICollection MakeJniPreloadIgnoreCollection (TaskLoggingHelper log, ICollection? alwaysPreload, ICollection? ignorePreload) + { + // There Can Be Only One, no matter what name casing is on the user's build OS. + var libsToIgnore = new HashSet (StringComparer.OrdinalIgnoreCase); + if (ignorePreload == null || ignorePreload.Count == 0) { + return libsToIgnore; + } + + var neverIgnore = new HashSet (StringComparer.OrdinalIgnoreCase); + + string? fileName; + if (alwaysPreload != null) { + foreach (ITaskItem item in alwaysPreload) { + fileName = GetFileName (log, item); + if (fileName == null) { + continue; + } + + neverIgnore.Add (fileName); + } + } + + foreach (ITaskItem item in ignorePreload) { + fileName = GetFileName (log, item); + if (fileName == null) { + continue; + } + + if (neverIgnore.Contains (fileName)) { + log.LogDebugMessage ($"Native library '{item.ItemSpec}' cannot be ignored when preloading JNI native libraries."); + continue; + } + + libsToIgnore.Add (fileName); + } + + return libsToIgnore; + } + + static string? GetFileName (TaskLoggingHelper log, ITaskItem item) + { + string? name = item.GetMetadata ("ArchiveFileName"); + if (String.IsNullOrEmpty (name)) { + name = MonoAndroidHelper.GetNormalizedNativeLibraryName (item); + } + + if (String.IsNullOrEmpty (name)) { + log.LogDebugMessage ($"Failed to convert item path '{item.ItemSpec}' to canonical native shared library name."); + return null; + } + + return name; + } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index ac4a8f888ea..4163bcb6276 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -886,5 +886,29 @@ public static int GetMinimumApiLevel (AndroidTargetArch arch, AndroidRuntime run return minValue; } + + /// + /// Takes `libItem.ItemSpec` and transforms it to a file name of a native library. It will + /// remove any paths from `ItemSpec` and will make sure that the file ends with the `.so` + /// extension. Empty string is returned if there's nothing to process. String comparisons + /// are ordinal and case-insensitive. + /// + public static string GetNormalizedNativeLibraryName (ITaskItem libItem) + { + if (String.IsNullOrEmpty (libItem.ItemSpec)) { + return String.Empty; + } + + string ret = Path.GetFileName (libItem.ItemSpec); + if (String.IsNullOrEmpty (ret)) { + return String.Empty; + } + + if (!String.Equals (Path.GetExtension (ret), ".so", StringComparison.OrdinalIgnoreCase)) { + return $"{ret}.so"; + } + + return ret; + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 30e7b9d3813..180eb879338 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -291,6 +291,8 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. <_AndroidStripNativeLibraries Condition=" '$(AndroidStripNativeLibraries)' != '' And '$(AndroidStripNativeLibraries)' == 'true' ">true <_AndroidStripNativeLibraries Condition=" '$(_AndroidStripNativeLibraries)' != 'true' ">false + + false @@ -1693,10 +1695,38 @@ because xbuild doesn't support framework reference assemblies. OutputDirectory="$(_AndroidIntermediateJavaSourceDirectory)mono"> + + + + <_AndroidNativeLibraryAlwaysJniPreload Include="libSystem.Security.Cryptography.Native.Android.so" /> + + + <_AndroidNativeLibraryNeverJniPreload Include="libmonodroid.so" /> + + + + <_AllNativeLibraries Include="@(AndroidNativeLibrary);@(EmbeddedNativeLibrary);@(FrameworkNativeLibrary)" /> + + + + + + + :${COMMON_CXX_ARGS}>") add_compile_options("$<$:${COMMON_C_ARGS}>") @@ -641,6 +642,8 @@ endmacro() if(BUILD_ARCHIVE_DSO_STUB) add_subdirectory(common/archive-dso-stub) +elseif(BUILD_TEST_JNI_LIBRARY) + add_subdirectory(common/test-jni-library) else() add_subdirectory(common/java-interop) diff --git a/src/native/CMakePresets.json.in b/src/native/CMakePresets.json.in index 5eca3fc4c0a..fb5c77267de 100644 --- a/src/native/CMakePresets.json.in +++ b/src/native/CMakePresets.json.in @@ -23,7 +23,8 @@ "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", "CMAKE_MAKE_PROGRAM": "@NinjaPath@", "XA_LIB_TOP_DIR": "@MicrosoftAndroidSdkOutDir@", - "XA_BUILD_CONFIGURATION": "@XA_BUILD_CONFIGURATION@" + "XA_BUILD_CONFIGURATION": "@XA_BUILD_CONFIGURATION@", + "XA_TEST_OUTPUT_DIR": "@XA_TEST_OUTPUT_DIR@" } }, diff --git a/src/native/common/test-jni-library/CMakeLists.txt b/src/native/common/test-jni-library/CMakeLists.txt new file mode 100644 index 00000000000..ca1f9a99f98 --- /dev/null +++ b/src/native/common/test-jni-library/CMakeLists.txt @@ -0,0 +1,38 @@ +set(LIB_NAME test-jni-library) + +set(LIB_SOURCES + stub.cc +) + +add_library( + ${LIB_NAME} + SHARED + ${LIB_SOURCES} +) + +set_target_properties( + ${LIB_NAME} + PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${XA_TEST_OUTPUT_DIR}/${ANDROID_RID}" +) + +target_compile_options( + ${LIB_NAME} + PRIVATE + ${XA_DEFAULT_SYMBOL_VISIBILITY} + ${XA_COMMON_CXX_ARGS} + -nostdlib -fno-exceptions -fno-rtti +) + +target_link_options( + ${LIB_NAME} + PRIVATE + ${XA_COMMON_CXX_LINKER_ARGS} + -nostdlib -fno-exceptions -fno-rtti -s +) + +add_custom_command( + TARGET ${LIB_NAME} + POST_BUILD + COMMAND ${CMAKE_STRIP} "$" +) diff --git a/src/native/common/test-jni-library/stub.cc b/src/native/common/test-jni-library/stub.cc new file mode 100644 index 00000000000..cfc427958be --- /dev/null +++ b/src/native/common/test-jni-library/stub.cc @@ -0,0 +1,8 @@ +#include + +JNIEXPORT jint JNICALL +JNI_OnLoad ([[maybe_unused]] JavaVM *vm, [[maybe_unused]] void *reserved) +{ + // no-op, need just the JNI_OnLoad symbol to be present for JNI preload tests + return JNI_VERSION_1_6; +} diff --git a/src/native/native.targets b/src/native/native.targets index d046fe7d40a..09472a6ab47 100644 --- a/src/native/native.targets +++ b/src/native/native.targets @@ -9,7 +9,7 @@ + DependsOnTargets="_PrepareCommonProperties;_ConfigureAndBuildArchiveDSOStub;_ConfigureAndBuildTestJniLibrary;_ConfigureRuntimes;_BuildAndroidRuntimes;_BuildAndroidAnalyzerRuntimes;_CopyToPackDirs">